Merge pull request #2560 from ralexstokes/add-broad-spectrum-spec-tests

Broad-spectrum randomized block tests

tests/core/pyspec/eth2spec/test/altair/random/test_random.py

@@ -0,0 +1,438 @@
+"""
+This module is generated from the ``random`` test generator.
+Please do not edit this file manually.
+See the README for that generator for more information.
+"""
+
+from eth2spec.test.helpers.constants import ALTAIR
+from eth2spec.test.context import (
+    misc_balances_in_default_range_with_many_validators,
+    with_phases,
+    zero_activation_threshold,
+    only_generator,
+)
+from eth2spec.test.context import (
+    always_bls,
+    spec_test,
+    with_custom_state,
+    single_phase,
+)
+from eth2spec.test.utils.randomized_block_tests import (
+    run_generated_randomized_test,
+)
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_0(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_1(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_2(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_3(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_4(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_5(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_6(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_7(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_8(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_9(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_10(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_11(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_12(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_13(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_14(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([ALTAIR])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_15(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )

tests/core/pyspec/eth2spec/test/context.py

@@ -152,6 +152,22 @@ def misc_balances(spec):
     return balances
 
 
+def misc_balances_in_default_range_with_many_validators(spec):
+    """
+    Helper method to create a series of balances that includes some misc. balances but
+    none that are below the ``EJECTION_BALANCE``.
+    """
+    # Double validators to facilitate randomized testing
+    num_validators = spec.SLOTS_PER_EPOCH * 8 * 2
+    floor = spec.config.EJECTION_BALANCE + spec.EFFECTIVE_BALANCE_INCREMENT
+    balances = [
+        max(spec.MAX_EFFECTIVE_BALANCE * 2 * i // num_validators, floor) for i in range(num_validators)
+    ]
+    rng = Random(1234)
+    rng.shuffle(balances)
+    return balances
+
+
 def low_single_balance(spec):
     """
     Helper method to create a single of balance of 1 Gwei.
@@ -440,6 +456,17 @@ with_altair_and_later = with_phases([ALTAIR, MERGE])
 with_merge_and_later = with_phases([MERGE])  # TODO: include sharding when spec stabilizes.
 
 
+def only_generator(reason):
+    def _decorator(inner):
+        def _wrapper(*args, **kwargs):
+            if is_pytest:
+                dump_skipping_message(reason)
+                return None
+            return inner(*args, **kwargs)
+        return _wrapper
+    return _decorator
+
+
 def fork_transition_test(pre_fork_name, post_fork_name, fork_epoch=None):
     """
     A decorator to construct a "transition" test from one fork of the eth2 spec

tests/core/pyspec/eth2spec/test/helpers/multi_operations.py

@@ -7,6 +7,10 @@ from eth2spec.test.helpers.state import (
 from eth2spec.test.helpers.block import (
     build_empty_block_for_next_slot,
 )
+from eth2spec.test.helpers.sync_committee import (
+    compute_committee_indices,
+    compute_aggregate_sync_committee_signature,
+)
 from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing
 from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing_by_indices
 from eth2spec.test.helpers.attestations import get_valid_attestation
@@ -44,8 +48,12 @@ def run_slash_and_exit(spec, state, slash_index, exit_index, valid=True):
 
 
 def get_random_proposer_slashings(spec, state, rng):
-    num_slashings = rng.randrange(spec.MAX_PROPOSER_SLASHINGS)
+    num_slashings = rng.randrange(1, spec.MAX_PROPOSER_SLASHINGS)
-    indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()
+    active_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()
+    indices = [
+        index for index in active_indices
+        if not state.validators[index].slashed
+    ]
     slashings = [
         get_valid_proposer_slashing(
             spec, state,
@@ -56,14 +64,33 @@ def get_random_proposer_slashings(spec, state, rng):
     return slashings
 
 
-def get_random_attester_slashings(spec, state, rng):
+def get_random_attester_slashings(spec, state, rng, slashed_indices=[]):
-    num_slashings = rng.randrange(spec.MAX_ATTESTER_SLASHINGS)
+    """
-    indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()
+    Caller can supply ``slashed_indices`` if they are aware of other indices
+    that will be slashed by other operations in the same block as the one that
+    contains the output of this function.
+    """
+    # ensure at least one attester slashing, the max count
+    # is small so not much room for random inclusion
+    num_slashings = rng.randrange(1, spec.MAX_ATTESTER_SLASHINGS)
+    active_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()
+    indices = [
+        index for index in active_indices
+        if (
+            not state.validators[index].slashed
+            and index not in slashed_indices
+        )
+    ]
+    sample_upper_bound = 4
+    max_slashed_count = num_slashings * sample_upper_bound - 1
+    if len(indices) < max_slashed_count:
+        return []
+
     slot_range = list(range(state.slot - spec.SLOTS_PER_HISTORICAL_ROOT + 1, state.slot))
     slashings = [
         get_valid_attester_slashing_by_indices(
             spec, state,
-            sorted([indices.pop(rng.randrange(len(indices))) for _ in range(rng.randrange(1, 4))]),
+            sorted([indices.pop(rng.randrange(len(indices))) for _ in range(rng.randrange(1, sample_upper_bound))]),
             slot=slot_range.pop(rng.randrange(len(slot_range))),
             signed_1=True, signed_2=True,
         )
@@ -73,7 +100,7 @@ def get_random_attester_slashings(spec, state, rng):
 
 
 def get_random_attestations(spec, state, rng):
-    num_attestations = rng.randrange(spec.MAX_ATTESTATIONS)
+    num_attestations = rng.randrange(1, spec.MAX_ATTESTATIONS)
 
     attestations = [
         get_valid_attestation(
@@ -87,7 +114,7 @@ def get_random_attestations(spec, state, rng):
 
 
 def prepare_state_and_get_random_deposits(spec, state, rng):
-    num_deposits = rng.randrange(spec.MAX_DEPOSITS)
+    num_deposits = rng.randrange(1, spec.MAX_DEPOSITS)
 
     deposit_data_leaves = [spec.DepositData() for _ in range(len(state.validators))]
     deposits = []
@@ -117,24 +144,65 @@ def prepare_state_and_get_random_deposits(spec, state, rng):
     return deposits
 
 
+def _eligible_for_exit(spec, state, index):
+    validator = state.validators[index]
+
+    not_slashed = not validator.slashed
+
+    current_epoch = spec.get_current_epoch(state)
+    activation_epoch = validator.activation_epoch
+    active_for_long_enough = current_epoch >= activation_epoch + spec.config.SHARD_COMMITTEE_PERIOD
+
+    not_exited = validator.exit_epoch == spec.FAR_FUTURE_EPOCH
+
+    return not_slashed and active_for_long_enough and not_exited
+
+
 def get_random_voluntary_exits(spec, state, to_be_slashed_indices, rng):
-    num_exits = rng.randrange(spec.MAX_VOLUNTARY_EXITS)
+    num_exits = rng.randrange(1, spec.MAX_VOLUNTARY_EXITS)
-    indices = set(spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy())
+    active_indices = set(spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy())
+    indices = set(
+        index for index in active_indices
+        if _eligible_for_exit(spec, state, index)
+    )
     eligible_indices = indices - to_be_slashed_indices
-    exit_indices = [eligible_indices.pop() for _ in range(num_exits)]
+    indices_count = min(num_exits, len(eligible_indices))
+    exit_indices = [eligible_indices.pop() for _ in range(indices_count)]
     return prepare_signed_exits(spec, state, exit_indices)
 
 
-def run_test_full_random_operations(spec, state, rng=Random(2080)):
+def get_random_sync_aggregate(spec, state, slot, fraction_participated=1.0, rng=Random(2099)):
-    # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit
+    committee_indices = compute_committee_indices(spec, state, state.current_sync_committee)
-    state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+    participant_count = int(len(committee_indices) * fraction_participated)
+    participant_indices = rng.sample(range(len(committee_indices)), participant_count)
+    participants = [
+        committee_indices[index]
+        for index in participant_indices
+    ]
+    signature = compute_aggregate_sync_committee_signature(
+        spec,
+        state,
+        slot,
+        participants,
+    )
+    return spec.SyncAggregate(
+        sync_committee_bits=[index in participant_indices for index in range(len(committee_indices))],
+        sync_committee_signature=signature,
+    )
 
+
+def build_random_block_from_state_for_next_slot(spec, state, rng=Random(2188)):
     # prepare state for deposits before building block
     deposits = prepare_state_and_get_random_deposits(spec, state, rng)
 
     block = build_empty_block_for_next_slot(spec, state)
-    block.body.proposer_slashings = get_random_proposer_slashings(spec, state, rng)
+    proposer_slashings = get_random_proposer_slashings(spec, state, rng)
-    block.body.attester_slashings = get_random_attester_slashings(spec, state, rng)
+    block.body.proposer_slashings = proposer_slashings
+    slashed_indices = [
+        slashing.signed_header_1.message.proposer_index
+        for slashing in proposer_slashings
+    ]
+    block.body.attester_slashings = get_random_attester_slashings(spec, state, rng, slashed_indices)
     block.body.attestations = get_random_attestations(spec, state, rng)
     block.body.deposits = deposits
 
@@ -148,6 +216,15 @@ def run_test_full_random_operations(spec, state, rng=Random(2080)):
         slashed_indices = slashed_indices.union(attester_slashing.attestation_2.attesting_indices)
     block.body.voluntary_exits = get_random_voluntary_exits(spec, state, slashed_indices, rng)
 
+    return block
+
+
+def run_test_full_random_operations(spec, state, rng=Random(2080)):
+    # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit
+    state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+
+    block = build_random_block_from_state_for_next_slot(spec, state, rng)
+
     yield 'pre', state
 
     signed_block = state_transition_and_sign_block(spec, state, block)

tests/core/pyspec/eth2spec/test/helpers/random.py

@@ -20,16 +20,20 @@ def set_some_new_deposits(spec, state, rng):
                 state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state)
 
 
-def exit_random_validators(spec, state, rng):
+def exit_random_validators(spec, state, rng, fraction=None):
+    if fraction is None:
+        # Exit ~1/2
+        fraction = 0.5
+
     if spec.get_current_epoch(state) < 5:
         # Move epochs forward to allow for some validators already exited/withdrawable
         for _ in range(5):
             next_epoch(spec, state)
 
     current_epoch = spec.get_current_epoch(state)
-    # Exit ~1/2 of validators
     for index in spec.get_active_validator_indices(state, current_epoch):
-        if rng.choice([True, False]):
+        sampled = rng.random() < fraction
+        if not sampled:
             continue
 
         validator = state.validators[index]
@@ -41,11 +45,15 @@ def exit_random_validators(spec, state, rng):
             validator.withdrawable_epoch = current_epoch + 1
 
 
-def slash_random_validators(spec, state, rng):
+def slash_random_validators(spec, state, rng, fraction=None):
-    # Slash ~1/2 of validators
+    if fraction is None:
+        # Slash ~1/2 of validators
+        fraction = 0.5
+
     for index in range(len(state.validators)):
         # slash at least one validator
-        if index == 0 or rng.choice([True, False]):
+        sampled = rng.random() < fraction
+        if index == 0 or sampled:
             spec.slash_validator(state, index)
 
 
@@ -115,8 +123,8 @@ def randomize_attestation_participation(spec, state, rng=Random(8020)):
     randomize_epoch_participation(spec, state, spec.get_current_epoch(state), rng)
 
 
-def randomize_state(spec, state, rng=Random(8020)):
+def randomize_state(spec, state, rng=Random(8020), exit_fraction=None, slash_fraction=None):
     set_some_new_deposits(spec, state, rng)
-    exit_random_validators(spec, state, rng)
+    exit_random_validators(spec, state, rng, fraction=exit_fraction)
-    slash_random_validators(spec, state, rng)
+    slash_random_validators(spec, state, rng, fraction=slash_fraction)
     randomize_attestation_participation(spec, state, rng)

tests/core/pyspec/eth2spec/test/helpers/state.py

@@ -1,5 +1,6 @@
 from eth2spec.test.context import expect_assertion_error, is_post_altair
 from eth2spec.test.helpers.block import apply_empty_block, sign_block, transition_unsigned_block
+from eth2spec.test.helpers.voluntary_exits import get_exited_validators
 
 
 def get_balance(state, index):
@@ -133,3 +134,24 @@ def _set_empty_participation(spec, state, current=True, previous=True):
 
 def set_empty_participation(spec, state, rng=None):
     _set_empty_participation(spec, state)
+
+
+def ensure_state_has_validators_across_lifecycle(spec, state):
+    """
+    Scan the validator registry to ensure there is at least 1 validator
+    for each of the following lifecycle states:
+        1. Pending / deposited
+        2. Active
+        3. Exited
+        4. Slashed
+    """
+    has_pending = any(filter(spec.is_eligible_for_activation_queue, state.validators))
+
+    current_epoch = spec.get_current_epoch(state)
+    has_active = any(filter(lambda v: spec.is_active_validator(v, current_epoch), state.validators))
+
+    has_exited = any(get_exited_validators(spec, state))
+
+    has_slashed = any(filter(lambda v: v.slashed, state.validators))
+
+    return has_pending and has_active and has_exited and has_slashed

tests/core/pyspec/eth2spec/test/phase0/random/test_random.py

@@ -0,0 +1,438 @@
+"""
+This module is generated from the ``random`` test generator.
+Please do not edit this file manually.
+See the README for that generator for more information.
+"""
+
+from eth2spec.test.helpers.constants import PHASE0
+from eth2spec.test.context import (
+    misc_balances_in_default_range_with_many_validators,
+    with_phases,
+    zero_activation_threshold,
+    only_generator,
+)
+from eth2spec.test.context import (
+    always_bls,
+    spec_test,
+    with_custom_state,
+    single_phase,
+)
+from eth2spec.test.utils.randomized_block_tests import (
+    run_generated_randomized_test,
+)
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_0(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_1(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_2(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_3(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_4(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_5(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_6(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_7(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_8(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_9(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_10(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_11(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_12(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:last_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_13(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:random_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_14(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )
+
+
+@only_generator("randomized test for broad coverage, not point-to-point CI")
+@with_phases([PHASE0])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_15(spec, state):
+    # scenario as high-level, informal text:
+    # epochs:epochs_until_leak,slots:0,with-block:no_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    # epochs:1,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:no_block
+    # epochs:0,slots:0,with-block:random_block
+    scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )

tests/core/pyspec/eth2spec/test/utils/__init__.py

@@ -0,0 +1,12 @@
+from .utils import (
+    vector_test,
+    with_meta_tags,
+    build_transition_test,
+)
+
+
+__all__ = [  # avoid "unused import" lint error
+    "vector_test",
+    "with_meta_tags",
+    "build_transition_test",
+]

tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py

@@ -0,0 +1,319 @@
+"""
+Utility code to generate randomized block tests
+"""
+
+import sys
+import warnings
+from random import Random
+from typing import Callable
+
+from eth2spec.test.helpers.multi_operations import (
+    build_random_block_from_state_for_next_slot,
+    get_random_sync_aggregate,
+)
+from eth2spec.test.helpers.inactivity_scores import (
+    randomize_inactivity_scores,
+)
+from eth2spec.test.helpers.random import (
+    randomize_state as randomize_state_helper,
+)
+from eth2spec.test.helpers.state import (
+    next_slot,
+    next_epoch,
+    ensure_state_has_validators_across_lifecycle,
+    state_transition_and_sign_block,
+)
+
+# primitives:
+# state
+
+
+def randomize_state(spec, state, exit_fraction=0.1, slash_fraction=0.1):
+    randomize_state_helper(spec, state, exit_fraction=exit_fraction, slash_fraction=slash_fraction)
+
+
+def randomize_state_altair(spec, state):
+    randomize_state(spec, state, exit_fraction=0.1, slash_fraction=0.1)
+    randomize_inactivity_scores(spec, state)
+
+
+# epochs
+
+def epochs_until_leak(spec):
+    """
+    State is "leaking" if the current epoch is at least
+    this value after the last finalized epoch.
+    """
+    return spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 1
+
+
+def epochs_for_shard_committee_period(spec):
+    return spec.config.SHARD_COMMITTEE_PERIOD
+
+
+# slots
+
+def last_slot_in_epoch(spec):
+    return spec.SLOTS_PER_EPOCH - 1
+
+
+def random_slot_in_epoch(spec, rng=Random(1336)):
+    return rng.randrange(1, spec.SLOTS_PER_EPOCH - 2)
+
+
+def penultimate_slot_in_epoch(spec):
+    return spec.SLOTS_PER_EPOCH - 2
+
+
+# blocks
+
+def no_block(_spec, _pre_state, _signed_blocks):
+    return None
+
+
+# May need to make several attempts to find a block that does not correspond to a slashed
+# proposer with the randomization helpers...
+BLOCK_ATTEMPTS = 32
+
+
+def _warn_if_empty_operations(block):
+    if len(block.body.deposits) == 0:
+        warnings.warn(f"deposits missing in block at slot {block.slot}")
+
+    if len(block.body.proposer_slashings) == 0:
+        warnings.warn(f"proposer slashings missing in block at slot {block.slot}")
+
+    if len(block.body.attester_slashings) == 0:
+        warnings.warn(f"attester slashings missing in block at slot {block.slot}")
+
+    if len(block.body.attestations) == 0:
+        warnings.warn(f"attestations missing in block at slot {block.slot}")
+
+    if len(block.body.voluntary_exits) == 0:
+        warnings.warn(f"voluntary exits missing in block at slot {block.slot}")
+
+
+def random_block(spec, state, _signed_blocks):
+    """
+    Produce a random block.
+    NOTE: this helper may mutate state, as it will attempt
+    to produce a block over ``BLOCK_ATTEMPTS`` slots in order
+    to find a valid block in the event that the proposer has already been slashed.
+    """
+    # NOTE: ``state`` has been "randomized" at this point and so will likely
+    # contain a large number of slashed validators. This function needs to return
+    # a valid block so it needs to check that the proposer of the next slot is not
+    # slashed.
+    # To do this, generate a ``temp_state`` to use for checking the propser in the next slot.
+    # This ensures no accidental mutations happen to the ``state`` the caller expects to get back
+    # after this function returns.
+    # Using a copy of the state for proposer sampling is also sound as any inputs used for the
+    # shuffling are fixed a few epochs prior to ``spec.get_current_epoch(state)``.
+    temp_state = state.copy()
+    next_slot(spec, temp_state)
+    for _ in range(BLOCK_ATTEMPTS):
+        proposer_index = spec.get_beacon_proposer_index(temp_state)
+        proposer = state.validators[proposer_index]
+        if proposer.slashed:
+            next_slot(spec, state)
+            next_slot(spec, temp_state)
+        else:
+            block = build_random_block_from_state_for_next_slot(spec, state)
+            _warn_if_empty_operations(block)
+            return block
+    else:
+        raise AssertionError("could not find a block with an unslashed proposer, check ``state`` input")
+
+
+SYNC_AGGREGATE_PARTICIPATION_BUCKETS = 4
+
+
+def random_block_altair_with_cycling_sync_committee_participation(spec,
+                                                                  state,
+                                                                  signed_blocks):
+    block = random_block(spec, state, signed_blocks)
+    block_index = len(signed_blocks) % SYNC_AGGREGATE_PARTICIPATION_BUCKETS
+    fraction_missed = block_index * (1 / SYNC_AGGREGATE_PARTICIPATION_BUCKETS)
+    fraction_participated = 1.0 - fraction_missed
+    block.body.sync_aggregate = get_random_sync_aggregate(
+        spec,
+        state,
+        block.slot - 1,
+        fraction_participated=fraction_participated,
+    )
+    return block
+
+
+# validations
+
+def no_op_validation(spec, state):
+    return True
+
+
+def validate_is_leaking(spec, state):
+    return spec.is_in_inactivity_leak(state)
+
+
+def validate_is_not_leaking(spec, state):
+    return not validate_is_leaking(spec, state)
+
+
+# transitions
+
+def with_validation(transition, validation):
+    if isinstance(transition, Callable):
+        transition = transition()
+    transition["validation"] = validation
+    return transition
+
+
+def no_op_transition():
+    return {}
+
+
+def epoch_transition(n=0):
+    return {
+        "epochs_to_skip": n,
+    }
+
+
+def slot_transition(n=0):
+    return {
+        "slots_to_skip": n,
+    }
+
+
+def transition_to_leaking():
+    return {
+        "epochs_to_skip": epochs_until_leak,
+        "validation": validate_is_leaking,
+    }
+
+
+transition_without_leak = with_validation(no_op_transition, validate_is_not_leaking)
+
+# block transitions
+
+
+def transition_with_random_block(block_randomizer):
+    """
+    Build a block transition with randomized data.
+    Provide optional sub-transitions to advance some
+    number of epochs or slots before applying the random block.
+    """
+    return {
+        "block_producer": block_randomizer,
+    }
+
+
+# setup and test gen
+
+
+def _randomized_scenario_setup(state_randomizer):
+    """
+    Return a sequence of pairs of ("mutation", "validation"),
+    a function that accepts (spec, state) arguments and performs some change
+    and a function that accepts (spec, state) arguments and validates some change was made.
+    """
+    def _skip_epochs(epoch_producer):
+        def f(spec, state):
+            """
+            The unoptimized spec implementation is too slow to advance via ``next_epoch``.
+            Instead, just overwrite the ``state.slot`` and continue...
+            """
+            epochs_to_skip = epoch_producer(spec)
+            slots_to_skip = epochs_to_skip * spec.SLOTS_PER_EPOCH
+            state.slot += slots_to_skip
+        return f
+
+    def _simulate_honest_execution(spec, state):
+        """
+        Want to start tests not in a leak state; the finality data
+        may not reflect this condition with prior (arbitrary) mutations,
+        so this mutator addresses that fact.
+        """
+        state.justification_bits = (True, True, True, True)
+        previous_epoch = spec.get_previous_epoch(state)
+        previous_root = spec.get_block_root(state, previous_epoch)
+        previous_previous_epoch = max(spec.GENESIS_EPOCH, spec.Epoch(previous_epoch - 1))
+        previous_previous_root = spec.get_block_root(state, previous_previous_epoch)
+        state.previous_justified_checkpoint = spec.Checkpoint(
+            epoch=previous_previous_epoch,
+            root=previous_previous_root,
+        )
+        state.current_justified_checkpoint = spec.Checkpoint(
+            epoch=previous_epoch,
+            root=previous_root,
+        )
+        state.finalized_checkpoint = spec.Checkpoint(
+            epoch=previous_previous_epoch,
+            root=previous_previous_root,
+        )
+
+    return (
+        # NOTE: the block randomization function assumes at least 1 shard committee period
+        # so advance the state before doing anything else.
+        (_skip_epochs(epochs_for_shard_committee_period), no_op_validation),
+        (_simulate_honest_execution, no_op_validation),
+        (state_randomizer, ensure_state_has_validators_across_lifecycle),
+    )
+
+# Run the generated tests:
+
+
+# while the test implementation works via code-gen,
+# references to helper code in this module are serialized as str names.
+# to resolve this references at runtime, we need a reference to this module:
+_this_module = sys.modules[__name__]
+
+
+def _resolve_ref(ref):
+    if isinstance(ref, str):
+        return getattr(_this_module, ref)
+    return ref
+
+
+def _iter_temporal(spec, description):
+    """
+    Intended to advance some number of {epochs, slots}.
+    Caller can provide a constant integer or a callable deriving a number from
+    the ``spec`` under consideration.
+    """
+    numeric = _resolve_ref(description)
+    if isinstance(numeric, Callable):
+        numeric = numeric(spec)
+    for i in range(numeric):
+        yield i
+
+
+def run_generated_randomized_test(spec, state, scenario):
+    if "setup" not in scenario:
+        state_randomizer = _resolve_ref(scenario.get("state_randomizer", randomize_state))
+        scenario["setup"] = _randomized_scenario_setup(state_randomizer)
+
+    for mutation, validation in scenario["setup"]:
+        mutation(spec, state)
+        validation(spec, state)
+
+    yield "pre", state
+
+    blocks = []
+    for transition in scenario["transitions"]:
+        epochs_to_skip = _iter_temporal(spec, transition["epochs_to_skip"])
+        for _ in epochs_to_skip:
+            next_epoch(spec, state)
+        slots_to_skip = _iter_temporal(spec, transition["slots_to_skip"])
+        for _ in slots_to_skip:
+            next_slot(spec, state)
+
+        block_producer = _resolve_ref(transition["block_producer"])
+        block = block_producer(spec, state, blocks)
+        if block:
+            signed_block = state_transition_and_sign_block(spec, state, block)
+            blocks.append(signed_block)
+
+        validation = _resolve_ref(transition["validation"])
+        assert validation(spec, state)
+
+    yield "blocks", blocks
+    yield "post", state

tests/generators/random/Makefile

@@ -0,0 +1,8 @@
+all:
+	if ! test -d venv; then python3 -m venv venv; fi;
+	. ./venv/bin/activate
+	pip3 install -r requirements.txt
+	rm -f ../../core/pyspec/eth2spec/test/phase0/random/test_random.py
+	rm -f ../../core/pyspec/eth2spec/test/altair/random/test_random.py
+	python3 generate.py phase0 > ../../core/pyspec/eth2spec/test/phase0/random/test_random.py
+	python3 generate.py altair > ../../core/pyspec/eth2spec/test/altair/random/test_random.py

tests/generators/random/README.md

@@ -0,0 +1,31 @@
+# Randomized tests
+
+Randomized tests in the format of `sanity` blocks tests, with randomized operations.
+
+Information on the format of the tests can be found in the [sanity test formats documentation](../../formats/sanity/README.md).
+
+# To generate test sources
+
+```bash
+$ make
+```
+
+The necessary commands are in the `Makefile`, as the only target.
+
+The generated files are committed to the repo so you should not need to do this.
+
+# To run tests
+
+Each of the generated test does produce a `pytest` test instance but by default is
+currently skipped. Running the test via the generator (see next) will trigger any errors
+that would arise during the running of `pytest`.
+
+# To generate spec tests (from the generated files)
+
+Run the test generator in the usual way.
+
+E.g. from the root of this repo, you can run:
+
+```bash
+$ make gen_random
+```

tests/generators/random/generate.py

@@ -0,0 +1,258 @@
+"""
+This test format currently uses code generation to assemble the tests
+as the current test infra does not have a facility to dynamically
+generate tests that can be seen by ``pytest``.
+
+This will likley change in future releases of the testing infra.
+
+NOTE: To add additional scenarios, add test cases below in ``_generate_randomized_scenarios``.
+"""
+
+import sys
+import random
+import warnings
+from typing import Callable
+import itertools
+
+from eth2spec.test.utils.randomized_block_tests import (
+    no_block,
+    no_op_validation,
+    randomize_state,
+    randomize_state_altair,
+    random_block,
+    random_block_altair_with_cycling_sync_committee_participation,
+    last_slot_in_epoch,
+    random_slot_in_epoch,
+    penultimate_slot_in_epoch,
+    epoch_transition,
+    slot_transition,
+    transition_with_random_block,
+    transition_to_leaking,
+    transition_without_leak,
+)
+from eth2spec.test.helpers.constants import PHASE0, ALTAIR
+
+
+# Ensure this many blocks are present in *each* randomized scenario
+BLOCK_TRANSITIONS_COUNT = 2
+
+
+def _normalize_transition(transition):
+    """
+    Provide "empty" or "no op" sub-transitions
+    to a given transition.
+    """
+    if isinstance(transition, Callable):
+        transition = transition()
+    if "epochs_to_skip" not in transition:
+        transition["epochs_to_skip"] = 0
+    if "slots_to_skip" not in transition:
+        transition["slots_to_skip"] = 0
+    if "block_producer" not in transition:
+        transition["block_producer"] = no_block
+    if "validation" not in transition:
+        transition["validation"] = no_op_validation
+    return transition
+
+
+def _normalize_scenarios(scenarios):
+    """
+    "Normalize" a "scenario" so that a producer of a test case
+    does not need to provide every expected key/value.
+    """
+    for scenario in scenarios:
+        transitions = scenario["transitions"]
+        for i, transition in enumerate(transitions):
+            transitions[i] = _normalize_transition(transition)
+
+
+def _flatten(t):
+    leak_transition = t[0]
+    result = [leak_transition]
+    for transition_batch in t[1]:
+        for transition in transition_batch:
+            if isinstance(transition, tuple):
+                for subtransition in transition:
+                    result.append(subtransition)
+            else:
+                result.append(transition)
+    return result
+
+
+def _generate_randomized_scenarios(block_randomizer):
+    """
+    Generates a set of randomized testing scenarios.
+    Return a sequence of "scenarios" where each scenario:
+    1. Provides some setup
+    2. Provides a sequence of transitions that mutate the state in some way,
+        possibly yielding blocks along the way
+    NOTE: scenarios are "normalized" with empty/no-op elements before returning
+    to the test generation to facilitate brevity when writing scenarios by hand.
+    NOTE: the main block driver builds a block for the **next** slot, so
+    the slot transitions are offset by -1 to target certain boundaries.
+    """
+    # go forward 0 or 1 epochs
+    epochs_set = (
+        epoch_transition(n=0),
+        epoch_transition(n=1),
+    )
+    # within those epochs, go forward to:
+    slots_set = (
+        # the first slot in an epoch (see note in docstring about offsets...)
+        slot_transition(last_slot_in_epoch),
+        # the second slot in an epoch
+        slot_transition(n=0),
+        # some random number of slots, but not at epoch boundaries
+        slot_transition(random_slot_in_epoch),
+        # the last slot in an epoch (see note in docstring about offsets...)
+        slot_transition(penultimate_slot_in_epoch),
+    )
+    # and produce a block...
+    blocks_set = (
+        transition_with_random_block(block_randomizer),
+    )
+
+    rng = random.Random(1447)
+    all_skips = list(itertools.product(epochs_set, slots_set))
+    randomized_skips = (
+        rng.sample(all_skips, len(all_skips))
+        for _ in range(BLOCK_TRANSITIONS_COUNT)
+    )
+
+    # build a set of block transitions from combinations of sub-transitions
+    transitions_generator = (
+        itertools.product(prefix, blocks_set)
+        for prefix in randomized_skips
+    )
+    block_transitions = zip(*transitions_generator)
+
+    # and preface each set of block transitions with the possible leak transitions
+    leak_transitions = (
+        transition_without_leak,
+        transition_to_leaking,
+    )
+    scenarios = [
+        {"transitions": _flatten(t)}
+        for t in itertools.product(leak_transitions, block_transitions)
+    ]
+    _normalize_scenarios(scenarios)
+    return scenarios
+
+
+def _id_from_scenario(test_description):
+    """
+    Construct a name for the scenario based its data.
+    """
+    def _to_id_part(prefix, x):
+        suffix = str(x)
+        if isinstance(x, Callable):
+            suffix = x.__name__
+        return f"{prefix}{suffix}"
+
+    def _id_from_transition(transition):
+        return ",".join((
+            _to_id_part("epochs:", transition["epochs_to_skip"]),
+            _to_id_part("slots:", transition["slots_to_skip"]),
+            _to_id_part("with-block:", transition["block_producer"])
+        ))
+
+    return "|".join(map(_id_from_transition, test_description["transitions"]))
+
+
+test_imports_template = """\"\"\"
+This module is generated from the ``random`` test generator.
+Please do not edit this file manually.
+See the README for that generator for more information.
+\"\"\"
+
+from eth2spec.test.helpers.constants import {phase}
+from eth2spec.test.context import (
+    misc_balances_in_default_range_with_many_validators,
+    with_phases,
+    zero_activation_threshold,
+    only_generator,
+)
+from eth2spec.test.context import (
+    always_bls,
+    spec_test,
+    with_custom_state,
+    single_phase,
+)
+from eth2spec.test.utils.randomized_block_tests import (
+    run_generated_randomized_test,
+)"""
+
+test_template = """
+@only_generator(\"randomized test for broad coverage, not point-to-point CI\")
+@with_phases([{phase}])
+@with_custom_state(
+    balances_fn=misc_balances_in_default_range_with_many_validators,
+    threshold_fn=zero_activation_threshold
+)
+@spec_test
+@single_phase
+@always_bls
+def test_randomized_{index}(spec, state):
+    # scenario as high-level, informal text:
+{name_as_comment}
+    scenario = {scenario}  # noqa: E501
+    yield from run_generated_randomized_test(
+        spec,
+        state,
+        scenario,
+    )"""
+
+
+def _to_comment(name, indent_level):
+    parts = name.split("|")
+    indentation = "    " * indent_level
+    parts = [
+        indentation + "# " + part for part in parts
+    ]
+    return "\n".join(parts)
+
+
+def run_generate_tests_to_std_out(phase, state_randomizer, block_randomizer):
+    scenarios = _generate_randomized_scenarios(block_randomizer)
+    test_content = {"phase": phase.upper()}
+    test_imports = test_imports_template.format(**test_content)
+    test_file = [test_imports]
+    for index, scenario in enumerate(scenarios):
+        # required for setup phase
+        scenario["state_randomizer"] = state_randomizer.__name__
+
+        # need to pass name, rather than function reference...
+        transitions = scenario["transitions"]
+        for transition in transitions:
+            for name, value in transition.items():
+                if isinstance(value, Callable):
+                    transition[name] = value.__name__
+
+        test_content = test_content.copy()
+        name = _id_from_scenario(scenario)
+        test_content["name_as_comment"] = _to_comment(name, 1)
+        test_content["index"] = index
+        test_content["scenario"] = scenario
+        test_instance = test_template.format(**test_content)
+        test_file.append(test_instance)
+    print("\n\n".join(test_file))
+
+
+if __name__ == "__main__":
+    did_generate = False
+    if PHASE0 in sys.argv:
+        did_generate = True
+        run_generate_tests_to_std_out(
+            PHASE0,
+            state_randomizer=randomize_state,
+            block_randomizer=random_block,
+        )
+    if ALTAIR in sys.argv:
+        did_generate = True
+        run_generate_tests_to_std_out(
+            ALTAIR,
+            state_randomizer=randomize_state_altair,
+            block_randomizer=random_block_altair_with_cycling_sync_committee_participation,
+        )
+    if not did_generate:
+        warnings.warn("no phase given for test generation")

tests/generators/random/main.py

@@ -0,0 +1,18 @@
+from eth2spec.test.helpers.constants import PHASE0, ALTAIR
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+
+
+if __name__ == "__main__":
+    phase_0_mods = {key: 'eth2spec.test.phase0.random.test_' + key for key in [
+        'random',
+    ]}
+    altair_mods = {key: 'eth2spec.test.altair.random.test_' + key for key in [
+        'random',
+    ]}
+
+    all_mods = {
+        PHASE0: phase_0_mods,
+        ALTAIR: altair_mods,
+    }
+
+    run_state_test_generators(runner_name="random", all_mods=all_mods)

tests/generators/random/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]