From 8060505743911f6855254a386a07f5efd85cabeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 16:00:59 -0600 Subject: [PATCH 1/8] refactor rewards/penalties tests to use a single structure --- .../pyspec/eth2spec/test/helpers/rewards.py | 250 +++++++++++++++--- ...est_get_source_deltas.py => test_basic.py} | 88 +++--- .../phase_0/rewards/test_get_head_deltas.py | 136 ---------- .../test_get_inactivity_penalty_deltas.py | 231 ---------------- .../test_get_inclusion_delay_deltas.py | 213 --------------- .../phase_0/rewards/test_get_target_deltas.py | 140 ---------- .../test/phase_0/rewards/test_leak.py | 165 ++++++++++++ .../test/phase_0/rewards/test_random.py | 22 ++ tests/formats/rewards/README.md | 57 ++-- tests/generators/rewards/main.py | 24 +- 10 files changed, 504 insertions(+), 822 deletions(-) rename tests/core/pyspec/eth2spec/test/phase_0/rewards/{test_get_source_deltas.py => test_basic.py} (67%) delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 9c445f968..fc9f1f93a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -25,17 +25,50 @@ def has_enough_for_reward(spec, state, index): ) -def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): +def run_deltas(spec, state): """ - Run ``component_delta_fn``, yielding: + Run all deltas functions yielding: - pre-state ('pre') - - deltas ('deltas') + - source deltas ('source_deltas') + - target deltas ('target_deltas') + - head deltas ('head_deltas') + - inclusion delay deltas ('inclusion_delay_deltas') + - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + yield from run_attestation_component_deltas( + spec, + state, + spec.get_source_deltas, + spec.get_matching_source_attestations, + 'source_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_target_deltas, + spec.get_matching_target_attestations, + 'target_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_head_deltas, + spec.get_matching_head_attestations, + 'head_deltas', + ) + yield from run_get_inclusion_delay_deltas(spec, state) + yield from run_get_inactivity_penalty_deltas(spec, state) + +def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): + """ + Run ``component_delta_fn``, yielding: + - deltas ('{``deltas_name``}') + """ rewards, penalties = component_delta_fn(state) - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) + yield deltas_name, Deltas(rewards=rewards, penalties=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) @@ -62,6 +95,81 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def run_get_inclusion_delay_deltas(spec, state): + """ + Run ``get_inclusion_delay_deltas``, yielding: + - inclusion delay deltas ('inclusion_delay_deltas') + """ + rewards, penalties = spec.get_inclusion_delay_deltas(state) + + yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) + + eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) + attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) + + rewarded_indices = set() + rewarded_proposer_indices = set() + # Ensure attesters with enough balance are rewarded for attestations + # Track those that are rewarded and track proposers that should be rewarded + for index in range(len(state.validators)): + if index in attesting_indices and has_enough_for_reward(spec, state, index): + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Track proposer of earliest included attestation for the validator defined by index + earliest_attestation = min([ + a for a in eligible_attestations + if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewarded_proposer_indices.add(earliest_attestation.proposer_index) + + # Ensure all expected proposers have been rewarded + # Track rewarde indices + proposing_indices = [a.proposer_index for a in eligible_attestations] + for index in proposing_indices: + if index in rewarded_proposer_indices: + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Ensure all expected non-rewarded indices received no reward + for index in range(len(state.validators)): + assert penalties[index] == 0 + if index not in rewarded_indices: + assert rewards[index] == 0 + + +def run_get_inactivity_penalty_deltas(spec, state): + """ + Run ``get_inactivity_penalty_deltas``, yielding: + - inactivity penalty deltas ('inactivity_penalty_deltas') + """ + rewards, penalties = spec.get_inactivity_penalty_deltas(state) + + yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) + + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + + finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + assert rewards[index] == 0 + if index not in eligible_indices: + assert penalties[index] == 0 + continue + + if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if not has_enough_for_reward(spec, state, index): + assert penalties[index] == 0 + elif index in matching_attesting_indices: + assert penalties[index] == base_penalty + else: + assert penalties[index] > base_penalty + else: + assert penalties[index] == 0 + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited @@ -102,74 +210,74 @@ def slash_random_validators(spec, state, rng): spec.slash_validator(state, index) -def run_test_empty(spec, state, runner): +def run_test_empty(spec, state): # Do not add any attestations to state - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_all_correct(spec, state, runner): +def run_test_full_all_correct(spec, state): prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): +def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_partial(spec, state, fraction_filled, runner): +def run_test_partial(spec, state, fraction_filled): prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_half_full(spec, state, runner): - yield from run_test_partial(spec, state, 0.5, runner) +def run_test_half_full(spec, state): + yield from run_test_partial(spec, state, 0.5) -def run_test_one_attestation_one_correct(spec, state, runner): +def run_test_one_attestation_one_correct(spec, state): prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_not_yet_activated_validators(spec, state, runner, rng=Random(5555)): +def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): +def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_slashed_validators(spec, state, runner, rng=Random(3322)): +def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_attested(spec, state, runner): +def run_test_some_very_low_effective_balances_that_attested(spec, state): state.balances prepare_state_with_attestations(spec, state) @@ -178,10 +286,10 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state, runner) for i, index in enumerate(range(5)): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): +def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): prepare_state_with_attestations(spec, state) # Remove attestation @@ -192,10 +300,10 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, r for i, index in enumerate(indices): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): +def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified @@ -206,10 +314,92 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, if not correct_head: pending_attestation.data.beacon_block_root = b'\x66' * 32 - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_random(spec, state, runner, rng=Random(8020)): +def run_test_full_delay_one_slot(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += 1 + + yield from run_deltas(spec, state) + + +def run_test_full_delay_max_slots(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += spec.SLOTS_PER_EPOCH + + yield from run_deltas(spec, state) + + +def run_test_full_mixed_delay(spec, state, rng=Random(1234)): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from run_deltas(spec, state) + + +def run_test_proposer_not_in_attestations(spec, state): + prepare_state_with_attestations(spec, state) + + # Get an attestation where the proposer is not in the committee + non_proposer_attestations = [] + for a in state.previous_epoch_attestations: + if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): + non_proposer_attestations.append(a) + + assert any(non_proposer_attestations) + state.previous_epoch_attestations = non_proposer_attestations + + yield from run_deltas(spec, state) + + +def run_test_duplicate_attestations_at_later_slots(spec, state): + prepare_state_with_attestations(spec, state) + + # Remove 2/3 of attestations to make it more interesting + num_attestations = int(len(state.previous_epoch_attestations) * 0.33) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + # Get map of the proposer at each slot to make valid-looking duplicate attestations + per_slot_proposers = { + (a.data.slot + a.inclusion_delay): a.proposer_index + for a in state.previous_epoch_attestations + } + max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) + later_attestations = [] + for a in state.previous_epoch_attestations: + # Only have proposers for previous epoch so do not create later + # duplicate if slot exceeds the max slot in previous_epoch_attestations + if a.data.slot + a.inclusion_delay >= max_slot: + continue + later_a = a.copy() + later_a.inclusion_delay += 1 + later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] + later_attestations.append(later_a) + + assert any(later_attestations) + + state.previous_epoch_attestations = sorted( + state.previous_epoch_attestations + later_attestations, + key=lambda a: a.data.slot + a.inclusion_delay + ) + + yield from run_deltas(spec, state) + + +def run_test_all_balances_too_low_for_reward(spec, state): + prepare_state_with_attestations(spec, state) + + for index in range(len(state.validators)): + state.validators[index].effective_balance = 10 + + yield from run_deltas(spec, state) + + +def run_test_full_random(spec, state, rng=Random(8020)): set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) @@ -228,4 +418,4 @@ def run_test_full_random(spec, state, runner, rng=Random(8020)): # Random inclusion delay pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - yield from runner(spec, state) + yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py similarity index 67% rename from tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py index 1d7891eca..92277fdd7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py @@ -1,89 +1,71 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas import eth2spec.test.helpers.rewards as rewards_helpers -def run_get_source_deltas(spec, state): - """ - Run ``get_source_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_source_deltas, - spec.get_matching_source_attestations, - ) - - @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full(spec, state): + yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_source_deltas - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_source_deltas, - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) # @@ -101,7 +83,6 @@ def test_full_half_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -113,7 +94,6 @@ def test_full_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=1.0, - runner=run_get_source_deltas ) @@ -125,7 +105,6 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): correct_target=False, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -137,11 +116,40 @@ def test_full_half_incorrect_target_correct_head(spec, state): correct_target=False, correct_head=True, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @with_all_phases @spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_source_deltas) +def test_full_delay_one_slot(spec, state): + yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_delay_max_slots(spec, state): + yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_mixed_delay(spec, state): + yield from rewards_helpers.run_test_full_mixed_delay(spec, state) + + +@with_all_phases +@spec_state_test +def test_proposer_not_in_attestations(spec, state): + yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) + + +@with_all_phases +@spec_state_test +def test_duplicate_attestations_at_later_slots(spec, state): + yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_all_balances_too_low_for_reward(spec, state): + yield from rewards_helpers.run_test_all_balances_too_low_for_reward(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py deleted file mode 100644 index e49a8567c..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ /dev/null @@ -1,136 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_head_deltas(spec, state): - """ - Run ``get_head_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_head_deltas, - spec.get_matching_head_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_head_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py deleted file mode 100644 index 8588b3f52..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ /dev/null @@ -1,231 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import has_enough_for_reward -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.rewards import Deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inactivity_penalty_deltas(spec, state): - """ - Run ``get_inactivity_penalty_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inactivity_penalty_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch - eligible_indices = spec.get_eligible_validator_indices(state) - for index in range(len(state.validators)): - assert rewards[index] == 0 - if index not in eligible_indices: - assert penalties[index] == 0 - continue - - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) - if not has_enough_for_reward(spec, state, index): - assert penalties[index] == 0 - elif index in matching_attesting_indices: - assert penalties[index] == base_penalty - else: - assert penalties[index] > base_penalty - else: - assert penalties[index] == 0 - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -@with_all_phases -@spec_state_test -def test_empty_no_leak(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_no_leak(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_no_leak(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_no_leak(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_no_leak(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_random_no_leak(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py deleted file mode 100644 index 94fef5777..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ /dev/null @@ -1,213 +0,0 @@ -from random import Random - -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import prepare_state_with_attestations -from eth2spec.test.helpers.rewards import Deltas, has_enough_for_reward -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inclusion_delay_deltas(spec, state): - """ - Run ``get_inclusion_delay_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inclusion_delay_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) - attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) - - rewarded_indices = set() - rewarded_proposer_indices = set() - # Ensure attesters with enough balance are rewarded for attestations - # Track those that are rewarded and track proposers that should be rewarded - for index in range(len(state.validators)): - if index in attesting_indices and has_enough_for_reward(spec, state, index): - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Track proposer of earliest included attestation for the validator defined by index - earliest_attestation = min([ - a for a in eligible_attestations - if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) - ], key=lambda a: a.inclusion_delay) - rewarded_proposer_indices.add(earliest_attestation.proposer_index) - - # Ensure all expected proposers have been rewarded - # Track rewarde indices - proposing_indices = [a.proposer_index for a in eligible_attestations] - for index in proposing_indices: - if index in rewarded_proposer_indices: - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Ensure all expected non-rewarded indices received no reward - for index in range(len(state.validators)): - assert penalties[index] == 0 - if index not in rewarded_indices: - assert rewards[index] == 0 - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inclusion_delay_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += 1 - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += spec.SLOTS_PER_EPOCH - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_mixed_delay(spec, state): - rng = Random(1234) - - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) - - # Get an attestation where the proposer is not in the committee - non_proposer_attestations = [] - for a in state.previous_epoch_attestations: - if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): - non_proposer_attestations.append(a) - - assert any(non_proposer_attestations) - state.previous_epoch_attestations = non_proposer_attestations - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) - - # Remove 2/3 of attestations to make it more interesting - num_attestations = int(len(state.previous_epoch_attestations) * 0.33) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - - # Get map of the proposer at each slot to make valid-looking duplicate attestations - per_slot_proposers = { - (a.data.slot + a.inclusion_delay): a.proposer_index - for a in state.previous_epoch_attestations - } - max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) - later_attestations = [] - for a in state.previous_epoch_attestations: - # Only have proposers for previous epoch so do not create later - # duplicate if slot exceeds the max slot in previous_epoch_attestations - if a.data.slot + a.inclusion_delay >= max_slot: - continue - later_a = a.copy() - later_a.inclusion_delay += 1 - later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] - later_attestations.append(later_a) - - assert any(later_attestations) - - state.previous_epoch_attestations = sorted( - state.previous_epoch_attestations + later_attestations, - key=lambda a: a.data.slot + a.inclusion_delay - ) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) - - for index in range(len(state.validators)): - state.validators[index].effective_balance = 10 - - yield from run_get_inclusion_delay_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py deleted file mode 100644 index 5a02a246b..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ /dev/null @@ -1,140 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_target_deltas(spec, state): - """ - Run ``get_target_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_target_deltas, - spec.get_matching_target_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_target_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_target_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py new file mode 100644 index 000000000..562a99b4b --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -0,0 +1,165 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.state import next_epoch +import eth2spec.test.helpers.rewards as rewards_helpers + + +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +@with_all_phases +@spec_state_test +def test_empty_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_empty(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_all_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_half_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_partial(spec, state, 0.25) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) + + +@with_all_phases +@spec_state_test +def test_one_attestation_one_correct_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) + + +# +# NOTE: No source incorrect tests +# All PendingAttestations in state have source validated +# We choose to keep this invariant in these tests to not force clients to test with degenerate states +# + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_random_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_five_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=5) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_ten_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=10) + yield from rewards_helpers.run_test_full_random(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py new file mode 100644 index 000000000..bda0ca687 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -0,0 +1,22 @@ +from random import Random + +from eth2spec.test.context import with_all_phases, spec_state_test +import eth2spec.test.helpers.rewards as rewards_helpers + + +@with_all_phases +@spec_state_test +def test_full_random_0(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(1010)) + + +@with_all_phases +@spec_state_test +def test_full_random_1(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(2020)) + + +@with_all_phases +@spec_state_test +def test_full_random_2(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index f70a20f9c..b00e7a32a 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -1,8 +1,15 @@ # Rewards tests -The different rewards deltas sub-functions are testing individually with the test handlers, each returning the related `rewards`/`penalties`. +All rewards deltas sub-functions are tested for each test case. There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. -Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) +(See test condition documentation on how to run the tests.) + +`Deltas` is defined as: +```python +class Deltas(Container): + rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] + penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] +``` ## Test case format @@ -22,31 +29,47 @@ A YAML-encoded `BeaconState`, the state before running the rewards sub-function. Also available as `pre.ssz`. -### `deltas.yaml` +### `source_deltas.yaml` -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards sub-function +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function -Where `Deltas` is defined as: -```python -class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] -``` +Also available as `source_deltas.ssz`. -Also available as `deltas.ssz`. +### `target_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function + +Also available as `target_deltas.ssz`. + +### `head_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function + +Also available as `head_deltas.ssz`. + +### `inclusion_delay_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function + +Also available as `inclusion_delay_deltas.ssz`. + +### `inactivity_penalty_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function + +Also available as `inactivity_penalty_deltas.ssz`. ## Condition A handler of the `rewards` test-runner should process these cases, - calling the corresponding rewards deltas function (same name in spec). -This excludes all other parts of `process_rewards_and_penalties` + calling the corresponding rewards deltas function for each set of deltas. -The provided pre-state is ready to be input into the designated handler. +The provided pre-state is ready to be input into each rewards deltas function. The provided `deltas` should match the return values of the - handler. Specifically the following must hold true: + deltas function. Specifically the following must hold true for each set of deltas: ```python - deltas.rewards == handler(state)[0] - deltas.penalties == handler(state)[1] + deltas.rewards == deltas_function(state)[0] + deltas.penalties == deltas_function(state)[1] ``` diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index fd95dcfaa..d8dae74fa 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -3,11 +3,9 @@ from typing import Iterable from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.phase_0.rewards import ( - test_get_source_deltas, - test_get_target_deltas, - test_get_head_deltas, - test_get_inclusion_delay_deltas, - test_get_inactivity_penalty_deltas, + test_basic, + test_leak, + test_random, ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests @@ -37,14 +35,10 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('get_source_deltas', test_get_source_deltas, 'minimal'), - create_provider('get_source_deltas', test_get_source_deltas, 'mainnet'), - create_provider('get_target_deltas', test_get_target_deltas, 'minimal'), - create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), - create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), - create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'minimal'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'mainnet'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'minimal'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'mainnet'), + create_provider('get_deltas', test_basic, 'minimal'), + create_provider('get_deltas', test_basic, 'mainnet'), + create_provider('get_deltas', test_leak, 'minimal'), + create_provider('get_deltas', test_leak, 'mainnet'), + create_provider('get_deltas', test_random, 'minimal'), + create_provider('get_deltas', test_random, 'mainnet'), ]) From 0f20d8a9ba9da9ada7c2fee0a7d7cfa08ff6ba64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 01:55:17 +0200 Subject: [PATCH 2/8] leak state decorator, and test pre-state caching --- tests/core/pyspec/eth2spec/test/context.py | 53 ++++++++++------ .../test/phase_0/rewards/test_leak.py | 60 +++++++++++++------ 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd31..303f680fb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -9,6 +9,8 @@ from .utils import vector_test, with_meta_tags from random import Random from typing import Any, Callable, NewType, Sequence, TypedDict, Protocol +from lru import LRU + from importlib import reload @@ -48,28 +50,45 @@ class SpecForks(TypedDict, total=False): PHASE1: SpecPhase1 +def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], + spec: Spec, phases: SpecForks): + + p0 = phases[PHASE0] + balances = balances_fn(p0) + activation_threshold = threshold_fn(p0) + + state = create_genesis_state(spec=p0, validator_balances=balances, + activation_threshold=activation_threshold) + if spec.fork == PHASE1: + # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. + # Decide based on performance/consistency results later. + state = phases[PHASE1].upgrade_to_phase1(state) + # Shard state slot must lag behind BeaconState slot by at least 1 + # Will handle this more elegantly with fork mechanics + spec.process_slots(state, state.slot + 1) + + return state + + +_custom_state_cache_dict = LRU(size=10) + + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): def deco(fn): + def entry(*args, spec: Spec, phases: SpecForks, **kw): - try: - p0 = phases[PHASE0] - balances = balances_fn(p0) - activation_threshold = threshold_fn(p0) + # Use fork and file path to make a key for th + key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + global _custom_state_cache_dict + if key not in _custom_state_cache_dict: + state = _prepare_state(balances_fn, threshold_fn, spec, phases) + _custom_state_cache_dict[key] = state.get_backing() - state = create_genesis_state(spec=p0, validator_balances=balances, - activation_threshold=activation_threshold) - if spec.fork == PHASE1: - # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. - # Decide based on performance/consistency results later. - state = phases[PHASE1].upgrade_to_phase1(state) - # Shard state slot must lag behind BeaconState slot by at least 1 - # Will handle this more elegantly with fork mechanics - spec.process_slots(state, state.slot + 1) - - kw['state'] = state - except KeyError: - raise TypeError('Spec decorator must come within state decorator to inject spec into state.') + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_custom_state_cache_dict[key]) + kw['state'] = state return fn(*args, spec=spec, phases=phases, **kw) return entry return deco diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 562a99b4b..6080ec751 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,6 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.state import next_epoch import eth2spec.test.helpers.rewards as rewards_helpers +from lru import LRU def transition_state_to_leak(spec, state, epochs=None): @@ -12,80 +13,103 @@ def transition_state_to_leak(spec, state, epochs=None): next_epoch(spec, state) +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() + + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + @with_all_phases @spec_state_test +@leaking() def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test +@leaking() def test_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_half_full(spec, state) @with_all_phases @spec_state_test +@leaking() def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test +@leaking() def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test +@leaking() def test_one_attestation_one_correct_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) @@ -98,8 +122,8 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -110,8 +134,8 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -122,8 +146,8 @@ def test_full_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -134,8 +158,8 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -146,20 +170,20 @@ def test_full_half_incorrect_target_correct_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=5) def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=10) def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) yield from rewards_helpers.run_test_full_random(spec, state) From 75a0d60eb3339d457bfc70652cc6cacc8010461f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:25:32 +0200 Subject: [PATCH 3/8] cached epoch attestation preparation --- .../eth2spec/test/helpers/attestations.py | 21 +++++++++++ .../pyspec/eth2spec/test/helpers/rewards.py | 37 +++++++++---------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index e4be6a521..8e0501e4a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -6,6 +6,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +from lru import LRU def run_attestation_processing(spec, state, attestation, valid=True): @@ -373,6 +374,26 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): return attestations +_prep_state_cache_dict = LRU(size=10) + + +def cached_prepare_state_with_attestations(spec, state): + """ + Cached version of prepare_state_with_attestations, + but does not return anything, and does not support a participation fn argument + """ + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (spec.fork, state.hash_tree_root()) + global _prep_state_cache_dict + if key not in _prep_state_cache_dict: + prepare_state_with_attestations(spec, state) + _prep_state_cache_dict[key] = state.get_backing() + + # Put the LRU cache result into the state view, as if we transitioned the original view + state.set_backing(_prep_state_cache_dict[key]) + + def fill_block_shard_transitions_by_attestations(spec, state, block): block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS for attestation in block.body.attestations: diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index fc9f1f93a..42e7a7614 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,7 +1,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.helpers.attestations import prepare_state_with_attestations +from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -217,13 +217,13 @@ def run_test_empty(spec, state): def run_test_full_all_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] @@ -232,7 +232,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): def run_test_partial(spec, state, fraction_filled): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) @@ -246,7 +246,7 @@ def run_test_half_full(spec, state): def run_test_one_attestation_one_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] @@ -256,14 +256,14 @@ def run_test_one_attestation_one_correct(spec, state): def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) @@ -272,14 +272,13 @@ def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_some_very_low_effective_balances_that_attested(spec, state): - state.balances - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Set some balances to be very low (including 0) assert len(state.validators) >= 5 @@ -290,7 +289,7 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove attestation attestation = state.previous_epoch_attestations[0] @@ -304,7 +303,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified num_incorrect = int(fraction_incorrect * len(state.previous_epoch_attestations)) @@ -318,7 +317,7 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += 1 @@ -326,7 +325,7 @@ def run_test_full_delay_one_slot(spec, state): def run_test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += spec.SLOTS_PER_EPOCH @@ -334,7 +333,7 @@ def run_test_full_delay_max_slots(spec, state): def run_test_full_mixed_delay(spec, state, rng=Random(1234)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) @@ -342,7 +341,7 @@ def run_test_full_mixed_delay(spec, state, rng=Random(1234)): def run_test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Get an attestation where the proposer is not in the committee non_proposer_attestations = [] @@ -357,7 +356,7 @@ def run_test_proposer_not_in_attestations(spec, state): def run_test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove 2/3 of attestations to make it more interesting num_attestations = int(len(state.previous_epoch_attestations) * 0.33) @@ -391,7 +390,7 @@ def run_test_duplicate_attestations_at_later_slots(spec, state): def run_test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for index in range(len(state.validators)): state.validators[index].effective_balance = 10 @@ -404,7 +403,7 @@ def run_test_full_random(spec, state, rng=Random(8020)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: # ~1/3 have bad target From 97e54b85d6c4c6663f97e4a9e3d51b10c1945300 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:46:27 +0200 Subject: [PATCH 4/8] fix cache for config change during runtime --- tests/core/pyspec/eth2spec/test/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 303f680fb..93b474517 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -78,8 +78,9 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def deco(fn): def entry(*args, spec: Spec, phases: SpecForks, **kw): - # Use fork and file path to make a key for th - key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + # make a key for the state + # genesis fork version separates configs during test-generation runtime. + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) global _custom_state_cache_dict if key not in _custom_state_cache_dict: state = _prepare_state(balances_fn, threshold_fn, spec, phases) From 85f6712363802b6b1f4a1a9464f8847f02a57fb1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 03:42:58 +0200 Subject: [PATCH 5/8] fix comments based on PR feedback --- tests/core/pyspec/eth2spec/test/context.py | 2 +- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 7 ++++--- .../pyspec/eth2spec/test/phase_0/rewards/test_leak.py | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 93b474517..20214908e 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -86,7 +86,7 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], state = _prepare_state(balances_fn, threshold_fn, spec, phases) _custom_state_cache_dict[key] = state.get_backing() - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_custom_state_cache_dict[key]) kw['state'] = state diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8e0501e4a..79f752411 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -382,13 +382,14 @@ def cached_prepare_state_with_attestations(spec, state): Cached version of prepare_state_with_attestations, but does not return anything, and does not support a participation fn argument """ - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # prepare it with attestations, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (spec.fork, state.hash_tree_root()) global _prep_state_cache_dict if key not in _prep_state_cache_dict: prepare_state_with_attestations(spec, state) - _prep_state_cache_dict[key] = state.get_backing() + _prep_state_cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 6080ec751..4e75079c0 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -20,15 +20,16 @@ def leaking(epochs=None): def deco(fn): def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) global _cache_dict if key not in _cache_dict: transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_cache_dict[key]) return fn(*args, spec=spec, state=state, **kw) From 6d45afeefbf5f76ea61eb0f4d71e90402c3744c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 19:53:46 -0600 Subject: [PATCH 6/8] add some more random reawrds tests --- .../pyspec/eth2spec/test/helpers/rewards.py | 3 +++ .../test/phase_0/rewards/test_random.py | 25 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 42e7a7614..d62fee6ce 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -174,6 +174,9 @@ def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited for index in range(num_validators): + # If not already active, skip + if not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)): + continue if rng.randrange(num_validators) < num_validators // 10: mock_deposit(spec, state, index) # Set ~half of selected to eligible for activation diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py index bda0ca687..83c7f7905 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -1,6 +1,13 @@ from random import Random -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import ( + with_all_phases, + spec_test, + spec_state_test, + with_custom_state, + single_phase, + low_balances, misc_balances, +) import eth2spec.test.helpers.rewards as rewards_helpers @@ -20,3 +27,19 @@ def test_full_random_1(spec, state): @spec_state_test def test_full_random_2(spec, state): yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) + + +@with_all_phases +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_low_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_misc_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) From 5c564f4d7c9463e553641fc1792f879a2aadecb5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 20:06:36 -0600 Subject: [PATCH 7/8] clean up reards generator --- tests/generators/rewards/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index d8dae74fa..c90943cab 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -14,7 +14,7 @@ from eth2spec.config import config_util from eth2spec.test.context import PHASE0 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -25,7 +25,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( runner_name='rewards', - handler_name=handler_name, + handler_name='core', src=tests_src, fork_name=PHASE0, ) @@ -34,11 +34,11 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('get_deltas', test_basic, 'minimal'), - create_provider('get_deltas', test_basic, 'mainnet'), - create_provider('get_deltas', test_leak, 'minimal'), - create_provider('get_deltas', test_leak, 'mainnet'), - create_provider('get_deltas', test_random, 'minimal'), - create_provider('get_deltas', test_random, 'mainnet'), + gen_runner.run_generator("rewards", [ + create_provider(test_basic, 'minimal'), + create_provider(test_basic, 'mainnet'), + create_provider(test_leak, 'minimal'), + create_provider(test_leak, 'mainnet'), + create_provider(test_random, 'minimal'), + create_provider(test_random, 'mainnet'), ]) From 65b5aa3c636234d024c89c3f3a9df87700b5258a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 14:31:28 +0200 Subject: [PATCH 8/8] update test format docs --- tests/formats/epoch_processing/README.md | 2 +- tests/formats/rewards/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 7c5e2dc70..57c9441c8 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -38,7 +38,7 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: - `justification_and_finalization` -- *`rewards_and_penalties` - planned testing extension* +- `rewards_and_penalties` (limited to `minimal` config) - `registry_updates` - `slashings` - `final_updates` diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index b00e7a32a..b229d9f98 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -7,8 +7,8 @@ There is no "change" factor, the rewards/penalties outputs are pure functions wi `Deltas` is defined as: ```python class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] + rewards: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + penalties: List[Gwei, VALIDATOR_REGISTRY_LIMIT] ``` ## Test case format