Fix Altair epoch processing and add/enable remaining epoch processing tests (#2619)

* fix Altair epoch state transitions

* re-enable and re-add near-complete Altair epoch processing test vectors: inactivity, justification & finalization, participation flag, slashings, and sync committee updates
This commit is contained in:
tersec 2021-05-30 20:23:10 +00:00 committed by GitHub
parent 820a6f65d5
commit 8912196335
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 146 additions and 60 deletions

View File

@ -161,6 +161,11 @@ FixtureAll-mainnet
+ [Valid] new_deposit_under_max OK
+ [Valid] success_top_up OK
+ [Valid] valid_sig_but_forked_state OK
+ [Valid] Official - Altair - Finality - finality_no_updates_at_genesis [Preset: mainnet] OK
+ [Valid] Official - Altair - Finality - finality_rule_1 [Preset: mainnet] OK
+ [Valid] Official - Altair - Finality - finality_rule_2 [Preset: mainnet] OK
+ [Valid] Official - Altair - Finality - finality_rule_3 [Preset: mainnet] OK
+ [Valid] Official - Altair - Finality - finality_rule_4 [Preset: mainnet] OK
+ [Valid] Official - Altair - Sanity - Blocks - attestation [Preset: mainnet] OK
+ [Valid] Official - Altair - Sanity - Blocks - attester_slashing [Preset: mainnet] OK
+ [Valid] Official - Altair - Sanity - Blocks - balance_driven_status_transitions [Preset: OK
@ -238,7 +243,7 @@ FixtureAll-mainnet
+ [Valid] sync_committee_rewards_empty_participants OK
+ [Valid] sync_committee_rewards_not_full_participants OK
```
OK: 235/235 Fail: 0/235 Skip: 0/235
OK: 240/240 Fail: 0/240 Skip: 0/240
## Official - Altair - Epoch Processing - Effective balance updates [Preset: mainnet]
```diff
+ Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK
@ -255,6 +260,40 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
+ Historical roots update - historical_root_accumulator [Preset: mainnet] OK
```
OK: 1/1 Fail: 0/1 Skip: 0/1
## Official - Altair - Epoch Processing - Inactivity [Preset: mainnet]
```diff
+ Inactivity - all_zero_inactivity_scores_empty_participation [Preset: mainnet] OK
+ Inactivity - all_zero_inactivity_scores_full_participation [Preset: mainnet] OK
+ Inactivity - all_zero_inactivity_scores_random_participation [Preset: mainnet] OK
+ Inactivity - genesis [Preset: mainnet] OK
+ Inactivity - random_inactivity_scores_empty_participation [Preset: mainnet] OK
+ Inactivity - random_inactivity_scores_full_participation [Preset: mainnet] OK
+ Inactivity - random_inactivity_scores_random_participation [Preset: mainnet] OK
```
OK: 7/7 Fail: 0/7 Skip: 0/7
## Official - Altair - Epoch Processing - Justification & Finalization [Preset: mainnet]
```diff
+ Justification & Finalization - 123_ok_support [Preset: mainnet] OK
+ Justification & Finalization - 123_poor_support [Preset: mainnet] OK
+ Justification & Finalization - 12_ok_support [Preset: mainnet] OK
+ Justification & Finalization - 12_ok_support_messed_target [Preset: mainnet] OK
+ Justification & Finalization - 12_poor_support [Preset: mainnet] OK
+ Justification & Finalization - 234_ok_support [Preset: mainnet] OK
+ Justification & Finalization - 234_poor_support [Preset: mainnet] OK
+ Justification & Finalization - 23_ok_support [Preset: mainnet] OK
+ Justification & Finalization - 23_poor_support [Preset: mainnet] OK
```
OK: 9/9 Fail: 0/9 Skip: 0/9
## Official - Altair - Epoch Processing - Participation flag updates [Preset: mainnet]
```diff
+ Participation flag updates - filled [Preset: mainnet] OK
+ Participation flag updates - prev_zeroed [Preset: mainnet] OK
+ Participation flag updates - random [Preset: mainnet] OK
+ Participation flag updates - random_genesis [Preset: mainnet] OK
+ Participation flag updates - zeroed [Preset: mainnet] OK
+ Participation flag updates - zeroing [Preset: mainnet] OK
```
OK: 6/6 Fail: 0/6 Skip: 0/6
## Official - Altair - Epoch Processing - RANDAO mixes reset [Preset: mainnet]
```diff
+ RANDAO mixes reset - updated_randao_mixes [Preset: mainnet] OK
@ -272,6 +311,14 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
+ Registry updates - ejection_past_churn_limit [Preset: mainnet] OK
```
OK: 8/8 Fail: 0/8 Skip: 0/8
## Official - Altair - Epoch Processing - Slashings [Preset: mainnet]
```diff
+ Slashings - low_penalty [Preset: mainnet] OK
+ Slashings - max_penalties [Preset: mainnet] OK
+ Slashings - minimal_penalty [Preset: mainnet] OK
+ Slashings - scaled_penalties [Preset: mainnet] OK
```
OK: 4/4 Fail: 0/4 Skip: 0/4
## Official - Altair - Epoch Processing - Slashings reset [Preset: mainnet]
```diff
+ Slashings reset - flush_slashings [Preset: mainnet] OK
@ -414,4 +461,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 27/27 Fail: 0/27 Skip: 0/27
---TOTAL---
OK: 340/340 Fail: 0/340 Skip: 0/340
OK: 371/371 Fail: 0/371 Skip: 0/371

View File

@ -169,10 +169,16 @@ func get_unslashed_participating_indices(
state.current_epoch_participation
else:
state.previous_epoch_participation
# TODO use cached version, or similar
active_validator_indices = get_active_validator_indices(state, epoch)
participating_indices = filterIt(
active_validator_indices, has_flag(epoch_participation[it], flag_index))
toHashSet(filterIt(participating_indices, not state.validators[it].slashed))
var res: HashSet[ValidatorIndex]
for validator_index in active_validator_indices:
if has_flag(epoch_participation[validator_index], flag_index) and
not state.validators[validator_index].slashed:
res.incl validator_index
res
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#justification-and-finalization
proc process_justification_and_finalization*(state: var phase0.BeaconState,
@ -275,7 +281,7 @@ proc process_justification_and_finalization*(state: var phase0.BeaconState,
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#justification-and-finalization
# TODO merge these things -- effectively, the phase0 process_justification_and_finalization is mostly a stub in this world
proc weigh_justification_and_finalization(state: var altair.BeaconState,
total_balances: TotalBalances,
total_active_balance: Gwei,
previous_epoch_target_balance: Gwei,
current_epoch_target_balance: Gwei,
flags: UpdateFlags = {}) =
@ -298,9 +304,7 @@ proc weigh_justification_and_finalization(state: var altair.BeaconState,
state.justification_bits = (state.justification_bits shl 1) and
cast[uint8]((2^JUSTIFICATION_BITS_LENGTH) - 1)
let total_active_balance = total_balances.current_epoch
if total_balances.previous_epoch_target_attesters * 3 >=
total_active_balance * 2:
if previous_epoch_target_balance * 3 >= total_active_balance * 2:
state.current_justified_checkpoint =
Checkpoint(epoch: previous_epoch,
root: get_block_root(state, previous_epoch))
@ -311,10 +315,12 @@ proc weigh_justification_and_finalization(state: var altair.BeaconState,
checkpoint = shortLog(state.current_justified_checkpoint)
elif verifyFinalization in flags:
warn "Low attestation participation in previous epoch",
total_balances, epoch = get_current_epoch(state)
total_active_balance,
previous_epoch_target_balance,
current_epoch_target_balance,
epoch = get_current_epoch(state)
if total_balances.current_epoch_target_attesters * 3 >=
total_active_balance * 2:
if current_epoch_target_balance * 3 >= total_active_balance * 2:
state.current_justified_checkpoint =
Checkpoint(epoch: current_epoch,
root: get_block_root(state, current_epoch))
@ -368,7 +374,7 @@ proc weigh_justification_and_finalization(state: var altair.BeaconState,
checkpoint = shortLog(state.finalized_checkpoint)
proc process_justification_and_finalization*(state: var altair.BeaconState,
total_balances: TotalBalances, flags: UpdateFlags = {}) {.nbench.} =
total_active_balance: Gwei, flags: UpdateFlags = {}) {.nbench.} =
# Initial FFG checkpoint values have a `0x00` stub for `root`.
# Skip FFG updates in the first two epochs to avoid corner cases that might
# result in modifying this stub.
@ -377,11 +383,15 @@ proc process_justification_and_finalization*(state: var altair.BeaconState,
let
# these ultimately differ from phase0 only in these lines
# ref: https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#justification-and-finalization
previous_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state))
current_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, get_current_epoch(state))
previous_indices = get_unslashed_participating_indices(
state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state))
current_indices = get_unslashed_participating_indices(
state, TIMELY_TARGET_FLAG_INDEX, get_current_epoch(state))
previous_target_balance = get_total_balance(state, previous_indices)
current_target_balance = get_total_balance(state, current_indices)
weigh_justification_and_finalization(state, total_balances, previous_target_balance, current_target_balance, flags)
weigh_justification_and_finalization(
state, total_active_balance, previous_target_balance,
current_target_balance, flags)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#helpers
func get_base_reward_sqrt*(state: phase0.BeaconState, index: ValidatorIndex,
@ -550,25 +560,30 @@ func get_attestation_deltas(state: phase0.BeaconState, rewards: var RewardInfo)
proposer_delta.get()[1])
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward_per_increment
func get_base_reward_per_increment(state: altair.BeaconState, total_balances: TotalBalances): Gwei =
func get_base_reward_per_increment(
state: altair.BeaconState, total_active_balance: Gwei): Gwei =
# TODO hoist this integer_squareroot, as with phase 0
EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR div
integer_squareroot(total_balances.current_epoch())
integer_squareroot(total_active_balance)
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward
func get_base_reward(state: altair.BeaconState, index: ValidatorIndex, total_balances: TotalBalances): Gwei =
func get_base_reward(
state: altair.BeaconState, index: ValidatorIndex, total_active_balance: Gwei):
Gwei =
## Return the base reward for the validator defined by ``index`` with respect
## to the current ``state``.
##
## Note: An optimally performing validator can earn one base reward per epoch
## over a long time horizon. This takes into account both per-epoch (e.g.
## attestation) and intermittent duties (e.g. block proposal and sync
## committees).
#
# Note: An optimally performing validator can earn one base reward per epoch
# over a long time horizon. This takes into account both per-epoch (e.g.
# attestation) and intermittent duties (e.g. block proposal and sync
# committees).
let increments =
state.validators[index].effective_balance div EFFECTIVE_BALANCE_INCREMENT
increments * get_base_reward_per_increment(state, total_balances)
increments * get_base_reward_per_increment(state, total_active_balance)
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_flag_index_deltas
proc get_flag_index_deltas(state: altair.BeaconState, flag_index: int, total_balances: TotalBalances):
proc get_flag_index_deltas(
state: altair.BeaconState, flag_index: int, total_active_balance: Gwei):
(seq[Gwei], seq[Gwei]) =
## Return the deltas for a given ``flag_index`` by scanning through the
## participation flags.
@ -581,7 +596,7 @@ proc get_flag_index_deltas(state: altair.BeaconState, flag_index: int, total_bal
weight = PARTICIPATION_FLAG_WEIGHTS[flag_index].uint64 # safe
unslashed_participating_balance = get_total_balance(state, unslashed_participating_indices)
unslashed_participating_increments = unslashed_participating_balance div EFFECTIVE_BALANCE_INCREMENT
active_increments = total_balances.current_epoch() div EFFECTIVE_BALANCE_INCREMENT
active_increments = total_active_balance div EFFECTIVE_BALANCE_INCREMENT
for index in 0 ..< state.validators.len:
# TODO Obviously not great
@ -589,7 +604,7 @@ proc get_flag_index_deltas(state: altair.BeaconState, flag_index: int, total_bal
if not (is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch)):
continue
let base_reward = get_base_reward(state, index.ValidatorIndex, total_balances)
let base_reward = get_base_reward(state, index.ValidatorIndex, total_active_balance)
if index.ValidatorIndex in unslashed_participating_indices:
if not is_in_inactivity_leak(state):
let reward_numerator = base_reward * weight * unslashed_participating_increments
@ -644,18 +659,14 @@ func process_rewards_and_penalties(
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#rewards-and-penalties
proc process_rewards_and_penalties(
state: var altair.BeaconState, rewards: var RewardInfo) {.nbench.} =
# No rewards are applied at the end of `GENESIS_EPOCH` because rewards are
# for work done in the previous epoch
doAssert rewards.statuses.len == state.validators.len
state: var altair.BeaconState, total_active_balance: Gwei) {.nbench.} =
if get_current_epoch(state) == GENESIS_EPOCH:
return
# TODO look at phase0 optimizations, however relevant they still are. Altair
# is supposed to incorporate some of this into the protocol, so re-assess
# TODO efficiency-wise, presumably a better way. look at again, once tests pass
var deltas = mapIt(0 ..< PARTICIPATION_FLAG_WEIGHTS.len, get_flag_index_deltas(state, it, rewards.total_balances))
# TODO assess relevance of missing phase0 optimizations
var deltas = mapIt(
0 ..< PARTICIPATION_FLAG_WEIGHTS.len,
get_flag_index_deltas(state, it, total_active_balance))
deltas.add get_inactivity_penalty_deltas(state)
for (rewards, penalties) in deltas:
for index in 0 ..< len(state.validators):
@ -757,7 +768,7 @@ func process_participation_record_updates*(state: var phase0.BeaconState) {.nben
swap(state.previous_epoch_attestations, state.current_epoch_attestations)
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#participation-flags-updates
func process_participation_flag_updates(state: var altair.BeaconState) =
func process_participation_flag_updates*(state: var altair.BeaconState) =
state.previous_epoch_participation = state.current_epoch_participation
# TODO more subtle clearing
@ -766,14 +777,14 @@ func process_participation_flag_updates(state: var altair.BeaconState) =
doAssert state.current_epoch_participation.add 0.ParticipationFlags
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-updates
proc process_sync_committee_updates(state: var altair.BeaconState) =
proc process_sync_committee_updates*(state: var altair.BeaconState) =
let next_epoch = get_current_epoch(state) + 1
if next_epoch mod EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0:
state.current_sync_committee = state.next_sync_committee
state.next_sync_committee = get_next_sync_committee(state)
# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#inactivity-scores
func process_inactivity_updates(state: var altair.BeaconState) =
func process_inactivity_updates*(state: var altair.BeaconState) =
# Score updates based on previous epoch participation, skip genesis epoch
if get_current_epoch(state) == GENESIS_EPOCH:
return
@ -851,11 +862,12 @@ proc process_epoch*(
current_epoch = currentEpoch
init(rewards, state)
when false:
# TODO this is the key thing which needs porting over
rewards.process_attestations(state, cache)
let total_active_balance = state.get_total_active_balance(cache)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#justification-and-finalization
process_justification_and_finalization(state, rewards.total_balances, flags)
process_justification_and_finalization(state, total_active_balance, flags)
# state.slot hasn't been incremented yet.
if verifyFinalization in flags and currentEpoch >= 2:
@ -870,13 +882,13 @@ proc process_epoch*(
process_inactivity_updates(state) # [New in Altair]
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#rewards-and-penalties-1
process_rewards_and_penalties(state, rewards)
process_rewards_and_penalties(state, total_active_balance)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#registry-updates
process_registry_updates(state, cache)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#slashings
process_slashings(state, rewards.total_balances.current_epoch)
process_slashings(state, total_active_balance)
process_eth1_data_reset(state)

View File

@ -74,6 +74,5 @@ suite "Official - Altair - Sanity - Blocks " & preset():
runTest("Official - Altair - Sanity - Blocks", SanityBlocksDir, path)
suite "Official - Altair - Finality " & preset():
when false:
for kind, path in walkDir(FinalityDir, true):
runTest("Official - Altair - Finality", FinalityDir, path)
for kind, path in walkDir(FinalityDir, true):
runTest("Official - Altair - Finality", FinalityDir, path)

View File

@ -12,7 +12,7 @@ import
os, strutils,
# Beacon chain internals
../../../beacon_chain/spec/state_transition_epoch,
../../../beacon_chain/spec/datatypes/altair,
../../../beacon_chain/spec/[datatypes/altair, beaconstate],
# Test utilities
../../testutil,
../fixtures_utils,
@ -22,7 +22,9 @@ import
from ../../../beacon_chain/spec/beaconstate import process_registry_updates
# XXX: move to state_transition_epoch?
template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, useCache: static bool): untyped =
template runSuite(
suiteDir, testName: string, transitionProc: untyped{ident},
useCache, useTAB: static bool = false): untyped =
suite "Official - Altair - Epoch Processing - " & testName & preset():
doAssert dirExists(suiteDir)
for testDir in walkDirRec(suiteDir, yieldFilter = {pcDir}):
@ -33,9 +35,14 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us
var preState = newClone(parseTest(testDir/"pre.ssz_snappy", SSZ, BeaconState))
let postState = newClone(parseTest(testDir/"post.ssz_snappy", SSZ, BeaconState))
doAssert not (useCache and useTAB)
when useCache:
var cache = StateCache()
transitionProc(preState[], cache)
elif useTAB:
var cache = StateCache()
let total_active_balance = preState[].get_total_active_balance(cache)
transitionProc(preState[], total_active_balance)
else:
transitionProc(preState[])
@ -45,9 +52,13 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us
# ---------------------------------------------------------------
const JustificationFinalizationDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests"
when false:
# TODO
runSuite(JustificationFinalizationDir, "Justification & Finalization", process_justification_and_finalization, useCache = false)
runSuite(JustificationFinalizationDir, "Justification & Finalization", process_justification_and_finalization, useCache = false, useTAB = true)
# Inactivity updates
# ---------------------------------------------------------------
const InactivityDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"inactivity_updates"/"pyspec_tests"
runSuite(InactivityDir, "Inactivity", process_inactivity_updates, useCache = false)
# Rewards & Penalties
# ---------------------------------------------------------------
@ -64,29 +75,46 @@ runSuite(RegistryUpdatesDir, "Registry updates", process_registry_updates, useC
# ---------------------------------------------------------------
const SlashingsDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"slashings"/"pyspec_tests"
when false:
# TODO needs totalbalance info
runSuite(SlashingsDir, "Slashings", process_slashings, useCache = false)
runSuite(SlashingsDir, "Slashings", process_slashings, useCache = false, useTAB = true)
# Final updates
# Eth1 data reset
# ---------------------------------------------------------------
const Eth1DataResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"eth1_data_reset/"/"pyspec_tests"
runSuite(Eth1DataResetDir, "Eth1 data reset", process_eth1_data_reset, useCache = false)
# Effective balance updates
# ---------------------------------------------------------------
const EffectiveBalanceUpdatesDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"effective_balance_updates"/"pyspec_tests"
runSuite(EffectiveBalanceUpdatesDir, "Effective balance updates", process_effective_balance_updates, useCache = false)
# Slashings reset
# ---------------------------------------------------------------
const SlashingsResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"slashings_reset"/"pyspec_tests"
runSuite(SlashingsResetDir, "Slashings reset", process_slashings_reset, useCache = false)
# RANDAO mixes reset
# ---------------------------------------------------------------
const RandaoMixesResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"randao_mixes_reset"/"pyspec_tests"
runSuite(RandaoMixesResetDir, "RANDAO mixes reset", process_randao_mixes_reset, useCache = false)
# Historical roots update
# ---------------------------------------------------------------
const HistoricalRootsUpdateDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"historical_roots_update"/"pyspec_tests"
runSuite(HistoricalRootsUpdateDir, "Historical roots update", process_historical_roots_update, useCache = false)
const ParticipationRecordsDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"participation_record_updates"/"pyspec_tests"
when false:
# TODO needs totalbalance info
runSuite(ParticipationRecordsDir, "Participation record updates", process_participation_record_updates, useCache = false)
# Participation flag updates
# ---------------------------------------------------------------
const ParticipationFlagDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"participation_flag_updates"/"pyspec_tests"
runSuite(ParticipationFlagDir, "Participation flag updates", process_participation_flag_updates, useCache = false)
# Sync committee updates
# ---------------------------------------------------------------
const SyncCommitteeDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"sync_committee_updates"/"pyspec_tests"
runSuite(SyncCommitteeDir, "Sync committee updates", process_sync_committee_updates, useCache = false)