diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 10dbb8a56..efda9fa7d 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -118,16 +118,16 @@ func compute_activation_exit_epoch(epoch: Epoch): Epoch = epoch + 1 + MAX_SEED_LOOKAHEAD # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_validator_churn_limit -func get_validator_churn_limit(state: BeaconState): uint64 = +func get_validator_churn_limit(state: BeaconState, cache: var StateCache): + uint64 = # Return the validator churn limit for the current epoch. - let active_validator_indices = - get_active_validator_indices(state, get_current_epoch(state)) max(MIN_PER_EPOCH_CHURN_LIMIT, - len(active_validator_indices) div CHURN_LIMIT_QUOTIENT).uint64 + len(cache.shuffled_active_validator_indices) div + CHURN_LIMIT_QUOTIENT).uint64 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#initiate_validator_exit func initiate_validator_exit*(state: var BeaconState, - index: ValidatorIndex) = + index: ValidatorIndex, cache: var StateCache) = # Initiate the exit of the validator with index ``index``. # Return if validator already initiated exit @@ -146,7 +146,7 @@ func initiate_validator_exit*(state: var BeaconState, a + (if b.exit_epoch == exit_queue_epoch: 1'u64 else: 0'u64), 0'u64) - if exit_queue_churn >= get_validator_churn_limit(state): + if exit_queue_churn >= get_validator_churn_limit(state, cache): exit_queue_epoch += 1 # Set validator exit epoch and withdrawable epoch @@ -156,10 +156,10 @@ func initiate_validator_exit*(state: var BeaconState, # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#slash_validator proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, - stateCache: var StateCache) = + cache: var StateCache) = # Slash the validator with index ``index``. let epoch = get_current_epoch(state) - initiate_validator_exit(state, slashed_index) + initiate_validator_exit(state, slashed_index, cache) let validator = addr state.validators[slashed_index] debug "slash_validator: ejecting validator via slashing (validator_leaving)", @@ -181,7 +181,7 @@ proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, # The rest doesn't make sense without there being any proposer index, so skip # Apply proposer and whistleblower rewards - let proposer_index = get_beacon_proposer_index(state, stateCache) + let proposer_index = get_beacon_proposer_index(state, cache) if proposer_index.isNone: debug "No beacon proposer index and probably no active validators" return @@ -282,6 +282,7 @@ proc initialize_hashed_beacon_state_from_eth1*( func is_valid_genesis_state*(state: BeaconState): bool = if state.genesis_time < MIN_GENESIS_TIME: return false + # This is an okay get_active_validator_indices(...) for the time being. if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: return false return true @@ -341,7 +342,8 @@ func is_eligible_for_activation(state: BeaconState, validator: Validator): validator.activation_epoch == FAR_FUTURE_EPOCH # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#registry-updates -proc process_registry_updates*(state: var BeaconState) {.nbench.}= +proc process_registry_updates*(state: var BeaconState, + cache: var StateCache) {.nbench.}= ## Process activation eligibility and ejections ## Try to avoid caching here, since this could easily become undefined @@ -354,6 +356,12 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}= active_validator_indices=get_active_validator_indices(state, epoch), epoch=epoch + # is_active_validator(...) is activation_epoch <= epoch < exit_epoch, + # and changes here to either activation_epoch or exit_epoch only take + # effect with a compute_activation_exit_epoch(...) delay of, based on + # the current epoch, 1 + MAX_SEED_LOOKAHEAD epochs ahead. Thus caches + # remain valid for this epoch through though this function along with + # the rest of the epoch transition. for index, validator in state.validators: if is_eligible_for_activation_queue(validator): state.validators[index].activation_eligibility_epoch = @@ -369,7 +377,7 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}= validator_withdrawable_epoch = validator.withdrawable_epoch, validator_exit_epoch = validator.exit_epoch, validator_effective_balance = validator.effective_balance - initiate_validator_exit(state, index.ValidatorIndex) + initiate_validator_exit(state, index.ValidatorIndex, cache) ## Queue validators eligible for activation and not dequeued for activation var activation_queue : seq[tuple[a: Epoch, b: int]] = @[] @@ -382,7 +390,7 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}= ## Dequeued validators for activation up to churn limit (without resetting ## activation epoch) - let churn_limit = get_validator_churn_limit(state) + let churn_limit = get_validator_churn_limit(state, cache) for i, epoch_and_index in activation_queue: if i.uint64 >= churn_limit: break diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index 6e5ad3a8d..413ece23b 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -490,6 +490,7 @@ proc `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) = # `ValidatorIndex` Nim integration proc `==`*(x, y: ValidatorIndex) : bool {.borrow.} +proc `<`*(x, y: ValidatorIndex) : bool {.borrow.} proc hash*(x: ValidatorIndex): Hash {.borrow.} proc `$`*(x: ValidatorIndex): auto = $(x.int64) diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index dec2a1caf..99b6509e0 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -303,7 +303,9 @@ proc process_voluntary_exit*( validator_withdrawable_epoch = validator.withdrawable_epoch, validator_exit_epoch = validator.exit_epoch, validator_effective_balance = validator.effective_balance - initiate_validator_exit(state, voluntary_exit.validator_index.ValidatorIndex) + var cache = get_empty_per_epoch_cache() + initiate_validator_exit( + state, voluntary_exit.validator_index.ValidatorIndex, cache) true diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 50fcf0148..4e6c39838 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -67,14 +67,23 @@ declareGauge beacon_current_epoch, "Current epoch" # -------------------------------------------------------- # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_total_active_balance -func get_total_active_balance*(state: BeaconState): Gwei = +func get_total_active_balance*(state: BeaconState, cache: var StateCache): Gwei = # Return the combined effective balance of the active validators. # Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei # minimum to avoid divisions by zero. - # TODO it calls get_total_balance with set(g_a_v_i(...)) - get_total_balance( - state, - get_active_validator_indices(state, get_current_epoch(state))) + + # TODO refactor get_epoch_per_epoch_cache() not to be, well, empty, so can + # avoid this ever refilling, and raiseAssert, and get rid of var + let + epoch = state.slot.compute_epoch_at_slot + try: + if epoch notin cache.shuffled_active_validator_indices: + cache.shuffled_active_validator_indices[epoch] = + get_shuffled_active_validator_indices(state, epoch) + + get_total_balance(state, cache.shuffled_active_validator_indices[epoch]) + except KeyError: + raiseAssert("get_total_active_balance(): cache always filled before usage") # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#helper-functions-1 func get_matching_source_attestations(state: BeaconState, @@ -155,6 +164,7 @@ proc process_justification_and_finalization*(state: var BeaconState, ## and ## https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#final-updates ## after which the state.previous_epoch_attestations is replaced. + let total_active_balance = get_total_active_balance(state, stateCache) trace "Non-attesting indices in previous epoch", missing_all_validators= difference(active_validator_indices, @@ -167,10 +177,10 @@ proc process_justification_and_finalization*(state: var BeaconState, prev_attestations_len=len(state.previous_epoch_attestations), cur_attestations_len=len(state.current_epoch_attestations), num_active_validators=len(active_validator_indices), - required_balance = get_total_active_balance(state) * 2, + required_balance = total_active_balance * 2, attesting_balance_prev = get_attesting_balance(state, matching_target_attestations_previous, stateCache) if get_attesting_balance(state, matching_target_attestations_previous, - stateCache) * 3 >= get_total_active_balance(state) * 2: + stateCache) * 3 >= total_active_balance * 2: state.current_justified_checkpoint = Checkpoint(epoch: previous_epoch, root: get_block_root(state, previous_epoch)) @@ -184,7 +194,7 @@ proc process_justification_and_finalization*(state: var BeaconState, let matching_target_attestations_current = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations_current, - stateCache) * 3 >= get_total_active_balance(state) * 2: + stateCache) * 3 >= total_active_balance * 2: state.current_justified_checkpoint = Checkpoint(epoch: current_epoch, root: get_block_root(state, current_epoch)) @@ -256,7 +266,7 @@ func get_attestation_deltas(state: BeaconState, stateCache: var StateCache): tuple[a: seq[Gwei], b: seq[Gwei]] {.nbench.}= let previous_epoch = get_previous_epoch(state) - total_balance = get_total_active_balance(state) + total_balance = get_total_active_balance(state, stateCache) var rewards = repeat(0'u64, len(state.validators)) penalties = repeat(0'u64, len(state.validators)) @@ -360,10 +370,10 @@ func process_rewards_and_penalties( decrease_balance(state, i.ValidatorIndex, penalties[i]) # https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#slashings -func process_slashings*(state: var BeaconState) {.nbench.}= +func process_slashings*(state: var BeaconState, cache: var StateCache) {.nbench.}= let epoch = get_current_epoch(state) - total_balance = get_total_active_balance(state) + total_balance = get_total_active_balance(state, cache) for index, validator in state.validators: if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR div 2 == @@ -443,16 +453,10 @@ proc process_epoch*(state: var BeaconState, updateFlags: UpdateFlags, process_rewards_and_penalties(state, per_epoch_cache) # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#registry-updates - # Don't rely on caching here. - process_registry_updates(state) - - ## Caching here for get_beacon_committee(...) can break otherwise, since - ## get_active_validator_indices(...) usually changes. - per_epoch_cache.shuffled_active_validator_indices[currentEpoch] = - get_shuffled_active_validator_indices(state, currentEpoch) + process_registry_updates(state, per_epoch_cache) # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#slashings - process_slashings(state) + process_slashings(state, per_epoch_cache) # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#final-updates process_final_updates(state) diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 6be71c582..ce6f1ad98 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -9,7 +9,7 @@ {.push raises: [Defect].} import - options, sequtils, math, tables, + algorithm, options, sequtils, math, tables, ./datatypes, ./digest, ./helpers # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_shuffled_index @@ -138,7 +138,7 @@ func get_beacon_committee*( # missing cases here. if epoch notin cache.shuffled_active_validator_indices: cache.shuffled_active_validator_indices[epoch] = - get_shuffledactive_validator_indices(state, epoch) + get_shuffled_active_validator_indices(state, epoch) # Constant throughout an epoch if epoch notin cache.committee_count_cache: @@ -164,7 +164,7 @@ func get_empty_per_epoch_cache*(): StateCache = # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_proposer_index func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex], - seed: Eth2Digest, stateCache: var StateCache): Option[ValidatorIndex] = + seed: Eth2Digest): Option[ValidatorIndex] = # Return from ``indices`` a random index sampled by effective balance. const MAX_RANDOM_BYTE = 255 @@ -194,7 +194,7 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex], i += 1 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index -func get_beacon_proposer_index*(state: BeaconState, stateCache: var StateCache, slot: Slot): +func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot): Option[ValidatorIndex] = # Return the beacon proposer index at the current slot. let epoch = get_current_epoch(state) @@ -205,11 +205,19 @@ func get_beacon_proposer_index*(state: BeaconState, stateCache: var StateCache, # TODO fixme; should only be run once per slot and cached # There's exactly one beacon proposer per slot. - let - seed = eth2hash(buffer) - indices = get_active_validator_indices(state, epoch) + if epoch notin cache.shuffled_active_validator_indices: + cache.shuffled_active_validator_indices[epoch] = + get_shuffled_active_validator_indices(state, epoch) - compute_proposer_index(state, indices, seed, stateCache) + try: + let + seed = eth2hash(buffer) + indices = + sorted(cache.shuffled_active_validator_indices[epoch], system.cmp) + + compute_proposer_index(state, indices, seed) + except KeyError: + raiseAssert("Cached entries are added before use") # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_beacon_proposer_index func get_beacon_proposer_index*(state: BeaconState, stateCache: var StateCache): diff --git a/nbench/scenarios.nim b/nbench/scenarios.nim index 14d55f1e9..9d83cbb87 100644 --- a/nbench/scenarios.nim +++ b/nbench/scenarios.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2020 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -7,11 +7,11 @@ import # Standard library - os, + os, tables, # Status libraries confutils/defs, serialization, # Beacon-chain - ../beacon_chain/spec/[datatypes, crypto, beaconstate, validator, state_transition_block, state_transition_epoch], + ../beacon_chain/spec/[datatypes, crypto, helpers, beaconstate, validator, state_transition_block, state_transition_epoch], ../beacon_chain/[ssz, state_transition, extras] # Nimbus Bench - Scenario configuration @@ -185,6 +185,9 @@ template processEpochScenarioImpl( when needCache: var cache = get_empty_per_epoch_cache() + let epoch = state.data.slot.compute_epoch_at_slot + cache.shuffled_active_validator_indices[epoch] = + get_shuffled_active_validator_indices(state.data, epoch) # Epoch transitions can't fail (TODO is this true?) when needCache: @@ -251,11 +254,11 @@ genProcessEpochScenario(runProcessJustificationFinalization, genProcessEpochScenario(runProcessRegistryUpdates, process_registry_updates, - needCache = false) + needCache = true) genProcessEpochScenario(runProcessSlashings, process_slashings, - needCache = false) + needCache = true) genProcessEpochScenario(runProcessFinalUpdates, process_final_updates, diff --git a/tests/official/test_fixture_state_transition_epoch.nim b/tests/official/test_fixture_state_transition_epoch.nim index 6f9cf1096..4bb94652e 100644 --- a/tests/official/test_fixture_state_transition_epoch.nim +++ b/tests/official/test_fixture_state_transition_epoch.nim @@ -66,13 +66,13 @@ runSuite(JustificationFinalizationDir, "Justification & Finalization", process_ # --------------------------------------------------------------- const RegistryUpdatesDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"registry_updates"/"pyspec_tests" -runSuite(RegistryUpdatesDir, "Registry updates", process_registry_updates, useCache = false) +runSuite(RegistryUpdatesDir, "Registry updates", process_registry_updates, useCache = true) # Slashings # --------------------------------------------------------------- const SlashingsDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"slashings"/"pyspec_tests" -runSuite(SlashingsDir, "Slashings", process_slashings, useCache = false) +runSuite(SlashingsDir, "Slashings", process_slashings, useCache = true) # Final updates # --------------------------------------------------------------- diff --git a/tests/spec_epoch_processing/justification_finalization_helpers.nim b/tests/spec_epoch_processing/justification_finalization_helpers.nim index b0a7e04ae..5bce78436 100644 --- a/tests/spec_epoch_processing/justification_finalization_helpers.nim +++ b/tests/spec_epoch_processing/justification_finalization_helpers.nim @@ -7,7 +7,7 @@ import # Standard library - strformat, + strformat, tables, # Specs ../../beacon_chain/spec/[datatypes, state_transition_epoch, validator, helpers], # Test helpers @@ -34,7 +34,10 @@ proc addMockAttestations*( raise newException(ValueError, &"Cannot include attestations from epoch {state.get_current_epoch()} in epoch {epoch}") # TODO: Working with an unsigned Gwei balance is a recipe for underflows to happen - var remaining_balance = state.get_total_active_balance().int64 * 2 div 3 + var cache = get_empty_per_epoch_cache() + cache.shuffled_active_validator_indices[epoch] = + get_shuffled_active_validator_indices(state, epoch) + var remaining_balance = state.get_total_active_balance(cache).int64 * 2 div 3 let start_slot = compute_start_slot_at_epoch(epoch)