From c06ffc7804876def642e56f85b5bf0fc0ac426f7 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 28 May 2021 15:25:58 +0000 Subject: [PATCH] proposed structure for altair (#2323) * proposed structure for hf1 * refactor datatypes.nim into datatypes/{base, phase0, hf1}.nim * hf1 is Altair * some syncing with alpha 2 * adjust epoch processing to disambiguate access to RewardFlags * relocate StateData to stay consistent with meaning phase 0 StateData * passes v1.1.0 alpha 5 SSZ consensus object tests * Altair block header test fixtures work * fix slash_validator() so that Altair attester slashings, proposer slashings, and voluntary exit textures work * deposit operation Altair test fixtures work * slot sanity and all but a couple epoch transition tests switched to Altair * attestation Altair test fixtures work * Altair block sanity test fixtures work * add working altair sync committee tests * improve workarounds for sum-types-across-modules Nim bug; incorporate SignedBeaconBlock root reconstuction to SSZ byte reader --- FixtureAll-mainnet.md | 41 +- FixtureSSZConsensus-mainnet.md | 13 +- beacon_chain/spec/beaconstate.nim | 330 +++++++++++-- beacon_chain/spec/datatypes/altair.nim | 393 ++++++++++++++-- beacon_chain/spec/datatypes/base.nim | 288 +----------- beacon_chain/spec/datatypes/phase0.nim | 294 ++++++++++++ beacon_chain/spec/helpers.nim | 25 +- beacon_chain/spec/presets/altair/mainnet.nim | 18 +- beacon_chain/spec/signatures.nim | 4 +- beacon_chain/spec/signatures_batch.nim | 3 + beacon_chain/spec/state_transition.nim | 51 ++- beacon_chain/spec/state_transition_block.nim | 111 ++++- beacon_chain/spec/state_transition_epoch.nim | 433 +++++++++++++++--- beacon_chain/spec/validator.nim | 34 +- beacon_chain/ssz/bytes_reader.nim | 9 +- tests/official/all_fixtures_require_ssz.nim | 3 +- .../test_fixture_const_sanity_check.nim | 1 + .../test_fixture_operations_attestations.nim | 5 +- ..._fixture_operations_attester_slashings.nim | 5 +- .../test_fixture_operations_block_header.nim | 5 +- .../test_fixture_operations_deposits.nim | 5 +- ..._fixture_operations_proposer_slashings.nim | 8 +- ...test_fixture_operations_sync_committee.nim | 70 +++ ...test_fixture_operations_voluntary_exit.nim | 5 +- tests/official/test_fixture_sanity_blocks.nim | 8 +- tests/official/test_fixture_sanity_slots.nim | 6 +- .../test_fixture_ssz_consensus_objects.nim | 23 +- .../test_fixture_state_transition_epoch.nim | 33 +- .../justification_finalization_helpers.nim | 4 +- tests/testblockutil.nim | 1 + 30 files changed, 1682 insertions(+), 547 deletions(-) create mode 100644 tests/official/test_fixture_operations_sync_committee.nim diff --git a/FixtureAll-mainnet.md b/FixtureAll-mainnet.md index 67f165c68..eca16fd3e 100644 --- a/FixtureAll-mainnet.md +++ b/FixtureAll-mainnet.md @@ -66,7 +66,6 @@ FixtureAll-mainnet + [Invalid] Official - Sanity - Blocks - invalid_state_root [Preset: mainnet] OK + [Invalid] Official - Sanity - Blocks - parent_from_same_slot [Preset: mainnet] OK + [Invalid] Official - Sanity - Blocks - prev_slot_block_transition [Preset: mainnet] OK -+ [Invalid] Official - Sanity - Blocks - proposal_for_genesis_slot [Preset: mainnet] OK + [Invalid] Official - Sanity - Blocks - same_slot_block_transition [Preset: mainnet] OK + [Invalid] Official - Sanity - Blocks - slash_and_exit_same_index [Preset: mainnet] OK + [Invalid] Official - Sanity - Blocks - zero_block_sig [Preset: mainnet] OK @@ -108,6 +107,9 @@ FixtureAll-mainnet + [Invalid] invalid_sig_1_and_2_swap OK + [Invalid] invalid_sig_2 OK + [Invalid] invalid_signature OK ++ [Invalid] invalid_signature_extra_participant OK ++ [Invalid] invalid_signature_missing_participant OK ++ [Invalid] invalid_signature_past_block OK + [Invalid] invalid_slot_block_header OK + [Invalid] mismatched_target_and_slot OK + [Invalid] new_source_epoch OK @@ -151,10 +153,6 @@ FixtureAll-mainnet + [Valid] Official - Sanity - Blocks - deposit_top_up [Preset: mainnet] OK + [Valid] Official - Sanity - Blocks - empty_block_transition [Preset: mainnet] OK + [Valid] Official - Sanity - Blocks - empty_epoch_transition [Preset: mainnet] OK -+ [Valid] Official - Sanity - Blocks - full_random_operations_0 [Preset: mainnet] OK -+ [Valid] Official - Sanity - Blocks - full_random_operations_1 [Preset: mainnet] OK -+ [Valid] Official - Sanity - Blocks - full_random_operations_2 [Preset: mainnet] OK -+ [Valid] Official - Sanity - Blocks - full_random_operations_3 [Preset: mainnet] OK + [Valid] Official - Sanity - Blocks - high_proposer_index [Preset: mainnet] OK + [Valid] Official - Sanity - Blocks - historical_batch [Preset: mainnet] OK + [Valid] Official - Sanity - Blocks - multiple_attester_slashings_no_overlap [Preset: mai OK @@ -193,8 +191,11 @@ FixtureAll-mainnet + [Valid] success_slashed_and_proposer_index_the_same OK + [Valid] success_surround OK + [Valid] success_with_effective_balance_disparity OK ++ [Valid] sync_committee_rewards_duplicate_committee OK ++ [Valid] sync_committee_rewards_empty_participants OK ++ [Valid] sync_committee_rewards_not_full_participants OK ``` -OK: 191/191 Fail: 0/191 Skip: 0/191 +OK: 192/192 Fail: 0/192 Skip: 0/192 ## Official - Epoch Processing - Effective balance updates [Preset: mainnet] ```diff + Effective balance updates - effective_balance_hysteresis [Preset: mainnet] OK @@ -211,24 +212,6 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + Historical roots update - historical_root_accumulator [Preset: mainnet] OK ``` OK: 1/1 Fail: 0/1 Skip: 0/1 -## Official - Epoch Processing - Justification & Finalization [Preset: mainnet] -```diff -+ Justification & Finalization - 123_ok_support [Preset: mainnet] OK -+ Justification & Finalization - 123_poor_support [Preset: mainnet] OK -+ Justification & Finalization - 12_ok_support [Preset: mainnet] OK -+ Justification & Finalization - 12_ok_support_messed_target [Preset: mainnet] OK -+ Justification & Finalization - 12_poor_support [Preset: mainnet] OK -+ Justification & Finalization - 234_ok_support [Preset: mainnet] OK -+ Justification & Finalization - 234_poor_support [Preset: mainnet] OK -+ Justification & Finalization - 23_ok_support [Preset: mainnet] OK -+ Justification & Finalization - 23_poor_support [Preset: mainnet] OK -``` -OK: 9/9 Fail: 0/9 Skip: 0/9 -## Official - Epoch Processing - Participation record updates [Preset: mainnet] -```diff -+ Participation record updates - updated_participation_record [Preset: mainnet] OK -``` -OK: 1/1 Fail: 0/1 Skip: 0/1 ## Official - Epoch Processing - RANDAO mixes reset [Preset: mainnet] ```diff + RANDAO mixes reset - updated_randao_mixes [Preset: mainnet] OK @@ -246,14 +229,6 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 + Registry updates - ejection_past_churn_limit [Preset: mainnet] OK ``` OK: 8/8 Fail: 0/8 Skip: 0/8 -## Official - Epoch Processing - Slashings [Preset: mainnet] -```diff -+ Slashings - low_penalty [Preset: mainnet] OK -+ Slashings - max_penalties [Preset: mainnet] OK -+ Slashings - minimal_penalty [Preset: mainnet] OK -+ Slashings - scaled_penalties [Preset: mainnet] OK -``` -OK: 4/4 Fail: 0/4 Skip: 0/4 ## Official - Epoch Processing - Slashings reset [Preset: mainnet] ```diff + Slashings reset - flush_slashings [Preset: mainnet] OK @@ -261,4 +236,4 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 OK: 1/1 Fail: 0/1 Skip: 0/1 ---TOTAL--- -OK: 219/219 Fail: 0/219 Skip: 0/219 +OK: 206/206 Fail: 0/206 Skip: 0/206 diff --git a/FixtureSSZConsensus-mainnet.md b/FixtureSSZConsensus-mainnet.md index eeb59c7cd..e7c7906d3 100644 --- a/FixtureSSZConsensus-mainnet.md +++ b/FixtureSSZConsensus-mainnet.md @@ -11,6 +11,7 @@ FixtureSSZConsensus-mainnet + Testing BeaconBlockHeader OK + Testing BeaconState OK + Testing Checkpoint OK ++ Testing ContributionAndProof OK + Testing Deposit OK + Testing DepositData OK + Testing DepositMessage OK @@ -20,17 +21,25 @@ FixtureSSZConsensus-mainnet + Testing ForkData OK + Testing HistoricalBatch OK + Testing IndexedAttestation OK ++ Testing LightClientSnapshot OK ++ Testing LightClientUpdate OK + Testing PendingAttestation OK + Testing ProposerSlashing OK + Testing SignedAggregateAndProof OK + Testing SignedBeaconBlock OK + Testing SignedBeaconBlockHeader OK ++ Testing SignedContributionAndProof OK + Testing SignedVoluntaryExit OK + Testing SigningData OK ++ Testing SyncAggregate OK ++ Testing SyncAggregatorSelectionData OK ++ Testing SyncCommittee OK ++ Testing SyncCommitteeContribution OK ++ Testing SyncCommitteeSignature OK + Testing Validator OK + Testing VoluntaryExit OK ``` -OK: 27/27 Fail: 0/27 Skip: 0/27 +OK: 36/36 Fail: 0/36 Skip: 0/36 ---TOTAL--- -OK: 27/27 Fail: 0/27 Skip: 0/27 +OK: 36/36 Fail: 0/36 Skip: 0/36 diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 52410eb9c..cd5ba0ad0 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -13,9 +13,11 @@ import json_serialization/std/sets, chronicles, ../extras, ../ssz/merkleization, - ./crypto, ./datatypes, ./digest, ./helpers, ./signatures, ./validator, + ./crypto, ./datatypes/[phase0, altair], ./digest, ./helpers, ./signatures, ./validator, ../../nbench/bench_lab +import blscurve # TODO bad + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_valid_merkle_branch func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], depth: int, index: uint64, @@ -41,7 +43,7 @@ func increase_balance*(balance: var Gwei, delta: Gwei) = balance += delta func increase_balance*( - state: var BeaconState, index: ValidatorIndex, delta: Gwei) = + state: var SomeBeaconState, index: ValidatorIndex, delta: Gwei) = ## Increase the validator balance at index ``index`` by ``delta``. if delta != 0: # avoid dirtying the balance cache if not needed increase_balance(state.balances[index], delta) @@ -55,13 +57,14 @@ func decrease_balance*(balance: var Gwei, delta: Gwei) = balance - delta func decrease_balance*( - state: var BeaconState, index: ValidatorIndex, delta: Gwei) = + state: var SomeBeaconState, index: ValidatorIndex, delta: Gwei) = ## Decrease the validator balance at index ``index`` by ``delta``, with ## underflow protection. if delta != 0: # avoid dirtying the balance cache if not needed decrease_balance(state.balances[index], delta) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#deposits +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#modified-process_deposit func get_validator_from_deposit(deposit: DepositData): Validator = let @@ -80,7 +83,7 @@ func get_validator_from_deposit(deposit: DepositData): ) proc process_deposit*(preset: RuntimePreset, - state: var BeaconState, + state: var SomeBeaconState, deposit: Deposit, flags: UpdateFlags = {}): Result[void, cstring] {.nbench.}= ## Process an Eth1 deposit, registering a validator or increasing its balance. @@ -128,6 +131,14 @@ proc process_deposit*(preset: RuntimePreset, static: doAssert state.balances.maxLen == state.validators.maxLen raiseAssert "adding validator succeeded, so should balances" + when state is altair.BeaconState: + if not state.previous_epoch_participation.add(ParticipationFlags(0)): + return err("process_deposit: too many validators (previous_epoch_participation)") + if not state.current_epoch_participation.add(ParticipationFlags(0)): + return err("process_deposit: too many validators (current_epoch_participation)") + if not state.inactivity_scores.add(0'u64): + return err("process_deposit: too many validators (inactivity_scores)") + doAssert state.validators.len == state.balances.len else: # Deposits may come with invalid signatures - in that case, they are not @@ -145,7 +156,8 @@ func compute_activation_exit_epoch(epoch: Epoch): Epoch = epoch + 1 + MAX_SEED_LOOKAHEAD # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_validator_churn_limit -func get_validator_churn_limit(state: BeaconState, cache: var StateCache): uint64 = +func get_validator_churn_limit(state: SomeBeaconState, cache: var StateCache): + uint64 = ## Return the validator churn limit for the current epoch. max( MIN_PER_EPOCH_CHURN_LIMIT, @@ -153,7 +165,7 @@ func get_validator_churn_limit(state: BeaconState, cache: var StateCache): uint6 state, state.get_current_epoch(), cache) div CHURN_LIMIT_QUOTIENT) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#initiate_validator_exit -func initiate_validator_exit*(state: var BeaconState, +func initiate_validator_exit*(state: var SomeBeaconState, index: ValidatorIndex, cache: var StateCache) = ## Initiate the exit of the validator with index ``index``. @@ -193,7 +205,8 @@ func initiate_validator_exit*(state: var BeaconState, validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#slash_validator -proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#modified-slash_validator +proc slash_validator*(state: var SomeBeaconState, slashed_index: ValidatorIndex, cache: var StateCache) = ## Slash the validator with index ``index``. let epoch = get_current_epoch(state) @@ -214,8 +227,17 @@ proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, max(validator.withdrawable_epoch, epoch + EPOCHS_PER_SLASHINGS_VECTOR) state.slashings[int(epoch mod EPOCHS_PER_SLASHINGS_VECTOR)] += validator.effective_balance - decrease_balance(state, slashed_index, - validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT) + + # TODO Consider whether this is better than splitting the functions apart; in + # each case, tradeoffs. Here, it's just changing a couple of constants. + when state is phase0.BeaconState: + decrease_balance(state, slashed_index, + validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT) + elif state is altair.BeaconState: + decrease_balance(state, slashed_index, + validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR) + else: + raiseAssert "invalid BeaconState type" # The rest doesn't make sense without there being any proposer index, so skip let proposer_index = get_beacon_proposer_index(state, cache) @@ -227,14 +249,21 @@ proc slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex, let # Spec has whistleblower_index as optional param, but it's never used. whistleblower_index = proposer_index.get - whistleblowing_reward = + whistleblower_reward = (validator.effective_balance div WHISTLEBLOWER_REWARD_QUOTIENT).Gwei - proposer_reward = whistleblowing_reward div PROPOSER_REWARD_QUOTIENT + proposer_reward = + when state is phase0.BeaconState: + whistleblower_reward div PROPOSER_REWARD_QUOTIENT + elif state is altair.BeaconState: + whistleblower_reward * PROPOSER_WEIGHT div WEIGHT_DENOMINATOR + else: + raiseAssert "invalid BeaconState type" + increase_balance(state, proposer_index.get, proposer_reward) # TODO: evaluate if spec bug / underflow can be triggered - doAssert(whistleblowing_reward >= proposer_reward, "Spec bug: underflow in slash_validator") + doAssert(whistleblower_reward >= proposer_reward, "Spec bug: underflow in slash_validator") increase_balance( - state, whistleblower_index, whistleblowing_reward - proposer_reward) + state, whistleblower_index, whistleblower_reward - proposer_reward) func genesis_time_from_eth1_timestamp*(preset: RuntimePreset, eth1_timestamp: uint64): uint64 = eth1_timestamp + preset.GENESIS_DELAY @@ -245,7 +274,7 @@ proc initialize_beacon_state_from_eth1*( eth1_block_hash: Eth2Digest, eth1_timestamp: uint64, deposits: openArray[DepositData], - flags: UpdateFlags = {}): BeaconStateRef {.nbench.} = + flags: UpdateFlags = {}): phase0.BeaconStateRef {.nbench.} = ## Get the genesis ``BeaconState``. ## ## Before the beacon chain starts, validators will register in the Eth1 chain @@ -262,7 +291,7 @@ proc initialize_beacon_state_from_eth1*( # at that point :) doAssert deposits.lenu64 >= SLOTS_PER_EPOCH - var state = BeaconStateRef( + var state = phase0.BeaconStateRef( fork: Fork( previous_version: preset.GENESIS_FORK_VERSION, current_version: preset.GENESIS_FORK_VERSION, @@ -272,7 +301,7 @@ proc initialize_beacon_state_from_eth1*( Eth1Data(block_hash: eth1_block_hash, deposit_count: uint64(len(deposits))), latest_block_header: BeaconBlockHeader( - body_root: hash_tree_root(BeaconBlockBody()))) + body_root: hash_tree_root(default(phase0.BeaconBlockBody)))) # Seed RANDAO with Eth1 entropy state.randao_mixes.fill(eth1_block_hash) @@ -334,23 +363,26 @@ proc initialize_hashed_beacon_state_from_eth1*( eth1_block_hash: Eth2Digest, eth1_timestamp: uint64, deposits: openArray[DepositData], - flags: UpdateFlags = {}): HashedBeaconState = + flags: UpdateFlags = {}): phase0.HashedBeaconState = let genesisState = initialize_beacon_state_from_eth1( preset, eth1_block_hash, eth1_timestamp, deposits, flags) - HashedBeaconState(data: genesisState[], root: hash_tree_root(genesisState[])) + phase0.HashedBeaconState( + data: genesisState[], root: hash_tree_root(genesisState[])) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#genesis-block -func get_initial_beacon_block*(state: BeaconState): TrustedSignedBeaconBlock = +func get_initial_beacon_block*(state: phase0.BeaconState): + phase0.TrustedSignedBeaconBlock = # The genesis block is implicitly trusted - let message = TrustedBeaconBlock( + let message = phase0.TrustedBeaconBlock( slot: state.slot, state_root: hash_tree_root(state),) # parent_root, randao_reveal, eth1_data, signature, and body automatically # initialized to default values. - TrustedSignedBeaconBlock(message: message, root: hash_tree_root(message)) + phase0.TrustedSignedBeaconBlock( + message: message, root: hash_tree_root(message)) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_block_root_at_slot -func get_block_root_at_slot*(state: BeaconState, +func get_block_root_at_slot*(state: SomeBeaconState, slot: Slot): Eth2Digest = ## Return the block root at a recent ``slot``. @@ -363,12 +395,12 @@ func get_block_root_at_slot*(state: BeaconState, state.block_roots[slot mod SLOTS_PER_HISTORICAL_ROOT] # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_block_root -func get_block_root*(state: BeaconState, epoch: Epoch): Eth2Digest = +func get_block_root*(state: SomeBeaconState, epoch: Epoch): Eth2Digest = ## Return the block root at the start of a recent ``epoch``. get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_total_balance -func get_total_balance*(state: BeaconState, validators: auto): Gwei = +func get_total_balance*(state: SomeBeaconState, validators: auto): Gwei = ## Return the combined effective balance of the ``indices``. ## ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. ## Math safe up to ~10B ETH, afterwhich this overflows uint64. @@ -383,7 +415,7 @@ func is_eligible_for_activation_queue(validator: Validator): bool = validator.effective_balance == MAX_EFFECTIVE_BALANCE # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_eligible_for_activation -func is_eligible_for_activation(state: BeaconState, validator: Validator): +func is_eligible_for_activation(state: SomeBeaconState, validator: Validator): bool = ## Check if ``validator`` is eligible for activation. @@ -393,7 +425,7 @@ func is_eligible_for_activation(state: BeaconState, validator: Validator): validator.activation_epoch == FAR_FUTURE_EPOCH # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#registry-updates -proc process_registry_updates*(state: var BeaconState, +proc process_registry_updates*(state: var SomeBeaconState, cache: var StateCache) {.nbench.} = ## Process activation eligibility and ejections @@ -444,7 +476,7 @@ proc process_registry_updates*(state: var BeaconState, # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#is_valid_indexed_attestation proc is_valid_indexed_attestation*( - state: BeaconState, indexed_attestation: SomeIndexedAttestation, + state: SomeBeaconState, indexed_attestation: SomeIndexedAttestation, flags: UpdateFlags): Result[void, cstring] = ## Check if ``indexed_attestation`` is not empty, has sorted and unique ## indices and has a valid aggregate signature. @@ -481,7 +513,7 @@ proc is_valid_indexed_attestation*( ok() # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_attesting_indices -iterator get_attesting_indices*(state: BeaconState, +iterator get_attesting_indices*(state: SomeBeaconState, data: AttestationData, bits: CommitteeValidatorsBits, cache: var StateCache): ValidatorIndex = @@ -498,7 +530,7 @@ iterator get_attesting_indices*(state: BeaconState, inc i proc is_valid_indexed_attestation*( - state: BeaconState, attestation: SomeAttestation, flags: UpdateFlags, + state: SomeBeaconState, attestation: SomeAttestation, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] = # This is a variation on `is_valid_indexed_attestation` that works directly # with an attestation instead of first constructing an `IndexedAttestation` @@ -567,9 +599,71 @@ func check_attestation_index( ok() +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices +func get_attestation_participation_flag_indices(state: altair.BeaconState, + data: AttestationData, + inclusion_delay: uint64): seq[int] = + ## Return the flag indices that are satisfied by an attestation. + let justified_checkpoint = + if data.target.epoch == get_current_epoch(state): + state.current_justified_checkpoint + else: + state.previous_justified_checkpoint + + # Matching roots + let + is_matching_source = data.source == justified_checkpoint + is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch) + is_matching_head = is_matching_target and data.beacon_block_root == get_block_root_at_slot(state, data.slot) + + # TODO probably this needs to be robustly failable + doAssert is_matching_source + + var participation_flag_indices: seq[int] + if is_matching_source and inclusion_delay <= integer_squareroot(SLOTS_PER_EPOCH): + participation_flag_indices.add(TIMELY_SOURCE_FLAG_INDEX) + if is_matching_target and inclusion_delay <= SLOTS_PER_EPOCH: + participation_flag_indices.add(TIMELY_TARGET_FLAG_INDEX) + if is_matching_head and inclusion_delay == MIN_ATTESTATION_INCLUSION_DELAY: + participation_flag_indices.add(TIMELY_HEAD_FLAG_INDEX) + + participation_flag_indices + +# TODO these aren't great here +# TODO these duplicate some stuff in state_transition_epoch which uses TotalBalances +# better to centralize around that if feasible + +# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_total_active_balance +func get_total_active_balance*(state: SomeBeaconState, cache: var StateCache): Gwei = + ## Return the combined effective balance of the active validators. + # Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei + # minimum to avoid divisions by zero. + + let epoch = state.get_current_epoch() + + get_total_balance( + state, cache.get_shuffled_active_validator_indices(state, epoch)) + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward_per_increment +func get_base_reward_per_increment*(state: altair.BeaconState, cache: var StateCache): Gwei = + EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR div + integer_squareroot(get_total_active_balance(state, cache)) + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward +func get_base_reward(state: altair.BeaconState, index: ValidatorIndex, cache: var StateCache): Gwei = + ## Return the base reward for the validator defined by ``index`` with respect to the current ``state``. + + # Note: An optimally performing validator can earn one base reward per + # epoch over a long time horizon. This takes into account both per-epoch + # (e.g. attestation) and intermittent duties (e.g. block proposal and sync + # committees). + let increments = + state.validators[index].effective_balance div EFFECTIVE_BALANCE_INCREMENT + increments * get_base_reward_per_increment(state, cache) + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#attestations proc check_attestation*( - state: BeaconState, attestation: SomeAttestation, flags: UpdateFlags, + state: SomeBeaconState, attestation: SomeAttestation, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] = ## Check that an attestation follows the rules of being included in the state ## at the current slot. When acting as a proposer, the same rules need to @@ -603,7 +697,7 @@ proc check_attestation*( ok() proc process_attestation*( - state: var BeaconState, attestation: SomeAttestation, flags: UpdateFlags, + state: var SomeBeaconState, attestation: SomeAttestation, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = # In the spec, attestation validation is mixed with state mutation, so here # we've split it into two functions so that the validation logic can be @@ -615,6 +709,11 @@ proc process_attestation*( ? check_attestation(state, attestation, flags, cache) + # TODO this should be split between two functions, but causes type errors + # in state_transition_block.process_operations() + # TODO investigate and, if real, file Nim bug + + # For phase0 template addPendingAttestation(attestations: typed) = # The genericSeqAssign generated by the compiler to copy the attestation # data sadly is a processing hotspot - the business with the addDefault @@ -627,9 +726,172 @@ proc process_attestation*( pa[].inclusion_delay = state.slot - attestation.data.slot pa[].proposer_index = proposer_index.get().uint64 - if attestation.data.target.epoch == get_current_epoch(state): - addPendingAttestation(state.current_epoch_attestations) + # For Altair + template updateParticipationFlags(epoch_participation: untyped) = + var proposer_reward_numerator = 0'u64 + + # Participation flag indices + let participation_flag_indices = get_attestation_participation_flag_indices(state, attestation.data, state.slot - attestation.data.slot) + + for index in get_attesting_indices(state, attestation.data, attestation.aggregation_bits, cache): + for flag_index, weight in PARTICIPATION_FLAG_WEIGHTS: + if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): + epoch_participation[index] = add_flag(epoch_participation[index], flag_index) + proposer_reward_numerator += get_base_reward(state, index, cache) * weight.uint64 # these are all valid, #TODO statically verify or do it type-safely + + # Reward proposer + let + # TODO use correct type at source + proposer_reward_denominator = (WEIGHT_DENOMINATOR.uint64 - PROPOSER_WEIGHT.uint64) * WEIGHT_DENOMINATOR.uint64 div PROPOSER_WEIGHT.uint64 + proposer_reward = Gwei(proposer_reward_numerator div proposer_reward_denominator) + increase_balance(state, proposer_index.get, proposer_reward) + + when state is phase0.BeaconState: + if attestation.data.target.epoch == get_current_epoch(state): + addPendingAttestation(state.current_epoch_attestations) + else: + addPendingAttestation(state.previous_epoch_attestations) + elif state is altair.BeaconState: + if attestation.data.target.epoch == get_current_epoch(state): + updateParticipationFlags(state.current_epoch_participation) + else: + updateParticipationFlags(state.previous_epoch_participation) else: - addPendingAttestation(state.previous_epoch_attestations) + static: doAssert false ok() + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.3/specs/altair/fork.md#upgrading-the-state +func upgrade_to_altair(pre: phase0.BeaconState): altair.BeaconState = + let epoch = get_current_epoch(pre) + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.3/specs/altair/fork.md#configuration + const ALTAIR_FORK_VERSION = Version [byte 1, 0, 0, 0] + + var empty_participation = + HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]() + for _ in 0 ..< len(pre.validators): + doAssert empty_participation.add 0.ParticipationFlags + + altair.BeaconState( + genesis_time: pre.genesis_time, + genesis_validators_root: pre.genesis_validators_root, + slot: pre.slot, + fork: Fork( + previous_version: pre.fork.current_version, + current_version: ALTAIR_FORK_VERSION, + epoch: epoch + ), + + # History + latest_block_header: pre.latest_block_header, + block_roots: pre.block_roots, + state_roots: pre.state_roots, + historical_roots: pre.historical_roots, + + # Eth1 + eth1_data: pre.eth1_data, + eth1_data_votes: pre.eth1_data_votes, + eth1_deposit_index: pre.eth1_deposit_index, + + # Registry + validators: pre.validators, + balances: pre.balances, + + # Randomness + randao_mixes: pre.randao_mixes, + + # Slashings + slashings: pre.slashings, + + # Attestations + previous_epoch_participation: empty_participation, + current_epoch_participation: empty_participation, + + # Finality + justification_bits: pre.justification_bits, + previous_justified_checkpoint: pre.previous_justified_checkpoint, + current_justified_checkpoint: pre.current_justified_checkpoint, + finalized_checkpoint: pre.finalized_checkpoint + ) + + #TODO + # Fill in sync committees + #post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) + #post.next_sync_committee = get_sync_committee(post, get_current_epoch(post) + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + #post + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_next_sync_committee_indices +func get_next_sync_committee_indices(state: altair.BeaconState): seq[ValidatorIndex] = + ## Return the sequence of sync committee indices (which may include + ## duplicate indices) for the next sync committee, given a ``state`` at a + ## sync committee period boundary. + + # Note: Committee can contain duplicate indices for small validator sets + # (< SYNC_COMMITTEE_SIZE + 128) + let epoch = Epoch(get_current_epoch(state) + 1) + + const MAX_RANDOM_BYTE = 255 + let + active_validator_indices = get_active_validator_indices(state, epoch) + active_validator_count = uint64(len(active_validator_indices)) + seed = get_seed(state, epoch, DOMAIN_SYNC_COMMITTEE) + var + i = 0'u64 + sync_committee_indices: seq[ValidatorIndex] + hash_buffer: array[40, byte] + hash_buffer[0..31] = seed.data + while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: + hash_buffer[32..39] = uint_to_bytes8(uint64(i div 32)) + let + shuffled_index = compute_shuffled_index(uint64(i mod active_validator_count), active_validator_count, seed) + candidate_index = active_validator_indices[shuffled_index] + random_byte = eth2digest(hash_buffer).data[i mod 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + sync_committee_indices.add candidate_index + i += 1'u64 + sync_committee_indices + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_next_sync_committee +proc get_next_sync_committee*(state: altair.BeaconState): SyncCommittee = + ## Return the *next* sync committee for a given ``state``. + # + # ``SyncCommittee`` contains an aggregate pubkey that enables + # resource-constrained clients to save some computation when verifying the + # sync committee's signature. + # + # ``SyncCommittee`` can also contain duplicate pubkeys, when + # ``get_next_sync_committee_indices`` returns duplicate indices. + # Implementations must take care when handling optimizations relating to + # aggregation and verification in the presence of duplicates. + # + # Note: This function should only be called at sync committee period + # boundaries by ``process_sync_committee_updates`` as + # ``get_next_sync_committee_indices`` is not stable within a given period. + let + indices = get_next_sync_committee_indices(state) + pubkeys = mapIt(indices, state.validators[it].pubkey) + + # see signatures_batch, TODO shouldn't be here + var + aggregate_pubkey: blscurve.PublicKey + attestersAgg: AggregatePublicKey + let ck = pubkeys[0].loadWithCache() + if ck.isNone: + attestersAgg.init(ck.get) + for i in 1 ..< pubkeys.len: + let cookedKey = pubkeys[i].loadWithCache() + if cookedKey.isNone(): + # TODO is this the correct failure handling? + continue + attestersAgg.aggregate(cookedKey.get) + aggregate_pubkey.finish(attestersAgg) + + var res = SyncCommittee(aggregate_pubkey: ValidatorPubKey(blob: aggregate_pubkey.exportRaw())) + doAssert indices.len == SYNC_COMMITTEE_SIZE # needs to fill vector exactly + doAssert pubkeys.len == SYNC_COMMITTEE_SIZE + for i in 0 ..< SYNC_COMMITTEE_SIZE: + # obviously ineffecient + res.pubkeys[i] = pubkeys[i] + res diff --git a/beacon_chain/spec/datatypes/altair.nim b/beacon_chain/spec/datatypes/altair.nim index 3a624e524..1089f19c8 100644 --- a/beacon_chain/spec/datatypes/altair.nim +++ b/beacon_chain/spec/datatypes/altair.nim @@ -24,29 +24,47 @@ {.push raises: [Defect].} +# std/[intsets, json, strutils, tables], +# stew/byteutils, + import + chronicles, std/macros, - stew/assign2, + stew/[assign2, bitops2], json_serialization/types as jsonTypes, ../../ssz/types as sszTypes, ../crypto, ../digest, ../presets -import ./base +import ./base, ./phase0 export base const - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#incentivization-weights - TIMELY_HEAD_WEIGHT* = 12 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#incentivization-weights TIMELY_SOURCE_WEIGHT* = 12 TIMELY_TARGET_WEIGHT* = 24 + TIMELY_HEAD_WEIGHT* = 12 SYNC_REWARD_WEIGHT* = 8 + PROPOSER_WEIGHT* = 8 WEIGHT_DENOMINATOR* = 64 - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#misc + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#misc TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE* = 4 - SYNC_COMMITTEE_SUBNET_COUNT* = 8 + SYNC_COMMITTEE_SUBNET_COUNT* = 4 + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/setup.py#L470 + FINALIZED_ROOT_INDEX* = 105'u16 + NEXT_SYNC_COMMITTEE_INDEX* = 55'u16 + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#participation-flag-indices + TIMELY_SOURCE_FLAG_INDEX* = 0 + TIMELY_TARGET_FLAG_INDEX* = 1 + TIMELY_HEAD_FLAG_INDEX* = 2 + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#inactivity-penalties + #INACTIVITY_SCORE_BIAS* = 4 + INACTIVITY_SCORE_RECOVERY_RATE* = 16 let - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#misc + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#misc # Cannot be computed at compile-time due to importc dependency G2_POINT_AT_INFINITY* = ValidatorSig.fromRaw([ 0xc0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -55,24 +73,26 @@ let 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + PARTICIPATION_FLAG_WEIGHTS* = + [TIMELY_SOURCE_WEIGHT, TIMELY_TARGET_WEIGHT, TIMELY_HEAD_WEIGHT] type - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#custom-types - ParticipationFlags* = distinct uint8 + ### New types - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#syncaggregate + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#custom-types + # TODO could be distinct + ParticipationFlags* = uint8 + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#syncaggregate SyncAggregate* = object sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE] sync_committee_signature*: ValidatorSig - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#synccommittee + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#synccommittee SyncCommittee* = object pubkeys*: HashArray[Limit SYNC_COMMITTEE_SIZE, ValidatorPubKey] - pubkey_aggregates*: - HashArray[ - Limit SYNC_COMMITTEE_SIZE div SYNC_PUBKEYS_PER_AGGREGATE, - ValidatorPubKey] + aggregate_pubkey*: ValidatorPubKey - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#synccommitteesignature + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#synccommitteesignature SyncCommitteeSignature* = object slot*: Slot ##\ ## Slot to which this contribution pertains @@ -86,7 +106,7 @@ type signature*: ValidatorSig ##\ ## Signature by the validator over the block root of `slot` - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#synccommitteecontribution + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#synccommitteecontribution SyncCommitteeContribution* = object slot*: Slot ##\ ## Slot to which this contribution pertains @@ -106,30 +126,345 @@ type signature*: ValidatorSig ##\ ## Signature by the validator(s) over the block root of `slot` - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#contributionandproof + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#contributionandproof ContributionAndProof* = object aggregator_index*: uint64 contribution*: SyncCommitteeContribution selection_proof*: ValidatorSig - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#signedcontributionandproof + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#signedcontributionandproof SignedContributionAndProof* = object message*: ContributionAndProof signature*: ValidatorSig - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/validator.md#synccommitteesigningdata - SyncCommitteeSigningData* = object + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/validator.md#syncaggregatorselectiondata + SyncAggregatorSelectionData* = object slot*: Slot subcommittee_index*: uint64 - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#participation-flag-indices - ParticipationFlag* = enum - TIMELY_HEAD_FLAG_INDEX = 0 - TIMELY_SOURCE_FLAG_INDEX = 1 - TIMELY_TARGET_FLAG_INDEX = 2 + ### Modified/overloaded + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/sync-protocol.md#lightclientsnapshot + LightClientSnapshot* = object + header*: BeaconBlockHeader ##\ + ## Beacon block header + + current_sync_committee*: SyncCommittee ##\ + ## Sync committees corresponding to the header + + next_sync_committee*: SyncCommittee + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/sync-protocol.md#lightclientupdate + LightClientUpdate* = object + header*: BeaconBlockHeader ##\ + ## Update beacon block header + + next_sync_committee*: SyncCommittee ##\ + ## Next sync committee corresponding to the header + + next_sync_committee_branch*: array[log2trunc(NEXT_SYNC_COMMITTEE_INDEX), Eth2Digest] ##\ + ## Finality proof for the update header + + finality_header*: BeaconBlockHeader + finality_branch*: array[log2trunc(FINALIZED_ROOT_INDEX), Eth2Digest] + + sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE] ##\ + ## Sync committee aggregate signature + + sync_committee_signature*: ValidatorSig + + fork_version*: Version ##\ + ## Fork version for the aggregate signature + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#beaconstate + BeaconState* = object + # Versioning + genesis_time*: uint64 + genesis_validators_root*: Eth2Digest + slot*: Slot + fork*: Fork + + # History + latest_block_header*: BeaconBlockHeader ##\ + ## `latest_block_header.state_root == ZERO_HASH` temporarily + + block_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] ##\ + ## Needed to process attestations, older to newer + + state_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] + historical_roots*: HashList[Eth2Digest, Limit HISTORICAL_ROOTS_LIMIT] + + # Eth1 + eth1_data*: Eth1Data + eth1_data_votes*: + HashList[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)] + eth1_deposit_index*: uint64 + + # Registry + validators*: HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT] + balances*: HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT] + + # Randomness + randao_mixes*: HashArray[Limit EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest] + + # Slashings + slashings*: HashArray[Limit EPOCHS_PER_SLASHINGS_VECTOR, uint64] ##\ + ## Per-epoch sums of slashed effective balances + + # Participation + previous_epoch_participation*: + HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation*: + HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + + # Finality + justification_bits*: uint8 ##\ + ## Bit set for every recent justified epoch + ## Model a Bitvector[4] as a one-byte uint, which should remain consistent + ## with ssz/hashing. + + previous_justified_checkpoint*: Checkpoint ##\ + ## Previous epoch snapshot + + current_justified_checkpoint*: Checkpoint + finalized_checkpoint*: Checkpoint + + # Inactivity + inactivity_scores*: HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT] # [New in Altair] + + # Light client sync committees + current_sync_committee*: SyncCommittee # [New in Altair] + next_sync_committee*: SyncCommittee # [New in Altair] + + # TODO Careful, not nil analysis is broken / incomplete and the semantics will + # likely change in future versions of the language: + # https://github.com/nim-lang/RFCs/issues/250 + BeaconStateRef* = ref BeaconState not nil + NilableBeaconStateRef* = ref BeaconState + + HashedBeaconState* = object + data*: BeaconState + root*: Eth2Digest # hash_tree_root(data) + + # HF1 implies knowledge of phase 0, and this saves creating some other + # module to merge such knowledge. Another approach is to have imported + # set of phase 0/HF1 symbols be independently combined by each module, + # when necessary, but that spreads such detailed abstraction knowledge + # more widely through codebase than strictly required. Do not export a + # phase 0 version of symbols; anywhere which specially handles it will + # have to do so itself. + SomeBeaconState* = BeaconState | phase0.BeaconState + SomeHashedBeaconState* = HashedBeaconState | phase0.HashedBeaconState # probably not useful long-term, + ## since process_slots will need to be StateData + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblock + BeaconBlock* = object + ## For each slot, a proposer is chosen from the validator pool to propose + ## a new block. Once the block as been proposed, it is transmitted to + ## validators that will have a chance to vote on it through attestations. + ## Each block collects attestations, or votes, on past blocks, thus a chain + ## is formed. + + slot*: Slot + proposer_index*: uint64 + + parent_root*: Eth2Digest ##\ + ## Root hash of the previous block + + state_root*: Eth2Digest ##\ + ## The state root, _after_ this block has been processed + + body*: BeaconBlockBody + + SigVerifiedBeaconBlock* = object + ## A BeaconBlock that contains verified signatures + ## but that has not been verified for state transition + slot*: Slot + proposer_index*: uint64 + + parent_root*: Eth2Digest ##\ + ## Root hash of the previous block + + state_root*: Eth2Digest ##\ + ## The state root, _after_ this block has been processed + + body*: SigVerifiedBeaconBlockBody + + TrustedBeaconBlock* = object + ## When we receive blocks from outside sources, they are untrusted and go + ## through several layers of validation. Blocks that have gone through + ## validations can be trusted to be well-formed, with a correct signature, + ## having a parent and applying cleanly to the state that their parent + ## left them with. + ## + ## When loading such blocks from the database, to rewind states for example, + ## it is expensive to redo the validations (in particular, the signature + ## checks), thus `TrustedBlock` uses a `TrustedSig` type to mark that these + ## checks can be skipped. + ## + ## TODO this could probably be solved with some type trickery, but there + ## too many bugs in nim around generics handling, and we've used up + ## the trickery budget in the serialization library already. Until + ## then, the type must be manually kept compatible with its untrusted + ## cousin. + slot*: Slot + proposer_index*: uint64 + parent_root*: Eth2Digest ##\ + state_root*: Eth2Digest ##\ + body*: TrustedBeaconBlockBody + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#beaconblockbody + BeaconBlockBody* = object + randao_reveal*: ValidatorSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[Attestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + # [New in Altair] + sync_aggregate*: SyncAggregate + + SigVerifiedBeaconBlockBody* = object + ## A BeaconBlock body with signatures verified + ## including: + ## - Randao reveal + ## - Attestations + ## - ProposerSlashing (SignedBeaconBlockHeader) + ## - AttesterSlashing (IndexedAttestation) + ## - SignedVoluntaryExits + ## + ## - ETH1Data (Deposits) can contain invalid BLS signatures + ## + ## The block state transition has NOT been verified + randao_reveal*: TrustedSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + # [New in Altair] + sync_aggregate*: SyncAggregate + + TrustedBeaconBlockBody* = object + ## A full verified block + randao_reveal*: TrustedSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + # [New in Altair] + sync_aggregate*: SyncAggregate + + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#signedbeaconblock + SignedBeaconBlock* = object + message*: BeaconBlock + signature*: ValidatorSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + SigVerifiedSignedBeaconBlock* = object + ## A SignedBeaconBlock with signatures verified + ## including: + ## - Block signature + ## - BeaconBlockBody + ## - Randao reveal + ## - Attestations + ## - ProposerSlashing (SignedBeaconBlockHeader) + ## - AttesterSlashing (IndexedAttestation) + ## - SignedVoluntaryExits + ## + ## - ETH1Data (Deposits) can contain invalid BLS signatures + ## + ## The block state transition has NOT been verified + message*: SigVerifiedBeaconBlock + signature*: TrustedSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + TrustedSignedBeaconBlock* = object + message*: TrustedBeaconBlock + signature*: TrustedSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + SomeSignedBeaconBlock* = SignedBeaconBlock | SigVerifiedSignedBeaconBlock | TrustedSignedBeaconBlock + SomeBeaconBlock* = BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock + SomeBeaconBlockBody* = BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody + + # TODO rename + + # TODO why does this fail? + #SomeSomeBeaconBlock* = SomeBeaconBlock | phase0.SomeBeaconBlock + SomeSomeBeaconBlock* = + BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock | + phase0.BeaconBlock | phase0.SigVerifiedBeaconBlock | phase0.TrustedBeaconBlock + + # TODO see above, re why does it fail + SomeSomeBeaconBlockBody* = + BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody | + phase0.BeaconBlockBody | phase0.SigVerifiedBeaconBlockBody | phase0.TrustedBeaconBlockBody + #SomeSomeBeaconBlockBody* = SomeBeaconBlockBody | phase0.SomeBeaconBlockBody + + SomeSomeSignedBeaconBlock* = SomeSignedBeaconBlock | phase0.SomeSignedBeaconBlock # TODO when https://github.com/nim-lang/Nim/issues/14440 lands in Status's Nim, # switch proc {.noSideEffect.} to func. -proc `or`*(x, y: ParticipationFlags) : ParticipationFlags {.borrow, noSideEffect.} -proc `and`*(x, y: ParticipationFlags) : ParticipationFlags {.borrow, noSideEffect.} -proc `==`*(x, y: ParticipationFlags) : bool {.borrow, noSideEffect.} +when false: + # TODO if ParticipationFlags is distinct + proc `or`*(x, y: ParticipationFlags) : ParticipationFlags {.borrow, noSideEffect.} + proc `and`*(x, y: ParticipationFlags) : ParticipationFlags {.borrow, noSideEffect.} + proc `==`*(x, y: ParticipationFlags) : bool {.borrow, noSideEffect.} + +chronicles.formatIt BeaconBlock: it.shortLog + +Json.useCustomSerialization(BeaconState.justification_bits): + read: + let s = reader.readValue(string) + + if s.len != 4: + raiseUnexpectedValue(reader, "A string with 4 characters expected") + + try: + s.parseHexInt.uint8 + except ValueError: + raiseUnexpectedValue(reader, "The `justification_bits` value must be a hex string") + + write: + writer.writeValue "0x" & value.toHex + +func shortLog*(v: SomeBeaconBlock): auto = + ( + slot: shortLog(v.slot), + proposer_index: v.proposer_index, + parent_root: shortLog(v.parent_root), + state_root: shortLog(v.state_root), + eth1data: v.body.eth1_data, + graffiti: $v.body.graffiti, + proposer_slashings_len: v.body.proposer_slashings.len(), + attester_slashings_len: v.body.attester_slashings.len(), + attestations_len: v.body.attestations.len(), + deposits_len: v.body.deposits.len(), + voluntary_exits_len: v.body.voluntary_exits.len(), + ) + +func shortLog*(v: SomeSignedBeaconBlock): auto = + ( + blck: shortLog(v.message), + signature: shortLog(v.signature) + ) diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index 162143f96..8dcb0ce8d 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -61,7 +61,7 @@ const # Not part of spec. Still useful, pending removing usage if appropriate. ZERO_HASH* = Eth2Digest() - MAX_GRAFFITI_SIZE = 32 + MAX_GRAFFITI_SIZE* = 32 FAR_FUTURE_SLOT* = (not 0'u64).Slot # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/p2p-interface.md#configuration @@ -264,129 +264,6 @@ type validator_index*: uint64 - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblock - BeaconBlock* = object - ## For each slot, a proposer is chosen from the validator pool to propose - ## a new block. Once the block as been proposed, it is transmitted to - ## validators that will have a chance to vote on it through attestations. - ## Each block collects attestations, or votes, on past blocks, thus a chain - ## is formed. - - slot*: Slot - proposer_index*: uint64 - - parent_root*: Eth2Digest ##\ - ## Root hash of the previous block - - state_root*: Eth2Digest ##\ - ## The state root, _after_ this block has been processed - - body*: BeaconBlockBody - - SigVerifiedBeaconBlock* = object - ## A BeaconBlock that contains verified signatures - ## but that has not been verified for state transition - slot*: Slot - proposer_index*: uint64 - - parent_root*: Eth2Digest ##\ - ## Root hash of the previous block - - state_root*: Eth2Digest ##\ - ## The state root, _after_ this block has been processed - - body*: SigVerifiedBeaconBlockBody - - TrustedBeaconBlock* = object - ## When we receive blocks from outside sources, they are untrusted and go - ## through several layers of validation. Blocks that have gone through - ## validations can be trusted to be well-formed, with a correct signature, - ## having a parent and applying cleanly to the state that their parent - ## left them with. - ## - ## When loading such blocks from the database, to rewind states for example, - ## it is expensive to redo the validations (in particular, the signature - ## checks), thus `TrustedBlock` uses a `TrustedSig` type to mark that these - ## checks can be skipped. - ## - ## TODO this could probably be solved with some type trickery, but there - ## too many bugs in nim around generics handling, and we've used up - ## the trickery budget in the serialization library already. Until - ## then, the type must be manually kept compatible with its untrusted - ## cousin. - slot*: Slot - proposer_index*: uint64 - parent_root*: Eth2Digest ##\ - state_root*: Eth2Digest ##\ - body*: TrustedBeaconBlockBody - - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblockheader - BeaconBlockHeader* = object - slot*: Slot - proposer_index*: uint64 - parent_root*: Eth2Digest - state_root*: Eth2Digest - body_root*: Eth2Digest - - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signingdata - SigningData* = object - object_root*: Eth2Digest - domain*: Eth2Domain - - GraffitiBytes* = distinct array[MAX_GRAFFITI_SIZE, byte] - - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblockbody - BeaconBlockBody* = object - randao_reveal*: ValidatorSig - eth1_data*: Eth1Data - graffiti*: GraffitiBytes - - # Operations - proposer_slashings*: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] - attester_slashings*: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] - attestations*: List[Attestation, Limit MAX_ATTESTATIONS] - deposits*: List[Deposit, Limit MAX_DEPOSITS] - voluntary_exits*: List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] - - SigVerifiedBeaconBlockBody* = object - ## A BeaconBlock body with signatures verified - ## including: - ## - Randao reveal - ## - Attestations - ## - ProposerSlashing (SignedBeaconBlockHeader) - ## - AttesterSlashing (IndexedAttestation) - ## - SignedVoluntaryExits - ## - ## - ETH1Data (Deposits) can contain invalid BLS signatures - ## - ## The block state transition has NOT been verified - randao_reveal*: TrustedSig - eth1_data*: Eth1Data - graffiti*: GraffitiBytes - - # Operations - proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] - attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] - attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] - deposits*: List[Deposit, Limit MAX_DEPOSITS] - voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] - - TrustedBeaconBlockBody* = object - ## A full verified block - randao_reveal*: TrustedSig - eth1_data*: Eth1Data - graffiti*: GraffitiBytes - - # Operations - proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] - attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] - attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] - deposits*: List[Deposit, Limit MAX_DEPOSITS] - voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] - - SomeSignedBeaconBlock* = SignedBeaconBlock | SigVerifiedSignedBeaconBlock | TrustedSignedBeaconBlock - SomeBeaconBlock* = BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock - SomeBeaconBlockBody* = BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody SomeAttestation* = Attestation | TrustedAttestation SomeIndexedAttestation* = IndexedAttestation | TrustedIndexedAttestation SomeProposerSlashing* = ProposerSlashing | TrustedProposerSlashing @@ -394,65 +271,6 @@ type SomeSignedBeaconBlockHeader* = SignedBeaconBlockHeader | TrustedSignedBeaconBlockHeader SomeSignedVoluntaryExit* = SignedVoluntaryExit | TrustedSignedVoluntaryExit - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconstate - BeaconState* = object - # Versioning - genesis_time*: uint64 - genesis_validators_root*: Eth2Digest - slot*: Slot - fork*: Fork - - # History - latest_block_header*: BeaconBlockHeader ##\ - ## `latest_block_header.state_root == ZERO_HASH` temporarily - - block_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] ##\ - ## Needed to process attestations, older to newer - - state_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] - historical_roots*: HashList[Eth2Digest, Limit HISTORICAL_ROOTS_LIMIT] - - # Eth1 - eth1_data*: Eth1Data - eth1_data_votes*: - HashList[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)] - eth1_deposit_index*: uint64 - - # Registry - validators*: HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT] - balances*: HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT] - - # Randomness - randao_mixes*: HashArray[Limit EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest] - - # Slashings - slashings*: HashArray[Limit EPOCHS_PER_SLASHINGS_VECTOR, uint64] ##\ - ## Per-epoch sums of slashed effective balances - - # Attestations - previous_epoch_attestations*: - HashList[PendingAttestation, Limit(MAX_ATTESTATIONS * SLOTS_PER_EPOCH)] - current_epoch_attestations*: - HashList[PendingAttestation, Limit(MAX_ATTESTATIONS * SLOTS_PER_EPOCH)] - - # Finality - justification_bits*: uint8 ##\ - ## Bit set for every recent justified epoch - ## Model a Bitvector[4] as a one-byte uint, which should remain consistent - ## with ssz/hashing. - - previous_justified_checkpoint*: Checkpoint ##\ - ## Previous epoch snapshot - - current_justified_checkpoint*: Checkpoint - finalized_checkpoint*: Checkpoint - - # TODO Careful, not nil analysis is broken / incomplete and the semantics will - # likely change in future versions of the language: - # https://github.com/nim-lang/RFCs/issues/250 - BeaconStateRef* = ref BeaconState not nil - NilableBeaconStateRef* = ref BeaconState - # Please note that this type is not part of the spec ImmutableValidatorData* = object pubkey*: ValidatorPubKey @@ -517,37 +335,20 @@ type message*: VoluntaryExit signature*: TrustedSig - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signedbeaconblock - SignedBeaconBlock* = object - message*: BeaconBlock - signature*: ValidatorSig + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblockheader + BeaconBlockHeader* = object + slot*: Slot + proposer_index*: uint64 + parent_root*: Eth2Digest + state_root*: Eth2Digest + body_root*: Eth2Digest - root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signingdata + SigningData* = object + object_root*: Eth2Digest + domain*: Eth2Domain - SigVerifiedSignedBeaconBlock* = object - ## A SignedBeaconBlock with signatures verified - ## including: - ## - Block signature - ## - BeaconBlockBody - ## - Randao reveal - ## - Attestations - ## - ProposerSlashing (SignedBeaconBlockHeader) - ## - AttesterSlashing (IndexedAttestation) - ## - SignedVoluntaryExits - ## - ## - ETH1Data (Deposits) can contain invalid BLS signatures - ## - ## The block state transition has NOT been verified - message*: SigVerifiedBeaconBlock - signature*: TrustedSig - - root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block - - TrustedSignedBeaconBlock* = object - message*: TrustedBeaconBlock - signature*: TrustedSig - - root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + GraffitiBytes* = distinct array[MAX_GRAFFITI_SIZE, byte] # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signedbeaconblockheader SignedBeaconBlockHeader* = object @@ -569,10 +370,6 @@ type message*: AggregateAndProof signature*: ValidatorSig - HashedBeaconState* = object - data*: BeaconState - root*: Eth2Digest # hash_tree_root(data) - # This doesn't know about forks or branches in the DAG. It's for straight, # linear chunks of the chain. StateCache* = object @@ -753,25 +550,6 @@ type statuses*: seq[RewardStatus] total_balances*: TotalBalances - BlockRef* = ref object - ## Node in object graph guaranteed to lead back to tail block, and to have - ## a corresponding entry in database. - ## Block graph should form a tree - in particular, there are no cycles. - - root*: Eth2Digest ##\ - ## Root that can be used to retrieve block data from database - - parent*: BlockRef ##\ - ## Not nil, except for the tail - - slot*: Slot # could calculate this by walking to root, but.. - - StateData* = object - data*: HashedBeaconState - - blck*: BlockRef ##\ - ## The block associated with the state found in data - func getImmutableValidatorData*(validator: Validator): ImmutableValidatorData = ImmutableValidatorData( pubkey: validator.pubkey, @@ -894,6 +672,9 @@ static: doAssert high(int) >= high(int32) func `[]`*[T](a: var seq[T], b: ValidatorIndex): var T = a[b.int] +func `[]=`*[T](a: var seq[T], b: ValidatorIndex, c: T) = + a[b.int] = c + func `[]`*[T](a: seq[T], b: ValidatorIndex): auto = a[b.int] @@ -928,21 +709,6 @@ func `as`*(d: DepositData, T: type DepositMessage): T = ethTimeUnit Slot ethTimeUnit Epoch -Json.useCustomSerialization(BeaconState.justification_bits): - read: - let s = reader.readValue(string) - - if s.len != 4: - raiseUnexpectedValue(reader, "A string with 4 characters expected") - - try: - s.parseHexInt.uint8 - except ValueError: - raiseUnexpectedValue(reader, "The `justification_bits` value must be a hex string") - - write: - writer.writeValue "0x" & value.toHex - Json.useCustomSerialization(BitSeq): read: try: @@ -1013,27 +779,6 @@ func shortLog*(s: Slot): uint64 = func shortLog*(e: Epoch): uint64 = e - GENESIS_EPOCH -func shortLog*(v: SomeBeaconBlock): auto = - ( - slot: shortLog(v.slot), - proposer_index: v.proposer_index, - parent_root: shortLog(v.parent_root), - state_root: shortLog(v.state_root), - eth1data: v.body.eth1_data, - graffiti: $v.body.graffiti, - proposer_slashings_len: v.body.proposer_slashings.len(), - attester_slashings_len: v.body.attester_slashings.len(), - attestations_len: v.body.attestations.len(), - deposits_len: v.body.deposits.len(), - voluntary_exits_len: v.body.voluntary_exits.len(), - ) - -func shortLog*(v: SomeSignedBeaconBlock): auto = - ( - blck: shortLog(v.message), - signature: shortLog(v.signature) - ) - func shortLog*(v: BeaconBlockHeader): auto = ( slot: shortLog(v.slot), @@ -1119,7 +864,6 @@ func shortLog*(v: SomeSignedVoluntaryExit): auto = chronicles.formatIt Slot: it.shortLog chronicles.formatIt Epoch: it.shortLog -chronicles.formatIt BeaconBlock: it.shortLog chronicles.formatIt AttestationData: it.shortLog chronicles.formatIt Attestation: it.shortLog chronicles.formatIt Checkpoint: it.shortLog diff --git a/beacon_chain/spec/datatypes/phase0.nim b/beacon_chain/spec/datatypes/phase0.nim index 57034f68c..91dcf4b8e 100644 --- a/beacon_chain/spec/datatypes/phase0.nim +++ b/beacon_chain/spec/datatypes/phase0.nim @@ -1,2 +1,296 @@ +# beacon_chain +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# This file contains data types that are part of the spec and thus subject to +# serialization and spec updates. +# +# The spec folder in general contains code that has been hoisted from the +# specification and that follows the spec as closely as possible, so as to make +# it easy to keep up-to-date. +# +# These datatypes are used as specifications for serialization - thus should not +# be altered outside of what the spec says. Likewise, they should not be made +# `ref` - this can be achieved by wrapping them in higher-level +# types / composition + +# TODO Careful, not nil analysis is broken / incomplete and the semantics will +# likely change in future versions of the language: +# https://github.com/nim-lang/RFCs/issues/250 +{.experimental: "notnil".} + +{.push raises: [Defect].} + +import + std/[macros, intsets, json, strutils, tables], + stew/[assign2, byteutils], chronicles, + json_serialization/types as jsonTypes, + ../../ssz/types as sszTypes, ../crypto, ../digest, ../presets + import ./base export base + +type + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconstate + BeaconState* = object + # Versioning + genesis_time*: uint64 + genesis_validators_root*: Eth2Digest + slot*: Slot + fork*: Fork + + # History + latest_block_header*: BeaconBlockHeader ##\ + ## `latest_block_header.state_root == ZERO_HASH` temporarily + + block_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] ##\ + ## Needed to process attestations, older to newer + + state_roots*: HashArray[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest] + historical_roots*: HashList[Eth2Digest, Limit HISTORICAL_ROOTS_LIMIT] + + # Eth1 + eth1_data*: Eth1Data + eth1_data_votes*: + HashList[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)] + eth1_deposit_index*: uint64 + + # Registry + validators*: HashList[Validator, Limit VALIDATOR_REGISTRY_LIMIT] + balances*: HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT] + + # Randomness + randao_mixes*: HashArray[Limit EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest] + + # Slashings + slashings*: HashArray[Limit EPOCHS_PER_SLASHINGS_VECTOR, uint64] ##\ + ## Per-epoch sums of slashed effective balances + + # Attestations + previous_epoch_attestations*: + HashList[PendingAttestation, Limit(MAX_ATTESTATIONS * SLOTS_PER_EPOCH)] + current_epoch_attestations*: + HashList[PendingAttestation, Limit(MAX_ATTESTATIONS * SLOTS_PER_EPOCH)] + + # Finality + justification_bits*: uint8 ##\ + ## Bit set for every recent justified epoch + ## Model a Bitvector[4] as a one-byte uint, which should remain consistent + ## with ssz/hashing. + + previous_justified_checkpoint*: Checkpoint ##\ + ## Previous epoch snapshot + + current_justified_checkpoint*: Checkpoint + finalized_checkpoint*: Checkpoint + + # TODO Careful, not nil analysis is broken / incomplete and the semantics will + # likely change in future versions of the language: + # https://github.com/nim-lang/RFCs/issues/250 + BeaconStateRef* = ref BeaconState not nil + NilableBeaconStateRef* = ref BeaconState + + HashedBeaconState* = object + data*: BeaconState + root*: Eth2Digest # hash_tree_root(data) + + BlockRef* = ref object + ## Node in object graph guaranteed to lead back to tail block, and to have + ## a corresponding entry in database. + ## Block graph should form a tree - in particular, there are no cycles. + + root*: Eth2Digest ##\ + ## Root that can be used to retrieve block data from database + + parent*: BlockRef ##\ + ## Not nil, except for the tail + + slot*: Slot # could calculate this by walking to root, but.. + + StateData* = object + data*: HashedBeaconState + + blck*: BlockRef ##\ + ## The block associated with the state found in data + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblock + BeaconBlock* = object + ## For each slot, a proposer is chosen from the validator pool to propose + ## a new block. Once the block as been proposed, it is transmitted to + ## validators that will have a chance to vote on it through attestations. + ## Each block collects attestations, or votes, on past blocks, thus a chain + ## is formed. + + slot*: Slot + proposer_index*: uint64 + + parent_root*: Eth2Digest ##\ + ## Root hash of the previous block + + state_root*: Eth2Digest ##\ + ## The state root, _after_ this block has been processed + + body*: BeaconBlockBody + + SigVerifiedBeaconBlock* = object + ## A BeaconBlock that contains verified signatures + ## but that has not been verified for state transition + slot*: Slot + proposer_index*: uint64 + + parent_root*: Eth2Digest ##\ + ## Root hash of the previous block + + state_root*: Eth2Digest ##\ + ## The state root, _after_ this block has been processed + + body*: SigVerifiedBeaconBlockBody + + TrustedBeaconBlock* = object + ## When we receive blocks from outside sources, they are untrusted and go + ## through several layers of validation. Blocks that have gone through + ## validations can be trusted to be well-formed, with a correct signature, + ## having a parent and applying cleanly to the state that their parent + ## left them with. + ## + ## When loading such blocks from the database, to rewind states for example, + ## it is expensive to redo the validations (in particular, the signature + ## checks), thus `TrustedBlock` uses a `TrustedSig` type to mark that these + ## checks can be skipped. + ## + ## TODO this could probably be solved with some type trickery, but there + ## too many bugs in nim around generics handling, and we've used up + ## the trickery budget in the serialization library already. Until + ## then, the type must be manually kept compatible with its untrusted + ## cousin. + slot*: Slot + proposer_index*: uint64 + parent_root*: Eth2Digest ##\ + state_root*: Eth2Digest ##\ + body*: TrustedBeaconBlockBody + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beaconblockbody + BeaconBlockBody* = object + randao_reveal*: ValidatorSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[Attestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + SigVerifiedBeaconBlockBody* = object + ## A BeaconBlock body with signatures verified + ## including: + ## - Randao reveal + ## - Attestations + ## - ProposerSlashing (SignedBeaconBlockHeader) + ## - AttesterSlashing (IndexedAttestation) + ## - SignedVoluntaryExits + ## + ## - ETH1Data (Deposits) can contain invalid BLS signatures + ## + ## The block state transition has NOT been verified + randao_reveal*: TrustedSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + TrustedBeaconBlockBody* = object + ## A full verified block + randao_reveal*: TrustedSig + eth1_data*: Eth1Data + graffiti*: GraffitiBytes + + # Operations + proposer_slashings*: List[TrustedProposerSlashing, Limit MAX_PROPOSER_SLASHINGS] + attester_slashings*: List[TrustedAttesterSlashing, Limit MAX_ATTESTER_SLASHINGS] + attestations*: List[TrustedAttestation, Limit MAX_ATTESTATIONS] + deposits*: List[Deposit, Limit MAX_DEPOSITS] + voluntary_exits*: List[TrustedSignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS] + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#signedbeaconblock + SignedBeaconBlock* = object + message*: BeaconBlock + signature*: ValidatorSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + SigVerifiedSignedBeaconBlock* = object + ## A SignedBeaconBlock with signatures verified + ## including: + ## - Block signature + ## - BeaconBlockBody + ## - Randao reveal + ## - Attestations + ## - ProposerSlashing (SignedBeaconBlockHeader) + ## - AttesterSlashing (IndexedAttestation) + ## - SignedVoluntaryExits + ## + ## - ETH1Data (Deposits) can contain invalid BLS signatures + ## + ## The block state transition has NOT been verified + message*: SigVerifiedBeaconBlock + signature*: TrustedSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + TrustedSignedBeaconBlock* = object + message*: TrustedBeaconBlock + signature*: TrustedSig + + root* {.dontSerialize.}: Eth2Digest # cached root of signed beacon block + + SomeSignedBeaconBlock* = SignedBeaconBlock | SigVerifiedSignedBeaconBlock | TrustedSignedBeaconBlock + SomeBeaconBlock* = BeaconBlock | SigVerifiedBeaconBlock | TrustedBeaconBlock + SomeBeaconBlockBody* = BeaconBlockBody | SigVerifiedBeaconBlockBody | TrustedBeaconBlockBody + +chronicles.formatIt BeaconBlock: it.shortLog + +Json.useCustomSerialization(BeaconState.justification_bits): + read: + let s = reader.readValue(string) + + if s.len != 4: + raiseUnexpectedValue(reader, "A string with 4 characters expected") + + try: + s.parseHexInt.uint8 + except ValueError: + raiseUnexpectedValue(reader, "The `justification_bits` value must be a hex string") + + write: + writer.writeValue "0x" & value.toHex + +func shortLog*(v: SomeBeaconBlock): auto = + ( + slot: shortLog(v.slot), + proposer_index: v.proposer_index, + parent_root: shortLog(v.parent_root), + state_root: shortLog(v.state_root), + eth1data: v.body.eth1_data, + graffiti: $v.body.graffiti, + proposer_slashings_len: v.body.proposer_slashings.len(), + attester_slashings_len: v.body.attester_slashings.len(), + attestations_len: v.body.attestations.len(), + deposits_len: v.body.deposits.len(), + voluntary_exits_len: v.body.voluntary_exits.len(), + ) + +func shortLog*(v: SomeSignedBeaconBlock): auto = + ( + blck: shortLog(v.message), + signature: shortLog(v.signature) + ) diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 1b48807d3..0000db17d 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -13,7 +13,7 @@ import # Standard lib std/[math, tables], # Third-party - stew/endians2, + stew/[byteutils, endians2], # Internal ./datatypes/[phase0, altair], ./digest, ./crypto, ../ssz/merkleization @@ -52,7 +52,7 @@ func is_active_validator*(validator: Validator, epoch: Epoch): bool = validator.activation_epoch <= epoch and epoch < validator.exit_epoch # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_active_validator_indices -func get_active_validator_indices*(state: BeaconState, epoch: Epoch): +func get_active_validator_indices*(state: SomeBeaconState, epoch: Epoch): seq[ValidatorIndex] = ## Return the sequence of active validator indices at ``epoch``. result = newSeqOfCap[ValidatorIndex](state.validators.len) @@ -60,20 +60,20 @@ func get_active_validator_indices*(state: BeaconState, epoch: Epoch): if is_active_validator(state.validators[idx], epoch): result.add idx.ValidatorIndex -func get_active_validator_indices_len*(state: BeaconState, epoch: Epoch): uint64 = +func get_active_validator_indices_len*(state: SomeBeaconState, epoch: Epoch): + uint64 = for idx in 0..= GENESIS_SLOT, $state.slot compute_epoch_at_slot(state.slot) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_randao_mix -func get_randao_mix*(state: BeaconState, - epoch: Epoch): Eth2Digest = +func get_randao_mix*(state: SomeBeaconState, epoch: Epoch): Eth2Digest = ## Returns the randao mix at a recent ``epoch``. state.randao_mixes[epoch mod EPOCHS_PER_HISTORICAL_VECTOR] @@ -147,7 +147,7 @@ func get_domain*( compute_domain(domain_type, fork_version, genesis_validators_root) func get_domain*( - state: BeaconState, domain_type: DomainType, epoch: Epoch): Eth2Domain = + state: SomeBeaconState, domain_type: DomainType, epoch: Epoch): Eth2Domain = ## Return the signature domain (fork version concatenated with domain type) ## of a message. get_domain(state.fork, domain_type, epoch, state.genesis_validators_root) @@ -163,7 +163,8 @@ func compute_signing_root*(ssz_object: auto, domain: Eth2Domain): Eth2Digest = hash_tree_root(domain_wrapped_object) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_seed -func get_seed*(state: BeaconState, epoch: Epoch, domain_type: DomainType): Eth2Digest = +func get_seed*(state: SomeBeaconState, epoch: Epoch, domain_type: DomainType): + Eth2Digest = ## Return the seed at ``epoch``. var seed_input : array[4+8+32, byte] @@ -179,14 +180,6 @@ func get_seed*(state: BeaconState, epoch: Epoch, domain_type: DomainType): Eth2D epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1).data eth2digest(seed_input) -# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#get_flag_indices_and_weights -iterator get_flag_indices_and_weights*(): (ParticipationFlag, int) = - for item in [ - (TIMELY_HEAD_FLAG_INDEX, TIMELY_HEAD_WEIGHT), - (TIMELY_SOURCE_FLAG_INDEX, TIMELY_SOURCE_WEIGHT), - (TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_WEIGHT)]: - yield item - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/specs/altair/beacon-chain.md#add_flag func add_flag*(flags: ParticipationFlags, flag_index: int): ParticipationFlags = let flag = ParticipationFlags(1'u8 shl flag_index) diff --git a/beacon_chain/spec/presets/altair/mainnet.nim b/beacon_chain/spec/presets/altair/mainnet.nim index ec8ece409..3f492666e 100644 --- a/beacon_chain/spec/presets/altair/mainnet.nim +++ b/beacon_chain/spec/presets/altair/mainnet.nim @@ -11,7 +11,7 @@ const # Updated penalty values # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/configs/mainnet/altair.yaml#L5 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/configs/mainnet/altair.yaml#L3 CONFIG_NAME* = "mainnet" INACTIVITY_PENALTY_QUOTIENT_ALTAIR* = 50331648 ##\ @@ -23,28 +23,26 @@ const # Misc # --------------------------------------------------------------- # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/configs/mainnet/altair.yaml#L15 - SYNC_COMMITTEE_SIZE* = 1024 SYNC_PUBKEYS_PER_AGGREGATE* = 64 INACTIVITY_SCORE_BIAS* = 4 - # Time parameters + # Sync Committee # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/configs/mainnet/altair.yaml#L25 - EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 256 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/configs/mainnet/altair.yaml#L13 + SYNC_COMMITTEE_SIZE* = 512 + EPOCHS_PER_SYNC_COMMITTEE_PERIOD* = 512 # Signature domains (DOMAIN_SYNC_COMMITTEE) in spec/datatypes/base # Fork # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/configs/mainnet/altair.yaml#L36 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/configs/mainnet/altair.yaml#L36 # ALTAIR_FORK_VERSION is a runtime preset - ALTAIR_FORK_SLOT* = 0 # TBD + ALTAIR_FORK_EPOCH* = 18446744073709551615'u64 # Sync protocol # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.2/configs/mainnet/altair.yaml#L43 + # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/configs/mainnet/altair.yaml#L44 MIN_SYNC_COMMITTEE_PARTICIPANTS* = 1 - MAX_VALID_LIGHT_CLIENT_UPDATES* = 8192 - LIGHT_CLIENT_UPDATE_TIMEOUT* = 8192 diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index 43c0a4dd3..37d3d1748 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -9,7 +9,7 @@ import ../ssz/merkleization, - ./crypto, ./digest, ./datatypes, ./helpers, ./presets + ./crypto, ./digest, ./datatypes/[phase0, altair], ./helpers, ./presets template withTrust(sig: SomeSig, body: untyped): bool = when sig is TrustedSig: @@ -90,7 +90,7 @@ func get_block_signature*( proc verify_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - blck: Eth2Digest | SomeBeaconBlock | BeaconBlockHeader, + blck: Eth2Digest | SomeSomeBeaconBlock | BeaconBlockHeader, pubkey: ValidatorPubKey, signature: SomeSig): bool = withTrust(signature): diff --git a/beacon_chain/spec/signatures_batch.nim b/beacon_chain/spec/signatures_batch.nim index c98002ee5..4ed2820cb 100644 --- a/beacon_chain/spec/signatures_batch.nim +++ b/beacon_chain/spec/signatures_batch.nim @@ -16,6 +16,9 @@ import ./crypto, ./datatypes, ./helpers, ./presets, ./beaconstate, ./digest +# Otherwise, error. +import chronicles + export SignatureSet, BatchedBLSVerifierCache, batchVerify, batchVerifySerial, batchVerifyParallel func `$`*(s: SignatureSet): string = diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index 44c918d28..37dd2ae0f 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -45,13 +45,18 @@ import chronicles, stew/results, ../extras, ../ssz/merkleization, metrics, - ./datatypes, ./crypto, ./digest, ./helpers, ./signatures, ./validator, + ./datatypes/[phase0, altair], ./crypto, ./digest, ./helpers, ./signatures, ./validator, ./state_transition_block, ./state_transition_epoch, ../../nbench/bench_lab +# TODO why need anything except the first two? +type Foo = phase0.SomeSignedBeaconBlock | altair.SomeSignedBeaconBlock | phase0.SignedBeaconBlock | altair.SignedBeaconBlock | phase0.TrustedSignedBeaconBlock | altair.TrustedSignedBeaconBlock | phase0.SigVerifiedSignedBeaconBlock | altair.SigVerifiedSignedBeaconBlock + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function proc verify_block_signature*( - state: BeaconState, signed_block: SomeSignedBeaconBlock): bool {.nbench.} = + #state: SomeBeaconState, signed_block: SomeSomeSignedBeaconBlock): bool {.nbench.} = + state: SomeBeaconState, signed_block: Foo): bool {.nbench.} = + #state: SomeBeaconState, signed_block: phase0.SomeSignedBeaconBlock | altair.SomeSignedBeaconBlock): bool {.nbench.} = let proposer_index = signed_block.message.proposer_index if proposer_index >= state.validators.lenu64: @@ -70,7 +75,7 @@ proc verify_block_signature*( true # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function -proc verifyStateRoot(state: BeaconState, blck: BeaconBlock or SigVerifiedBeaconBlock): bool = +proc verifyStateRoot(state: SomeBeaconState, blck: phase0.BeaconBlock or phase0.SigVerifiedBeaconBlock or altair.BeaconBlock or altair.SigVerifiedBeaconBlock): bool = # This is inlined in state_transition(...) in spec. let state_root = hash_tree_root(state) if state_root != blck.state_root: @@ -80,25 +85,25 @@ proc verifyStateRoot(state: BeaconState, blck: BeaconBlock or SigVerifiedBeaconB else: true -proc verifyStateRoot(state: BeaconState, blck: TrustedBeaconBlock): bool = +proc verifyStateRoot(state: phase0.BeaconState, blck: phase0.TrustedBeaconBlock): bool = # This is inlined in state_transition(...) in spec. true type - RollbackProc* = proc(v: var BeaconState) {.gcsafe, raises: [Defect].} + RollbackProc* = proc(v: var phase0.BeaconState) {.gcsafe, raises: [Defect].} -proc noRollback*(state: var BeaconState) = +proc noRollback*(state: var phase0.BeaconState) = trace "Skipping rollback of broken state" type - RollbackHashedProc* = proc(state: var HashedBeaconState) {.gcsafe, raises: [Defect].} + RollbackHashedProc* = proc(state: var phase0.HashedBeaconState) {.gcsafe, raises: [Defect].} # Hashed-state transition functions # --------------------------------------------------------------- # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function func process_slot*( - state: var BeaconState, pre_state_root: Eth2Digest) {.nbench.} = + state: var SomeBeaconState, pre_state_root: Eth2Digest) {.nbench.} = # `process_slot` is the first stage of per-slot processing - it is run for # every slot, including epoch slots - it does not however update the slot # number! `pre_state_root` refers to the state root of the incoming @@ -126,7 +131,7 @@ func clear_epoch_from_cache(cache: var StateCache, epoch: Epoch) = # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function proc advance_slot( - state: var BeaconState, previous_slot_state_root: Eth2Digest, + state: var SomeBeaconState, previous_slot_state_root: Eth2Digest, flags: UpdateFlags, cache: var StateCache, rewards: var RewardInfo) {.nbench.} = # Do the per-slot and potentially the per-epoch processing, then bump the # slot number - we've now arrived at the slot state on top of which a block @@ -145,7 +150,7 @@ proc advance_slot( state.slot += 1 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function -proc process_slots*(state: var HashedBeaconState, slot: Slot, +proc process_slots*(state: var SomeHashedBeaconState, slot: Slot, cache: var StateCache, rewards: var RewardInfo, flags: UpdateFlags = {}): bool {.nbench.} = ## Process one or more slot transitions without blocks - if the slot transtion @@ -171,12 +176,14 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot, true -proc noRollback*(state: var HashedBeaconState) = +proc noRollback*(state: var phase0.HashedBeaconState) = trace "Skipping rollback of broken state" proc state_transition*( preset: RuntimePreset, - state: var HashedBeaconState, signedBlock: SomeSignedBeaconBlock, + # TODO this will be StateData + #state: var phase0.HashedBeaconState, signedBlock: phase0.SomeSignedBeaconBlock, + state: var (phase0.HashedBeaconState | altair.HashedBeaconState), signedBlock: phase0.SignedBeaconBlock | phase0.SigVerifiedSignedBeaconBlock | phase0.TrustedSignedBeaconBlock | altair.SignedBeaconBlock, cache: var StateCache, rewards: var RewardInfo, flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} = ## Apply a block to the state, advancing the slot counter as necessary. The @@ -219,7 +226,9 @@ proc state_transition*( # that the block is sane. if not (skipBLSValidation in flags or verify_block_signature(state.data, signedBlock)): - rollback(state) + when false: + # TODO fixme + rollback(state) return false trace "state_transition: processing block, signature passed", @@ -235,12 +244,16 @@ proc state_transition*( eth1_deposit_index = state.data.eth1_deposit_index, deposit_root = shortLog(state.data.eth1_data.deposit_root), error = res.error - rollback(state) + when false: + # TODO re-enable + rollback(state) return false if not (skipStateRootValidation in flags or verifyStateRoot(state.data, signedBlock.message)): - rollback(state) + when false: + # TODO re-enable + rollback(state) return false # only blocks currently being produced have an empty state root - we use a @@ -254,7 +267,7 @@ proc state_transition*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/validator.md#preparing-for-a-beaconblock proc makeBeaconBlock*( preset: RuntimePreset, - state: var HashedBeaconState, + state: var phase0.HashedBeaconState, proposer_index: ValidatorIndex, parent_root: Eth2Digest, randao_reveal: ValidatorSig, @@ -267,7 +280,7 @@ proc makeBeaconBlock*( voluntaryExits: seq[SignedVoluntaryExit], executionPayload: ExecutionPayload, rollback: RollbackHashedProc, - cache: var StateCache): Option[BeaconBlock] = + cache: var StateCache): Option[phase0.BeaconBlock] = ## Create a block for the given state. The last block applied to it must be ## the one identified by parent_root and process_slots must be called up to ## the slot for which a block is to be created. @@ -275,11 +288,11 @@ proc makeBeaconBlock*( # To create a block, we'll first apply a partial block to the state, skipping # some validations. - var blck = BeaconBlock( + var blck = phase0.BeaconBlock( slot: state.data.slot, proposer_index: proposer_index.uint64, parent_root: parent_root, - body: BeaconBlockBody( + body: phase0.BeaconBlockBody( randao_reveal: randao_reveal, eth1_data: eth1data, graffiti: graffiti, diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index d75e96074..9be43a6f4 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -20,16 +20,16 @@ {.push raises: [Defect].} import - std/[algorithm, intsets, options, sequtils], + std/[algorithm, intsets, options, sequtils, sets, tables], chronicles, ../extras, ../ssz/merkleization, metrics, - ./beaconstate, ./crypto, ./datatypes, ./digest, ./helpers, ./validator, - ./signatures, ./presets, + ./beaconstate, ./crypto, ./datatypes/[phase0, altair], ./digest, ./helpers, + ./validator, ./signatures, ./presets, ../../nbench/bench_lab # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#block-header func process_block_header*( - state: var BeaconState, blck: SomeBeaconBlock, flags: UpdateFlags, + state: var SomeBeaconState, blck: SomeSomeBeaconBlock, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = # Verify that the slots match if not (blck.slot == state.slot): @@ -72,7 +72,7 @@ func `xor`[T: array](a, b: T): T = # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#randao proc process_randao( - state: var BeaconState, body: SomeBeaconBlockBody, flags: UpdateFlags, + state: var SomeBeaconState, body: SomeSomeBeaconBlockBody, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = let proposer_index = get_beacon_proposer_index(state, cache) @@ -104,7 +104,7 @@ proc process_randao( ok() # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#eth1-data -func process_eth1_data(state: var BeaconState, body: SomeBeaconBlockBody): Result[void, cstring] {.nbench.}= +func process_eth1_data(state: var SomeBeaconState, body: SomeSomeBeaconBlockBody): Result[void, cstring] {.nbench.}= if not state.eth1_data_votes.add body.eth1_data: # Count is reset in process_final_updates, so this should never happen return err("process_eth1_data: no more room for eth1 data") @@ -123,7 +123,7 @@ func is_slashable_validator(validator: Validator, epoch: Epoch): bool = # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#proposer-slashings proc check_proposer_slashing*( - state: var BeaconState, proposer_slashing: SomeProposerSlashing, + state: var SomeBeaconState, proposer_slashing: SomeProposerSlashing, flags: UpdateFlags): Result[void, cstring] {.nbench.} = @@ -165,7 +165,7 @@ proc check_proposer_slashing*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#proposer-slashings proc process_proposer_slashing*( - state: var BeaconState, proposer_slashing: SomeProposerSlashing, + state: var SomeBeaconState, proposer_slashing: SomeProposerSlashing, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = ? check_proposer_slashing(state, proposer_slashing, flags) @@ -189,7 +189,7 @@ func is_slashable_attestation_data*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#attester-slashings proc check_attester_slashing*( - state: var BeaconState, + state: var SomeBeaconState, attester_slashing: SomeAttesterSlashing, flags: UpdateFlags ): Result[seq[ValidatorIndex], cstring] {.nbench.} = @@ -222,7 +222,7 @@ proc check_attester_slashing*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#attester-slashings proc process_attester_slashing*( - state: var BeaconState, + state: var SomeBeaconState, attester_slashing: SomeAttesterSlashing, flags: UpdateFlags, cache: var StateCache @@ -240,7 +240,7 @@ proc process_attester_slashing*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#voluntary-exits proc check_voluntary_exit*( - state: BeaconState, + state: SomeBeaconState, signed_voluntary_exit: SomeSignedVoluntaryExit, flags: UpdateFlags): Result[void, cstring] {.nbench.} = @@ -292,7 +292,7 @@ proc check_voluntary_exit*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#voluntary-exits proc process_voluntary_exit*( - state: var BeaconState, + state: var SomeBeaconState, signed_voluntary_exit: SomeSignedVoluntaryExit, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = @@ -303,8 +303,8 @@ proc process_voluntary_exit*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#operations proc process_operations(preset: RuntimePreset, - state: var BeaconState, - body: SomeBeaconBlockBody, + state: var SomeBeaconState, + body: SomeSomeBeaconBlockBody, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.} = # Verify that outstanding deposits are processed up to the maximum number of @@ -335,10 +335,70 @@ proc process_operations(preset: RuntimePreset, ok() +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing +proc process_sync_committee*( + state: var altair.BeaconState, aggregate: SyncAggregate, cache: var StateCache): + Result[void, cstring] {.nbench.} = + # Verify sync committee aggregate signature signing over the previous slot + # block root + let + committee_pubkeys = state.current_sync_committee.pubkeys + previous_slot = max(state.slot, Slot(1)) - 1 + domain = get_domain(state, DOMAIN_SYNC_COMMITTEE, compute_epoch_at_slot(previous_slot)) + signing_root = compute_signing_root(get_block_root_at_slot(state, previous_slot), domain) + + var participant_pubkeys: seq[ValidatorPubKey] + for i in 0 ..< committee_pubkeys.len: + if aggregate.sync_committee_bits[i]: + participant_pubkeys.add committee_pubkeys[i] + + # Empty participants allowed + if participant_pubkeys.len > 0 and not blsFastAggregateVerify( + participant_pubkeys, signing_root.data, aggregate.sync_committee_signature): + return err("process_sync_committee: invalid signature") + + # Compute participant and proposer rewards + let + total_active_increments = get_total_active_balance(state, cache) div EFFECTIVE_BALANCE_INCREMENT + total_base_rewards = get_base_reward_per_increment(state, cache) * total_active_increments + max_participant_rewards = total_base_rewards * SYNC_REWARD_WEIGHT div WEIGHT_DENOMINATOR div SLOTS_PER_EPOCH + participant_reward = max_participant_rewards div SYNC_COMMITTEE_SIZE + proposer_reward = participant_reward * PROPOSER_WEIGHT div (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT) + + # Apply participant and proposer rewards + + # stand-in to be replaced + # TODO obviously not viable as written + # TODO also, this could use the pubkey -> index map that's been approached a couple places + let s = toHashSet(state.current_sync_committee.pubkeys.data) # TODO leaking abstraction + var pubkeyIndices: Table[ValidatorPubKey, ValidatorIndex] + for i, v in state.validators: + if v.pubkey in s: + pubkeyIndices[v.pubkey] = i.ValidatorIndex + + let committee_indices = mapIt(state.current_sync_committee.pubkeys, pubkeyIndices.getOrDefault(it)) + var participant_indices: seq[ValidatorIndex] + for i, committee_index in committee_indices: + if aggregate.sync_committee_bits[i]: + participant_indices.add committee_index + for participant_index in participant_indices: + let proposer_index = get_beacon_proposer_index(state, cache) + if proposer_index.isSome: + increase_balance(state, participant_index, participant_reward) + increase_balance(state, proposer_index.get, proposer_reward) + else: + warn "process_sync_committee: get_beacon_proposer_index failed" + + ok() + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#block-processing +# TODO workaround for https://github.com/nim-lang/Nim/issues/18095 +# copy of datatypes/phase0.nim +type SomePhase0Block = + phase0.BeaconBlock | phase0.SigVerifiedBeaconBlock | phase0.TrustedBeaconBlock proc process_block*( preset: RuntimePreset, - state: var BeaconState, blck: SomeBeaconBlock, flags: UpdateFlags, + state: var phase0.BeaconState, blck: SomePhase0Block, flags: UpdateFlags, cache: var StateCache): Result[void, cstring] {.nbench.}= ## When there's a new block, we need to verify that the block is sane and ## update the state accordingly - the state is left in an unknown state when @@ -350,3 +410,24 @@ proc process_block*( ? process_operations(preset, state, blck.body, flags, cache) ok() + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#block-processing +# TODO workaround for https://github.com/nim-lang/Nim/issues/18095 +# copy of datatypes/altair.nim +type SomeAltairBlock = + altair.BeaconBlock | altair.SigVerifiedBeaconBlock | altair.TrustedBeaconBlock +proc process_block*( + preset: RuntimePreset, + state: var altair.BeaconState, blck: SomeAltairBlock, flags: UpdateFlags, + cache: var StateCache): Result[void, cstring] {.nbench.}= + ## When there's a new block, we need to verify that the block is sane and + ## update the state accordingly - the state is left in an unknown state when + ## block application fails (!) + + ? process_block_header(state, blck, flags, cache) + ? process_randao(state, blck.body, flags, cache) + ? process_eth1_data(state, blck.body) + ? process_operations(preset, state, blck.body, flags, cache) + ? process_sync_committee(state, blck.body.sync_aggregate, cache) # [New in Altair] + + ok() diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 477917c42..3de02a728 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -20,11 +20,11 @@ {.push raises: [Defect].} import - std/[math, sequtils, tables, algorithm], + std/[math, sequtils, sets, tables, algorithm], stew/[bitops2], chronicles, ../extras, ../ssz/merkleization, - ./beaconstate, ./crypto, ./datatypes, ./digest, ./helpers, ./validator, + ./beaconstate, ./crypto, ./datatypes/[phase0, altair], ./digest, ./helpers, ./validator, ../../nbench/bench_lab # Logging utilities @@ -49,7 +49,7 @@ template previous_epoch_target_attesters*(v: TotalBalances): Gwei = template previous_epoch_head_attesters*(v: TotalBalances): Gwei = max(EFFECTIVE_BALANCE_INCREMENT, v.previous_epoch_head_attesters_raw) -func init*(rewards: var RewardInfo, state: BeaconState) = +func init*(rewards: var RewardInfo, state: SomeBeaconState) = rewards.total_balances = TotalBalances() rewards.statuses.setLen(state.validators.len) @@ -58,15 +58,15 @@ func init*(rewards: var RewardInfo, state: BeaconState) = var flags: set[RewardFlags] if v[].slashed: - flags.incl(isSlashed) + flags.incl(RewardFlags.isSlashed) if state.get_current_epoch() >= v[].withdrawable_epoch: - flags.incl canWithdrawInCurrentEpoch + flags.incl RewardFlags.canWithdrawInCurrentEpoch if v[].is_active_validator(state.get_current_epoch()): rewards.total_balances.current_epoch_raw += v[].effective_balance if v[].is_active_validator(state.get_previous_epoch()): - flags.incl isActiveInPreviousEpoch + flags.incl RewardFlags.isActiveInPreviousEpoch rewards.total_balances.previous_epoch_raw += v[].effective_balance rewards.statuses[i] = RewardStatus( @@ -79,7 +79,7 @@ func add(a: var RewardDelta, b: RewardDelta) = a.penalties += b.penalties func process_attestation( - self: var RewardInfo, state: BeaconState, a: PendingAttestation, + self: var RewardInfo, state: phase0.BeaconState, a: PendingAttestation, cache: var StateCache) = # Collect information about the attestation var @@ -87,10 +87,10 @@ func process_attestation( is_previous_epoch_attester: Option[InclusionInfo] if a.data.target.epoch == state.get_current_epoch(): - flags.incl isCurrentEpochAttester + flags.incl RewardFlags.isCurrentEpochAttester if a.data.target.root == get_block_root(state, state.get_current_epoch()): - flags.incl isCurrentEpochTargetAttester + flags.incl RewardFlags.isCurrentEpochTargetAttester elif a.data.target.epoch == state.get_previous_epoch(): is_previous_epoch_attester = some(InclusionInfo( @@ -99,10 +99,10 @@ func process_attestation( )) if a.data.target.root == get_block_root(state, state.get_previous_epoch()): - flags.incl isPreviousEpochTargetAttester + flags.incl RewardFlags.isPreviousEpochTargetAttester if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot): - flags.incl isPreviousEpochHeadAttester + flags.incl RewardFlags.isPreviousEpochHeadAttester # Update the cache for all participants for validator_index in get_attesting_indices( @@ -120,7 +120,7 @@ func process_attestation( v.is_previous_epoch_attester = is_previous_epoch_attester func process_attestations*( - self: var RewardInfo, state: BeaconState, cache: var StateCache) = + self: var RewardInfo, state: phase0.BeaconState, cache: var StateCache) = # Walk state attestations and update the status information for a in state.previous_epoch_attestations: process_attestation(self, state, a, cache) @@ -128,47 +128,54 @@ func process_attestations*( process_attestation(self, state, a, cache) for idx, v in self.statuses: - if isSlashed in v.flags: + if v.flags.contains RewardFlags.isSlashed: continue let validator_balance = state.validators[idx].effective_balance - if isCurrentEpochAttester in v.flags: + if v.flags.contains RewardFlags.isCurrentEpochAttester: self.total_balances.current_epoch_attesters_raw += validator_balance - if isCurrentEpochTargetAttester in v.flags: + if v.flags.contains RewardFlags.isCurrentEpochTargetAttester: self.total_balances.current_epoch_target_attesters_raw += validator_balance if v.is_previous_epoch_attester.isSome(): self.total_balances.previous_epoch_attesters_raw += validator_balance - if isPreviousEpochTargetAttester in v.flags: + if v.flags.contains RewardFlags.isPreviousEpochTargetAttester: self.total_balances.previous_epoch_target_attesters_raw += validator_balance - if isPreviousEpochHeadAttester in v.flags: + if v.flags.contains RewardFlags.isPreviousEpochHeadAttester: self.total_balances.previous_epoch_head_attesters_raw += validator_balance func is_eligible_validator*(validator: RewardStatus): bool = - isActiveInPreviousEpoch in validator.flags or - (isSlashed in validator.flags and - (canWithdrawInCurrentEpoch notin validator.flags)) + validator.flags.contains(RewardFlags.isActiveInPreviousEpoch) or + (validator.flags.contains(RewardFlags.isSlashed) and not + (validator.flags.contains RewardFlags.canWithdrawInCurrentEpoch)) # Spec # -------------------------------------------------------- -# https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_total_active_balance -func get_total_active_balance*(state: BeaconState, cache: var StateCache): Gwei = - ## Return the combined effective balance of the active validators. - # Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei - # minimum to avoid divisions by zero. - - let epoch = state.get_current_epoch() - - get_total_balance( - state, cache.get_shuffled_active_validator_indices(state, epoch)) +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_unslashed_participating_indices +func get_unslashed_participating_indices( + state: altair.BeaconState, flag_index: int, epoch: Epoch): + HashSet[ValidatorIndex] = + ## Return the set of validator indices that are both active and unslashed for + ## the given ``flag_index`` and ``epoch``. + doAssert epoch in [get_previous_epoch(state), get_current_epoch(state)] + let + epoch_participation = + if epoch == get_current_epoch(state): + state.current_epoch_participation + else: + state.previous_epoch_participation + active_validator_indices = get_active_validator_indices(state, epoch) + participating_indices = filterIt( + active_validator_indices, has_flag(epoch_participation[it], flag_index)) + toHashSet(filterIt(participating_indices, not state.validators[it].slashed)) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#justification-and-finalization -proc process_justification_and_finalization*(state: var BeaconState, +proc process_justification_and_finalization*(state: var phase0.BeaconState, total_balances: TotalBalances, flags: UpdateFlags = {}) {.nbench.} = # Initial FFG checkpoint values have a `0x00` stub for `root`. # Skip FFG updates in the first two epochs to avoid corner cases that might @@ -264,8 +271,120 @@ proc process_justification_and_finalization*(state: var BeaconState, current_epoch = current_epoch, checkpoint = shortLog(state.finalized_checkpoint) +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#justification-and-finalization +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#justification-and-finalization +# TODO merge these things -- effectively, the phase0 process_justification_and_finalization is mostly a stub in this world +proc weigh_justification_and_finalization(state: var altair.BeaconState, + total_balances: TotalBalances, + previous_epoch_target_balance: Gwei, + current_epoch_target_balance: Gwei, + flags: UpdateFlags = {}) = + let + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + + ## Spec: + ## state.justification_bits[1:] = state.justification_bits[:-1] + ## state.justification_bits[0] = 0b0 + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#constants + const JUSTIFICATION_BITS_LENGTH = 4 + + state.justification_bits = (state.justification_bits shl 1) and + cast[uint8]((2^JUSTIFICATION_BITS_LENGTH) - 1) + + let total_active_balance = total_balances.current_epoch + if total_balances.previous_epoch_target_attesters * 3 >= + total_active_balance * 2: + state.current_justified_checkpoint = + Checkpoint(epoch: previous_epoch, + root: get_block_root(state, previous_epoch)) + state.justification_bits.setBit 1 + + trace "Justified with previous epoch", + current_epoch = current_epoch, + checkpoint = shortLog(state.current_justified_checkpoint) + elif verifyFinalization in flags: + warn "Low attestation participation in previous epoch", + total_balances, epoch = get_current_epoch(state) + + if total_balances.current_epoch_target_attesters * 3 >= + total_active_balance * 2: + state.current_justified_checkpoint = + Checkpoint(epoch: current_epoch, + root: get_block_root(state, current_epoch)) + state.justification_bits.setBit 0 + + trace "Justified with current epoch", + current_epoch = current_epoch, + checkpoint = shortLog(state.current_justified_checkpoint) + + # Process finalizations + let bitfield = state.justification_bits + + ## The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th + ## as source + if (bitfield and 0b1110) == 0b1110 and + old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + + trace "Finalized with rule 234", + current_epoch = current_epoch, + checkpoint = shortLog(state.finalized_checkpoint) + + ## The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as + ## source + if (bitfield and 0b110) == 0b110 and + old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + + trace "Finalized with rule 23", + current_epoch = current_epoch, + checkpoint = shortLog(state.finalized_checkpoint) + + ## The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as + ## source + if (bitfield and 0b111) == 0b111 and + old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + + trace "Finalized with rule 123", + current_epoch = current_epoch, + checkpoint = shortLog(state.finalized_checkpoint) + + ## The 1st/2nd most recent epochs are justified, the 1st using the 2nd as + ## source + if (bitfield and 0b11) == 0b11 and + old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + + trace "Finalized with rule 12", + current_epoch = current_epoch, + checkpoint = shortLog(state.finalized_checkpoint) + +proc process_justification_and_finalization*(state: var altair.BeaconState, + total_balances: TotalBalances, flags: UpdateFlags = {}) {.nbench.} = + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might + # result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + let + # these ultimately differ from phase0 only in these lines + # ref: https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#justification-and-finalization + previous_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state)) + current_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, get_current_epoch(state)) + previous_target_balance = get_total_balance(state, previous_indices) + current_target_balance = get_total_balance(state, current_indices) + weigh_justification_and_finalization(state, total_balances, previous_target_balance, current_target_balance, flags) + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#helpers -func get_base_reward_sqrt*(state: BeaconState, index: ValidatorIndex, +func get_base_reward_sqrt*(state: phase0.BeaconState, index: ValidatorIndex, total_balance_sqrt: auto): Gwei = # Spec function recalculates total_balance every time, which creates an # O(n^2) situation. @@ -280,9 +399,14 @@ func get_proposer_reward(base_reward: Gwei): Gwei = func is_in_inactivity_leak(finality_delay: uint64): bool = finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY -func get_finality_delay(state: BeaconState): uint64 = +func get_finality_delay(state: SomeBeaconState): uint64 = get_previous_epoch(state) - state.finalized_checkpoint.epoch +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/phase0/beacon-chain.md#rewards-and-penalties-1 +func is_in_inactivity_leak(state: altair.BeaconState): bool = + # TODO remove this, see above + get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY + func get_attestation_component_delta(is_unslashed_attester: bool, attesting_balance: Gwei, total_balance: Gwei, @@ -311,7 +435,7 @@ func get_source_delta*(validator: RewardStatus, ## Return attester micro-rewards/penalties for source-vote for each validator. get_attestation_component_delta( validator.is_previous_epoch_attester.isSome() and - (isSlashed notin validator.flags), + not (validator.flags.contains RewardFlags.isSlashed), total_balances.previous_epoch_attesters, total_balances.current_epoch, base_reward, @@ -323,8 +447,8 @@ func get_target_delta*(validator: RewardStatus, finality_delay: uint64): RewardDelta = ## Return attester micro-rewards/penalties for target-vote for each validator. get_attestation_component_delta( - isPreviousEpochTargetAttester in validator.flags and - (isSlashed notin validator.flags), + validator.flags.contains(RewardFlags.isPreviousEpochTargetAttester) and + not (validator.flags.contains(RewardFlags.isSlashed)), total_balances.previous_epoch_target_attesters, total_balances.current_epoch, base_reward, @@ -336,8 +460,8 @@ func get_head_delta*(validator: RewardStatus, finality_delay: uint64): RewardDelta = ## Return attester micro-rewards/penalties for head-vote for each validator. get_attestation_component_delta( - isPreviousEpochHeadAttester in validator.flags and - (isSlashed notin validator.flags), + validator.flags.contains(RewardFlags.isPreviousEpochHeadAttester) and + ((not validator.flags.contains(RewardFlags.isSlashed))), total_balances.previous_epoch_head_attesters, total_balances.current_epoch, base_reward, @@ -347,7 +471,7 @@ func get_inclusion_delay_delta*(validator: RewardStatus, base_reward: uint64): (RewardDelta, Option[(uint64, RewardDelta)]) = ## Return proposer and inclusion delay micro-rewards/penalties for each validator. - if validator.is_previous_epoch_attester.isSome() and (isSlashed notin validator.flags): + if validator.is_previous_epoch_attester.isSome() and ((not validator.flags.contains(RewardFlags.isSlashed))): let inclusion_info = validator.is_previous_epoch_attester.get() proposer_reward = get_proposer_reward(base_reward) @@ -373,8 +497,8 @@ func get_inactivity_penalty_delta*(validator: RewardStatus, # Additionally, all validators whose FFG target didn't match are penalized extra # This condition is equivalent to this condition from the spec: # `index not in get_unslashed_attesting_indices(state, matching_target_attestations)` - if (isSlashed in validator.flags) or - (isPreviousEpochTargetAttester notin validator.flags): + if (validator.flags.contains(RewardFlags.isSlashed)) or + ((not validator.flags.contains(RewardFlags.isPreviousEpochTargetAttester))): delta.penalties += validator.current_epoch_effective_balance * finality_delay div INACTIVITY_PENALTY_QUOTIENT @@ -382,7 +506,7 @@ func get_inactivity_penalty_delta*(validator: RewardStatus, delta # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_attestation_deltas -func get_attestation_deltas(state: BeaconState, rewards: var RewardInfo) = +func get_attestation_deltas(state: phase0.BeaconState, rewards: var RewardInfo) = ## Update rewards with attestation reward/penalty deltas for each validator. let @@ -425,9 +549,81 @@ func get_attestation_deltas(state: BeaconState, rewards: var RewardInfo) = rewards.statuses[proposer_index].delta.add( proposer_delta.get()[1]) +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward_per_increment +func get_base_reward_per_increment(state: altair.BeaconState, total_balances: TotalBalances): Gwei = + EFFECTIVE_BALANCE_INCREMENT * BASE_REWARD_FACTOR div + integer_squareroot(total_balances.current_epoch()) + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_base_reward +func get_base_reward(state: altair.BeaconState, index: ValidatorIndex, total_balances: TotalBalances): Gwei = + ## Return the base reward for the validator defined by ``index`` with respect + ## to the current ``state``. + ## + ## Note: An optimally performing validator can earn one base reward per epoch + ## over a long time horizon. This takes into account both per-epoch (e.g. + ## attestation) and intermittent duties (e.g. block proposal and sync + ## committees). + let increments = + state.validators[index].effective_balance div EFFECTIVE_BALANCE_INCREMENT + increments * get_base_reward_per_increment(state, total_balances) + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#get_flag_index_deltas +proc get_flag_index_deltas(state: altair.BeaconState, flag_index: int, total_balances: TotalBalances): + (seq[Gwei], seq[Gwei]) = + ## Return the deltas for a given ``flag_index`` by scanning through the + ## participation flags. + var + rewards = repeat(Gwei(0), len(state.validators)) + penalties = repeat(Gwei(0), len(state.validators)) + let + previous_epoch = get_previous_epoch(state) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag_index, previous_epoch) + weight = PARTICIPATION_FLAG_WEIGHTS[flag_index].uint64 # safe + unslashed_participating_balance = get_total_balance(state, unslashed_participating_indices) + unslashed_participating_increments = unslashed_participating_balance div EFFECTIVE_BALANCE_INCREMENT + active_increments = total_balances.current_epoch() div EFFECTIVE_BALANCE_INCREMENT + + for index in 0 ..< state.validators.len: + # TODO Obviously not great + let v = state.validators[index] + if not (is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch)): + continue + + let base_reward = get_base_reward(state, index.ValidatorIndex, total_balances) + if index.ValidatorIndex in unslashed_participating_indices: + if not is_in_inactivity_leak(state): + let reward_numerator = base_reward * weight * unslashed_participating_increments + rewards[index] += Gwei(reward_numerator div (active_increments * WEIGHT_DENOMINATOR)) + elif flag_index != TIMELY_HEAD_FLAG_INDEX: + penalties[index] += Gwei(base_reward * weight div WEIGHT_DENOMINATOR) + (rewards, penalties) + +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#modified-get_inactivity_penalty_deltas +func get_inactivity_penalty_deltas(state: altair.BeaconState): (seq[Gwei], seq[Gwei]) = + ## Return the inactivity penalty deltas by considering timely target + ## participation flags and inactivity scores. + var + rewards = repeat(Gwei(0), len(state.validators)) + penalties = repeat(Gwei(0), len(state.validators)) + let + previous_epoch = get_previous_epoch(state) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, previous_epoch) + for index in 0 ..< state.validators.len: + # get_eligible_validator_indices() + let v = state.validators[index] + if not (is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch)): + continue + + if not (index.ValidatorIndex in matching_target_indices): + let + penalty_numerator = state.validators[index].effective_balance * state.inactivity_scores[index] + penalty_denominator = uint64(INACTIVITY_SCORE_BIAS * INACTIVITY_PENALTY_QUOTIENT_ALTAIR) + penalties[index] += Gwei(penalty_numerator div penalty_denominator) + (rewards, penalties) + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#process_rewards_and_penalties func process_rewards_and_penalties( - state: var BeaconState, rewards: var RewardInfo) {.nbench.} = + state: var phase0.BeaconState, rewards: var RewardInfo) {.nbench.} = # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are # for work done in the previous epoch doAssert rewards.statuses.len == state.validators.len @@ -446,12 +642,42 @@ func process_rewards_and_penalties( increase_balance(state.balances.asSeq()[idx], v.delta.rewards) decrease_balance(state.balances.asSeq()[idx], v.delta.penalties) +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#rewards-and-penalties +proc process_rewards_and_penalties( + state: var altair.BeaconState, rewards: var RewardInfo) {.nbench.} = + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are + # for work done in the previous epoch + doAssert rewards.statuses.len == state.validators.len + + if get_current_epoch(state) == GENESIS_EPOCH: + return + + # TODO look at phase0 optimizations, however relevant they still are. Altair + # is supposed to incorporate some of this into the protocol, so re-assess + # TODO efficiency-wise, presumably a better way. look at again, once tests pass + var deltas = mapIt(0 ..< PARTICIPATION_FLAG_WEIGHTS.len, get_flag_index_deltas(state, it, rewards.total_balances)) + deltas.add get_inactivity_penalty_deltas(state) + for (rewards, penalties) in deltas: + for index in 0 ..< len(state.validators): + increase_balance(state, ValidatorIndex(index), rewards[index]) + decrease_balance(state, ValidatorIndex(index), penalties[index]) + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#slashings -func process_slashings*(state: var BeaconState, total_balance: Gwei) {.nbench.}= +# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#slashings +func process_slashings*(state: var SomeBeaconState, total_balance: Gwei) {.nbench.}= let epoch = get_current_epoch(state) + multiplier = + # tradeoff here about interleaving phase0/altair, but for these + # single-constant changes... + uint64(when state is phase0.BeaconState: + PROPORTIONAL_SLASHING_MULTIPLIER + elif state is altair.BeaconState: + PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR + else: + raiseAssert "process_slashings: incorrect BeaconState type") adjusted_total_slashing_balance = - min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + min(sum(state.slashings) * multiplier, total_balance) for index in 0..= 2: + doAssert state.current_justified_checkpoint.epoch + 2 >= currentEpoch + + if verifyFinalization in flags and currentEpoch >= 3: + # Rule 2/3/4 finalization results in the most pessimal case. The other + # three finalization rules finalize more quickly as long as the any of + # the finalization rules triggered. + doAssert state.finalized_checkpoint.epoch + 3 >= currentEpoch + + process_inactivity_updates(state) # [New in Altair] + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#rewards-and-penalties-1 + process_rewards_and_penalties(state, rewards) + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#registry-updates + process_registry_updates(state, cache) + + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#slashings + process_slashings(state, rewards.total_balances.current_epoch) + + process_eth1_data_reset(state) + + process_effective_balance_updates(state) + + process_slashings_reset(state) + + process_randao_mixes_reset(state) + + process_historical_roots_update(state) + + process_participation_flag_updates(state) # [New in Altair] + + process_sync_committee_updates(state) # [New in Altair] diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 7f78c394f..40e071100 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -10,7 +10,7 @@ import std/[options, math, tables], - ./datatypes, ./digest, ./helpers + ./datatypes/[phase0, altair], ./digest, ./helpers const SEED_SIZE = sizeof(Eth2Digest) @@ -122,8 +122,8 @@ func shuffle_list*(input: var seq[ValidatorIndex], seed: Eth2Digest) = shuffle -func get_shuffled_active_validator_indices*(state: BeaconState, epoch: Epoch): - seq[ValidatorIndex] = +func get_shuffled_active_validator_indices*( + state: SomeBeaconState, epoch: Epoch): seq[ValidatorIndex] = # Non-spec function, to cache a data structure from which one can cheaply # compute both get_active_validator_indexes() and get_beacon_committee(). var active_validator_indices = get_active_validator_indices(state, epoch) @@ -134,7 +134,7 @@ func get_shuffled_active_validator_indices*(state: BeaconState, epoch: Epoch): active_validator_indices func get_shuffled_active_validator_indices*( - cache: var StateCache, state: BeaconState, epoch: Epoch): + cache: var StateCache, state: SomeBeaconState, epoch: Epoch): var seq[ValidatorIndex] = # `cache` comes first because of nim's borrowing rules for the `var` return - # the `var` returns avoids copying the validator set. @@ -145,7 +145,7 @@ func get_shuffled_active_validator_indices*( return cache.shuffled_active_validator_indices.mgetOrPut(epoch, indices) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_active_validator_indices -func count_active_validators*(state: BeaconState, +func count_active_validators*(state: SomeBeaconState, epoch: Epoch, cache: var StateCache): uint64 = cache.get_shuffled_active_validator_indices(state, epoch).lenu64 @@ -156,7 +156,7 @@ func get_committee_count_per_slot*(num_active_validators: uint64): uint64 = num_active_validators div SLOTS_PER_EPOCH div TARGET_COMMITTEE_SIZE, 1'u64, MAX_COMMITTEES_PER_SLOT) -func get_committee_count_per_slot*(state: BeaconState, +func get_committee_count_per_slot*(state: SomeBeaconState, epoch: Epoch, cache: var StateCache): uint64 = # Return the number of committees at ``slot``. @@ -168,7 +168,7 @@ func get_committee_count_per_slot*(state: BeaconState, # Otherwise, get_beacon_committee(...) cannot access some committees. doAssert (SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT) >= uint64(result) -func get_committee_count_per_slot*(state: BeaconState, +func get_committee_count_per_slot*(state: SomeBeaconState, slot: Slot, cache: var StateCache): uint64 = get_committee_count_per_slot(state, slot.compute_epoch_at_slot, cache) @@ -181,8 +181,9 @@ func get_previous_epoch*(current_epoch: Epoch): Epoch = else: current_epoch - 1 -func get_previous_epoch*(state: BeaconState): Epoch = +func get_previous_epoch*(state: SomeBeaconState): Epoch = ## Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). + # Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). get_previous_epoch(get_current_epoch(state)) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#compute_committee @@ -228,7 +229,7 @@ func compute_committee_len*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_beacon_committee iterator get_beacon_committee*( - state: BeaconState, slot: Slot, index: CommitteeIndex, + state: SomeBeaconState, slot: Slot, index: CommitteeIndex, cache: var StateCache): ValidatorIndex = ## Return the beacon committee at ``slot`` for ``index``. let @@ -242,7 +243,7 @@ iterator get_beacon_committee*( ): yield idx func get_beacon_committee*( - state: BeaconState, slot: Slot, index: CommitteeIndex, + state: SomeBeaconState, slot: Slot, index: CommitteeIndex, cache: var StateCache): seq[ValidatorIndex] = ## Return the beacon committee at ``slot`` for ``index``. let @@ -257,7 +258,7 @@ func get_beacon_committee*( # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_beacon_committee func get_beacon_committee_len*( - state: BeaconState, slot: Slot, index: CommitteeIndex, + state: SomeBeaconState, slot: Slot, index: CommitteeIndex, cache: var StateCache): uint64 = # Return the number of members in the beacon committee at ``slot`` for ``index``. let @@ -272,7 +273,7 @@ func get_beacon_committee_len*( ) # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#compute_shuffled_index -func compute_shuffled_index( +func compute_shuffled_index*( index: uint64, index_count: uint64, seed: Eth2Digest): uint64 = ## Return the shuffled index corresponding to ``seed`` (and ``index_count``). doAssert index < index_count @@ -307,8 +308,8 @@ func compute_shuffled_index( cur_idx_permuted # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#compute_proposer_index -func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex], - seed: Eth2Digest): Option[ValidatorIndex] = +func compute_proposer_index(state: SomeBeaconState, + indices: seq[ValidatorIndex], seed: Eth2Digest): Option[ValidatorIndex] = ## Return from ``indices`` a random index sampled by effective balance. const MAX_RANDOM_BYTE = 255 @@ -334,7 +335,8 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex], i += 1 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_beacon_proposer_index -func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: Slot): +func get_beacon_proposer_index*( + state: SomeBeaconState, cache: var StateCache, slot: Slot): Option[ValidatorIndex] = cache.beacon_proposer_indices.withValue(slot, proposer) do: return proposer[] @@ -366,6 +368,6 @@ func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache, slot: return res # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#get_beacon_proposer_index -func get_beacon_proposer_index*(state: BeaconState, cache: var StateCache): +func get_beacon_proposer_index*(state: SomeBeaconState, cache: var StateCache): Option[ValidatorIndex] = get_beacon_proposer_index(state, cache, state.slot) diff --git a/beacon_chain/ssz/bytes_reader.nim b/beacon_chain/ssz/bytes_reader.nim index 347553d4a..2c1678d69 100644 --- a/beacon_chain/ssz/bytes_reader.nim +++ b/beacon_chain/ssz/bytes_reader.nim @@ -12,7 +12,8 @@ import std/[typetraits, options], stew/[endians2, objects], - ../spec/[digest, datatypes], ./types, ./spec_types, ./merkleization + ../spec/digest, ./types, ./spec_types, ./merkleization, + ../spec/datatypes/[phase0, altair] template raiseIncorrectSize(T: type) = const typeName = name(T) @@ -59,6 +60,9 @@ template fromSszBytes*(T: type Slot, bytes: openArray[byte]): T = template fromSszBytes*(T: type Epoch, bytes: openArray[byte]): T = T fromSszBytes(uint64, bytes) +template fromSszBytes*(T: type ParticipationFlags, bytes: openArray[byte]): T = + T fromSszBytes(uint8, bytes) + func fromSszBytes*(T: type ForkDigest, bytes: openArray[byte]): T {.raisesssz.} = if bytes.len != sizeof(result): raiseIncorrectSize T @@ -252,7 +256,8 @@ func readSszValue*[T](input: openArray[byte], type(field), input.toOpenArray(int(startOffset), int(endOffset - 1))) - when val is SignedBeaconBlock | TrustedSignedBeaconBlock: + when val is phase0.SignedBeaconBlock | phase0.TrustedSignedBeaconBlock | + altair.SignedBeaconBlock | altair.TrustedSignedBeaconBlock: if updateRoot: val.root = hash_tree_root(val.message) else: diff --git a/tests/official/all_fixtures_require_ssz.nim b/tests/official/all_fixtures_require_ssz.nim index d87412b39..648278493 100644 --- a/tests/official/all_fixtures_require_ssz.nim +++ b/tests/official/all_fixtures_require_ssz.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2021 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -20,6 +20,7 @@ import ./test_fixture_operations_attester_slashings, ./test_fixture_operations_block_header, ./test_fixture_operations_proposer_slashings, + ./test_fixture_operations_sync_committee, ./test_fixture_operations_voluntary_exit summarizeLongTests("FixtureAll") diff --git a/tests/official/test_fixture_const_sanity_check.nim b/tests/official/test_fixture_const_sanity_check.nim index c1f81e2fe..66a4df058 100644 --- a/tests/official/test_fixture_const_sanity_check.nim +++ b/tests/official/test_fixture_const_sanity_check.nim @@ -18,6 +18,7 @@ import ../testutil, ./fixtures_utils const + # TODO NimYAML issue SpecDir = currentSourcePath.rsplit(DirSep, 1)[0] / ".."/".."/"beacon_chain"/"spec" Config = SszTestsDir/const_preset/"config"/"phase0.yaml" diff --git a/tests/official/test_fixture_operations_attestations.nim b/tests/official/test_fixture_operations_attestations.nim index 14d5dfc54..33fa2155c 100644 --- a/tests/official/test_fixture_operations_attestations.nim +++ b/tests/official/test_fixture_operations_attestations.nim @@ -14,14 +14,15 @@ import unittest2, stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, beaconstate], + ../../beacon_chain/spec/beaconstate, + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OperationsAttestationsDir = SszTestsDir/const_preset/"phase0"/"operations"/"attestation"/"pyspec_tests" +const OperationsAttestationsDir = SszTestsDir/const_preset/"altair"/"operations"/"attestation"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_operations_attester_slashings.nim b/tests/official/test_fixture_operations_attester_slashings.nim index 3bbcbfc62..b683c8174 100644 --- a/tests/official/test_fixture_operations_attester_slashings.nim +++ b/tests/official/test_fixture_operations_attester_slashings.nim @@ -13,14 +13,15 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_block], + ../../beacon_chain/spec/state_transition_block, + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OpAttSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"attester_slashing"/"pyspec_tests" +const OpAttSlashingDir = SszTestsDir/const_preset/"altair"/"operations"/"attester_slashing"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_operations_block_header.nim b/tests/official/test_fixture_operations_block_header.nim index 96aff4850..4b38ddbad 100644 --- a/tests/official/test_fixture_operations_block_header.nim +++ b/tests/official/test_fixture_operations_block_header.nim @@ -13,14 +13,15 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_block, crypto], + ../../beacon_chain/spec/[state_transition_block, crypto], + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OpBlockHeaderDir = SszTestsDir/const_preset/"phase0"/"operations"/"block_header"/"pyspec_tests" +const OpBlockHeaderDir = SszTestsDir/const_preset/"altair"/"operations"/"block_header"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_operations_deposits.nim b/tests/official/test_fixture_operations_deposits.nim index 446146177..876005da0 100644 --- a/tests/official/test_fixture_operations_deposits.nim +++ b/tests/official/test_fixture_operations_deposits.nim @@ -13,14 +13,15 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, beaconstate, presets], + ../../beacon_chain/spec/[beaconstate, presets], + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OperationsDepositsDir = SszTestsDir/const_preset/"phase0"/"operations"/"deposit"/"pyspec_tests" +const OperationsDepositsDir = SszTestsDir/const_preset/"altair"/"operations"/"deposit"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_operations_proposer_slashings.nim b/tests/official/test_fixture_operations_proposer_slashings.nim index 44bb2689c..776feb117 100644 --- a/tests/official/test_fixture_operations_proposer_slashings.nim +++ b/tests/official/test_fixture_operations_proposer_slashings.nim @@ -13,14 +13,18 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_block], + ../../beacon_chain/spec/state_transition_block, + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OpProposerSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"proposer_slashing"/"pyspec_tests" +when isMainModule: + import chronicles # or some random compile error happens... + +const OpProposerSlashingDir = SszTestsDir/const_preset/"altair"/"operations"/"proposer_slashing"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_operations_sync_committee.nim b/tests/official/test_fixture_operations_sync_committee.nim new file mode 100644 index 000000000..d16a31ec1 --- /dev/null +++ b/tests/official/test_fixture_operations_sync_committee.nim @@ -0,0 +1,70 @@ +# beacon_chain +# Copyright (c) 2018-Present Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import + # Standard library + os, + # Utilities + stew/results, + # Beacon chain internals + ../../beacon_chain/spec/state_transition_block, + ../../beacon_chain/spec/datatypes/altair, + ../../beacon_chain/ssz, + # Test utilities + ../testutil, + ./fixtures_utils, + ../helpers/debug_state + +when isMainModule: + import chronicles # or some random compile error happens... + +const OpSyncCommitteeDir = SszTestsDir/const_preset/"altair"/"operations"/"sync_committee"/"pyspec_tests" + +proc runTest(identifier: string) = + # We wrap the tests in a proc to avoid running out of globals + # in the future: Nim supports up to 3500 globals + # but unittest with the macro/templates put everything as globals + # https://github.com/nim-lang/Nim/issues/12084#issue-486866402 + + let testDir = OpSyncCommitteeDir / identifier + + proc `testImpl_sync_committee _ identifier`() = + + var prefix: string + if existsFile(testDir/"post.ssz_snappy"): + prefix = "[Valid] " + else: + prefix = "[Invalid] " + + test prefix & identifier: + let syncAggregate = parseTest( + testDir/"sync_aggregate.ssz_snappy", SSZ, SyncAggregate) + var + preState = + newClone(parseTest(testDir/"pre.ssz_snappy", SSZ, BeaconState)) + cache = StateCache() + + if existsFile(testDir/"post.ssz_snappy"): + let + postState = + newClone(parseTest(testDir/"post.ssz_snappy", SSZ, BeaconState)) + done = process_sync_committee( + preState[], syncAggregate, cache).isOk + doAssert done, "Valid sync aggregate not processed" + check: preState[].hash_tree_root() == postState[].hash_tree_root() + reportDiff(preState, postState) + else: + let done = process_sync_committee(preState[], syncAggregate, cache).isOk + doAssert done == false, "We didn't expect this invalid proposer slashing to be processed." + + `testImpl_sync_committee _ identifier`() + +suite "Official - Operations - Sync Committee " & preset(): + for kind, path in walkDir(OpSyncCommitteeDir, true): + runTest(path) diff --git a/tests/official/test_fixture_operations_voluntary_exit.nim b/tests/official/test_fixture_operations_voluntary_exit.nim index 58eb4d3d5..ef70bd0bf 100644 --- a/tests/official/test_fixture_operations_voluntary_exit.nim +++ b/tests/official/test_fixture_operations_voluntary_exit.nim @@ -13,14 +13,15 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_block], + ../../beacon_chain/spec/state_transition_block, + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const OpVoluntaryExitDir = SszTestsDir/const_preset/"phase0"/"operations"/"voluntary_exit"/"pyspec_tests" +const OpVoluntaryExitDir = SszTestsDir/const_preset/"altair"/"operations"/"voluntary_exit"/"pyspec_tests" proc runTest(identifier: string) = # We wrap the tests in a proc to avoid running out of globals diff --git a/tests/official/test_fixture_sanity_blocks.nim b/tests/official/test_fixture_sanity_blocks.nim index 36dafad10..f575e873a 100644 --- a/tests/official/test_fixture_sanity_blocks.nim +++ b/tests/official/test_fixture_sanity_blocks.nim @@ -11,15 +11,16 @@ import # Standard library os, sequtils, chronicles, # Beacon chain internals - ../../beacon_chain/spec/[crypto, datatypes, state_transition, presets], + ../../beacon_chain/spec/[crypto, state_transition, presets], + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Test utilities ../testutil, ./fixtures_utils const - FinalityDir = SszTestsDir/const_preset/"phase0"/"finality"/"finality"/"pyspec_tests" - SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests" + FinalityDir = SszTestsDir/const_preset/"altair"/"finality"/"finality"/"pyspec_tests" + SanityBlocksDir = SszTestsDir/const_preset/"altair"/"sanity"/"blocks"/"pyspec_tests" proc runTest(testName, testDir, unitTestName: string) = # We wrap the tests in a proc to avoid running out of globals @@ -73,5 +74,6 @@ suite "Official - Sanity - Blocks " & preset(): runTest("Official - Sanity - Blocks", SanityBlocksDir, path) suite "Official - Finality " & preset(): + # these seem to only exist in minimal presets, both for phase0 and altair for kind, path in walkDir(FinalityDir, true): runTest("Official - Finality", FinalityDir, path) diff --git a/tests/official/test_fixture_sanity_slots.nim b/tests/official/test_fixture_sanity_slots.nim index f583dd61c..0b832812f 100644 --- a/tests/official/test_fixture_sanity_slots.nim +++ b/tests/official/test_fixture_sanity_slots.nim @@ -8,16 +8,18 @@ {.used.} import + chronicles, # Standard library os, strutils, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition], + ../../beacon_chain/spec/state_transition, + ../../beacon_chain/spec/datatypes/altair, # Test utilities ../testutil, ./fixtures_utils, ../helpers/debug_state -const SanitySlotsDir = SszTestsDir/const_preset/"phase0"/"sanity"/"slots"/"pyspec_tests" +const SanitySlotsDir = SszTestsDir/const_preset/"altair"/"sanity"/"slots"/"pyspec_tests" proc runTest(identifier: string) = let diff --git a/tests/official/test_fixture_ssz_consensus_objects.nim b/tests/official/test_fixture_ssz_consensus_objects.nim index e32fd6efa..52fe63b9e 100644 --- a/tests/official/test_fixture_ssz_consensus_objects.nim +++ b/tests/official/test_fixture_ssz_consensus_objects.nim @@ -12,7 +12,8 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/[crypto, datatypes, digest], + ../../beacon_chain/spec/[crypto, digest], + ../../beacon_chain/spec/datatypes/altair, ../../beacon_chain/ssz, # Status libraries snappy, @@ -25,7 +26,7 @@ import # ---------------------------------------------------------------- const - SSZDir = SszTestsDir/const_preset/"phase0"/"ssz_static" + SSZDir = SszTestsDir/const_preset/"altair"/"ssz_static" type SSZHashTreeRoot = object @@ -35,7 +36,7 @@ type # Some have a signing_root field signing_root {.defaultVal: "".}: string - # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/validator.md#eth1block + # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/altair/validator.md#eth1block Eth1Block* = object timestamp*: uint64 deposit_root*: Eth2Digest @@ -108,6 +109,7 @@ suite "Official - SSZ consensus objects " & preset(): of "BeaconBlockHeader": checkSSZ(BeaconBlockHeader, path, hash) of "BeaconState": checkSSZ(BeaconState, path, hash) of "Checkpoint": checkSSZ(Checkpoint, path, hash) + of "ContributionAndProof": checkSSZ(ContributionAndProof, path, hash) of "Deposit": checkSSZ(Deposit, path, hash) of "DepositData": checkSSZ(DepositData, path, hash) of "DepositMessage": checkSSZ(DepositMessage, path, hash) @@ -117,6 +119,8 @@ suite "Official - SSZ consensus objects " & preset(): of "ForkData": checkSSZ(ForkData, path, hash) of "HistoricalBatch": checkSSZ(HistoricalBatch, path, hash) of "IndexedAttestation": checkSSZ(IndexedAttestation, path, hash) + of "LightClientSnapshot": checkSSZ(LightClientSnapshot, path, hash) + of "LightClientUpdate": checkSSZ(LightClientUpdate, path, hash) of "PendingAttestation": checkSSZ(PendingAttestation, path, hash) of "ProposerSlashing": checkSSZ(ProposerSlashing, path, hash) of "SignedAggregateAndProof": @@ -124,9 +128,18 @@ suite "Official - SSZ consensus objects " & preset(): of "SignedBeaconBlock": checkSSZ(SignedBeaconBlock, path, hash) of "SignedBeaconBlockHeader": checkSSZ(SignedBeaconBlockHeader, path, hash) + of "SignedContributionAndProof": + checkSSZ(SignedContributionAndProof, path, hash) of "SignedVoluntaryExit": checkSSZ(SignedVoluntaryExit, path, hash) - of "SigningData": - checkSSZ(SigningData, path, hash) + of "SigningData": checkSSZ(SigningData, path, hash) + of "SyncAggregate": checkSSZ(SyncAggregate, path, hash) + of "SyncAggregatorSelectionData": + checkSSZ(SyncAggregatorSelectionData, path, hash) + of "SyncCommittee": checkSSZ(SyncCommittee, path, hash) + of "SyncCommitteeContribution": + checkSSZ(SyncCommitteeContribution, path, hash) + of "SyncCommitteeSignature": + checkSSZ(SyncCommitteeSignature, path, hash) of "Validator": checkSSZ(Validator, path, hash) of "VoluntaryExit": checkSSZ(VoluntaryExit, path, hash) else: diff --git a/tests/official/test_fixture_state_transition_epoch.nim b/tests/official/test_fixture_state_transition_epoch.nim index e5e6a2177..28492d440 100644 --- a/tests/official/test_fixture_state_transition_epoch.nim +++ b/tests/official/test_fixture_state_transition_epoch.nim @@ -11,7 +11,8 @@ import # Standard library os, strutils, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_epoch], + ../../beacon_chain/spec/state_transition_epoch, + ../../beacon_chain/spec/datatypes/altair, # Test utilities ../testutil, ./fixtures_utils, @@ -43,8 +44,10 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us # Justification & Finalization # --------------------------------------------------------------- -const JustificationFinalizationDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests" -runSuite(JustificationFinalizationDir, "Justification & Finalization", process_justification_and_finalization, useCache = false) +const JustificationFinalizationDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"justification_and_finalization"/"pyspec_tests" +when false: + # TODO + runSuite(JustificationFinalizationDir, "Justification & Finalization", process_justification_and_finalization, useCache = false) # Rewards & Penalties # --------------------------------------------------------------- @@ -54,32 +57,36 @@ runSuite(JustificationFinalizationDir, "Justification & Finalization", process_ # Registry updates # --------------------------------------------------------------- -const RegistryUpdatesDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"registry_updates"/"pyspec_tests" +const RegistryUpdatesDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"registry_updates"/"pyspec_tests" runSuite(RegistryUpdatesDir, "Registry updates", process_registry_updates, useCache = true) # Slashings # --------------------------------------------------------------- -const SlashingsDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"slashings"/"pyspec_tests" -runSuite(SlashingsDir, "Slashings", process_slashings, useCache = false) +const SlashingsDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"slashings"/"pyspec_tests" +when false: + # TODO needs totalbalance info + runSuite(SlashingsDir, "Slashings", process_slashings, useCache = false) # Final updates # --------------------------------------------------------------- -const Eth1DataResetDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"eth1_data_reset/"/"pyspec_tests" +const Eth1DataResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"eth1_data_reset/"/"pyspec_tests" runSuite(Eth1DataResetDir, "Eth1 data reset", process_eth1_data_reset, useCache = false) -const EffectiveBalanceUpdatesDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"effective_balance_updates"/"pyspec_tests" +const EffectiveBalanceUpdatesDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"effective_balance_updates"/"pyspec_tests" runSuite(EffectiveBalanceUpdatesDir, "Effective balance updates", process_effective_balance_updates, useCache = false) -const SlashingsResetDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"slashings_reset"/"pyspec_tests" +const SlashingsResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"slashings_reset"/"pyspec_tests" runSuite(SlashingsResetDir, "Slashings reset", process_slashings_reset, useCache = false) -const RandaoMixesResetDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"randao_mixes_reset"/"pyspec_tests" +const RandaoMixesResetDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"randao_mixes_reset"/"pyspec_tests" runSuite(RandaoMixesResetDir, "RANDAO mixes reset", process_randao_mixes_reset, useCache = false) -const HistoricalRootsUpdateDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"historical_roots_update"/"pyspec_tests" +const HistoricalRootsUpdateDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"historical_roots_update"/"pyspec_tests" runSuite(HistoricalRootsUpdateDir, "Historical roots update", process_historical_roots_update, useCache = false) -const ParticipationRecordsDir = SszTestsDir/const_preset/"phase0"/"epoch_processing"/"participation_record_updates"/"pyspec_tests" -runSuite(ParticipationRecordsDir, "Participation record updates", process_participation_record_updates, useCache = false) +const ParticipationRecordsDir = SszTestsDir/const_preset/"altair"/"epoch_processing"/"participation_record_updates"/"pyspec_tests" +when false: + # TODO needs totalbalance info + runSuite(ParticipationRecordsDir, "Participation record updates", process_participation_record_updates, useCache = false) diff --git a/tests/spec_epoch_processing/justification_finalization_helpers.nim b/tests/spec_epoch_processing/justification_finalization_helpers.nim index a1f79229b..8c8cf39bb 100644 --- a/tests/spec_epoch_processing/justification_finalization_helpers.nim +++ b/tests/spec_epoch_processing/justification_finalization_helpers.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2021 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -9,7 +9,7 @@ import # Standard library strformat, tables, # Specs - ../../beacon_chain/spec/[datatypes, state_transition_epoch, validator, helpers], + ../../beacon_chain/spec/[beaconstate, datatypes, validator, helpers], # Test helpers ../helpers/digest_helpers diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim index b32b0daa9..80a9d39ed 100644 --- a/tests/testblockutil.nim +++ b/tests/testblockutil.nim @@ -6,6 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import + chronicles, options, stew/endians2, ../beacon_chain/extras, ../beacon_chain/validators/validator_pool,