From 4140b3b9d9c58ea57bbab3299e69c8862d9063a1 Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Wed, 8 Jul 2020 22:36:26 +0200 Subject: [PATCH 1/3] update 29 spec refs to v0.12.1 --- beacon_chain/attestation_aggregation.nim | 4 +-- beacon_chain/block_pools/candidate_chains.nim | 2 +- beacon_chain/eth2_network.nim | 2 +- beacon_chain/spec/beaconstate.nim | 4 +-- beacon_chain/spec/helpers.nim | 12 +++---- beacon_chain/spec/presets/v0_12_1/mainnet.nim | 32 +++++++++---------- beacon_chain/spec/state_transition_block.nim | 4 +-- beacon_chain/validator_duties.nim | 2 +- beacon_chain/validator_pool.nim | 2 +- 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/beacon_chain/attestation_aggregation.nim b/beacon_chain/attestation_aggregation.nim index a8b59355d..f406c1cc4 100644 --- a/beacon_chain/attestation_aggregation.nim +++ b/beacon_chain/attestation_aggregation.nim @@ -18,7 +18,7 @@ import logScope: topics = "att_aggr" -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#aggregation-selection +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#aggregation-selection func is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: ValidatorSig, cache: var StateCache): bool = let @@ -60,7 +60,7 @@ proc aggregate_attestations*( index: index.uint64, beacon_block_root: get_block_root_at_slot(state, slot)) - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#construct-aggregate + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#construct-aggregate # TODO once EV goes in w/ refactoring of getAttestationsForBlock, pull out the getSlot version and use # it. This is incorrect. for attestation in getAttestationsForBlock(pool, state): diff --git a/beacon_chain/block_pools/candidate_chains.nim b/beacon_chain/block_pools/candidate_chains.nim index 3c7316128..c9b1d14aa 100644 --- a/beacon_chain/block_pools/candidate_chains.nim +++ b/beacon_chain/block_pools/candidate_chains.nim @@ -108,7 +108,7 @@ func getAncestorAt*(blck: BlockRef, slot: Slot): BlockRef = blck = blck.parent func get_ancestor*(blck: BlockRef, slot: Slot): BlockRef = - ## https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/fork-choice.md#get_ancestor + ## https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/fork-choice.md#get_ancestor ## Return ancestor at slot, or nil if queried block is older var blck = blck diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index e3d6ef51b..53de2af2b 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -189,7 +189,7 @@ const HandshakeTimeout = FaultOrError # Spec constants - # https://github.com/ethereum/eth2.0-specs/blob/dev/specs/networking/p2p-interface.md#eth-20-network-interaction-domains + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#eth2-network-interaction-domains MAX_CHUNK_SIZE* = 1 * 1024 * 1024 # bytes GOSSIP_MAX_SIZE* = 1 * 1024 * 1024 # bytes TTFB_TIMEOUT* = 5.seconds diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index d9ed9d431..76a1b915a 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -439,7 +439,7 @@ func is_valid_indexed_attestation*( true -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_attesting_indices +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_attesting_indices func get_attesting_indices*(state: BeaconState, data: AttestationData, bits: CommitteeValidatorsBits, @@ -691,7 +691,7 @@ func makeAttestationData*( doAssert slot.compute_epoch_at_slot == current_epoch - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.2/specs/phase0/validator.md#attestation-data + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#attestation-data AttestationData( slot: slot, index: committee_index, diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index ce33dae2a..0a9b123cd 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -56,7 +56,7 @@ func is_active_validator*(validator: Validator, epoch: Epoch): bool = ### Check if ``validator`` is active validator.activation_epoch <= epoch and epoch < validator.exit_epoch -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_active_validator_indices +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_active_validator_indices func get_active_validator_indices*(state: BeaconState, epoch: Epoch): seq[ValidatorIndex] = # Return the sequence of active validator indices at ``epoch``. @@ -85,13 +85,13 @@ func get_committee_count_at_slot*(state: BeaconState, slot: Slot): uint64 = # Otherwise, get_beacon_committee(...) cannot access some committees. doAssert (SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT).uint64 >= result -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_current_epoch +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_current_epoch func get_current_epoch*(state: BeaconState): Epoch = # Return the current epoch. doAssert state.slot >= GENESIS_SLOT, $state.slot compute_epoch_at_slot(state.slot) -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_randao_mix +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_randao_mix func get_randao_mix*(state: BeaconState, epoch: Epoch): Eth2Digest = ## Returns the randao mix at a recent ``epoch``. @@ -132,7 +132,7 @@ func int_to_bytes4*(x: uint64): array[4, byte] = result[2] = ((x shr 16) and 0xff).byte result[3] = ((x shr 24) and 0xff).byte -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_fork_data_root +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_fork_data_root func compute_fork_data_root(current_version: Version, genesis_validators_root: Eth2Digest): Eth2Digest = # Return the 32-byte fork data root for the ``current_version`` and @@ -144,7 +144,7 @@ func compute_fork_data_root(current_version: Version, genesis_validators_root: genesis_validators_root )) -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_fork_digest +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_fork_digest func compute_fork_digest*(current_version: Version, genesis_validators_root: Eth2Digest): ForkDigest = # Return the 4-byte fork digest for the ``current_version`` and @@ -194,7 +194,7 @@ func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest = ) hash_tree_root(domain_wrapped_object) -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_seed +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_seed func get_seed*(state: BeaconState, epoch: Epoch, domain_type: DomainType): Eth2Digest = # Return the seed at ``epoch``. diff --git a/beacon_chain/spec/presets/v0_12_1/mainnet.nim b/beacon_chain/spec/presets/v0_12_1/mainnet.nim index 9e29e93fe..8095be665 100644 --- a/beacon_chain/spec/presets/v0_12_1/mainnet.nim +++ b/beacon_chain/spec/presets/v0_12_1/mainnet.nim @@ -20,7 +20,7 @@ type const # Misc # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L6 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L6 MAX_COMMITTEES_PER_SLOT* {.intdefine.} = 64 @@ -50,7 +50,7 @@ const # Gwei values # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L58 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L58 MIN_DEPOSIT_AMOUNT* = 2'u64^0 * 10'u64^9 ##\ ## Minimum amounth of ETH that can be deposited in one call - deposits can @@ -67,13 +67,13 @@ const # Initial values # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L70 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L70 GENESIS_FORK_VERSION* = [0'u8, 0'u8, 0'u8, 0'u8] BLS_WITHDRAWAL_PREFIX* = 0'u8 # Time parameters # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L77 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L77 GENESIS_DELAY* {.intdefine.} = 172800 # 172800 seconds (2 days) SECONDS_PER_SLOT* {.intdefine.} = 12'u64 # Compile with -d:SECONDS_PER_SLOT=1 for 12x faster slots @@ -100,8 +100,6 @@ const MIN_SEED_LOOKAHEAD* = 1 ##\ ## epochs (~6.4 minutes) - SHARD_COMMITTEE_PERIOD* = 256 # epochs (~27 hours) - MAX_SEED_LOOKAHEAD* = 4 ##\ ## epochs (~25.6 minutes) @@ -114,6 +112,8 @@ const MIN_VALIDATOR_WITHDRAWABILITY_DELAY* = 2'u64^8 ##\ ## epochs (~27 hours) + SHARD_COMMITTEE_PERIOD* = 256 # epochs (~27 hours) + MAX_EPOCHS_PER_CROSSLINK* = 2'u64^6 ##\ ## epochs (~7 hours) @@ -122,7 +122,7 @@ const # State vector lengths # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L105 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L105 EPOCHS_PER_HISTORICAL_VECTOR* = 65536 ##\ ## epochs (~0.8 years) @@ -137,7 +137,7 @@ const # Reward and penalty quotients # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L117 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L117 BASE_REWARD_FACTOR* = 2'u64^6 WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9 PROPOSER_REWARD_QUOTIENT* = 2'u64^3 @@ -146,7 +146,7 @@ const # Max operations per block # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L131 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L131 MAX_PROPOSER_SLASHINGS* = 2^4 MAX_ATTESTER_SLASHINGS* = 2^1 MAX_ATTESTATIONS* = 2^7 @@ -155,12 +155,12 @@ const # Fork choice # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L32 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L32 SAFE_SLOTS_TO_UPDATE_JUSTIFIED* = 8 # 96 seconds # Validators # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L38 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L38 ETH1_FOLLOW_DISTANCE* {.intdefine.} = 1024 # blocks ~ 4 hours TARGET_AGGREGATORS_PER_COMMITTEE* = 16 # validators RANDOM_SUBNETS_PER_VALIDATOR* = 1 # subnet @@ -168,14 +168,14 @@ const SECONDS_PER_ETH1_BLOCK* {.intdefine.} = 14 # (estimate from Eth1 mainnet) # Phase 1: Upgrade from Phase 0 - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L161 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L161 PHASE_1_FORK_VERSION* = 1 PHASE_1_GENESIS_SLOT* = 32 # [STUB] INITIAL_ACTIVE_SHARDS* = 64 # Phase 1: General # --------------------------------------------------------------- - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L166 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L168 MAX_SHARDS* = 1024 ONLINE_PERIOD* = 8 # epochs (~51 min) LIGHT_CLIENT_COMMITTEE_SIZE* = 128 @@ -191,7 +191,7 @@ const # Phase 1: Custody game # --------------------------------------------------------------- # Time parameters - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L199 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L199 RANDAO_PENALTY_EPOCHS* = 2 # epochs (12.8 minutes) EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS* = 16384 # epochs (~73 days) EPOCHS_PER_CUSTODY_PERIOD* = 2048 # epochs (~9 days) @@ -199,12 +199,12 @@ const MAX_REVEAL_LATENESS_DECREMENT* = 128 # epochs (~14 hours) # Max operations - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L211 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L211 MAX_CUSTODY_KEY_REVEALS* = 256 MAX_EARLY_DERIVED_SECRET_REVEALS* = 1 MAX_CUSTODY_SLASHINGS* = 1 # Reward and penalty quotients - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/configs/mainnet.yaml#L217 + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/configs/mainnet.yaml#L217 EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE* = 2 MINOR_REWARD_QUOTIENT* = 256 diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index af32e086a..260ee3f26 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -225,7 +225,7 @@ proc process_attester_slashing*( return err("Attester slashing: Trying to slash participant(s) twice") ok() -# https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#voluntary-exits +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#voluntary-exits proc process_voluntary_exit*( state: var BeaconState, signed_voluntary_exit: SignedVoluntaryExit, @@ -243,7 +243,7 @@ proc process_voluntary_exit*( if not is_active_validator(validator, get_current_epoch(state)): return err("Exit: validator not active") - # Verify the validator has not yet exited + # Verify exit has not been initiated if validator.exit_epoch != FAR_FUTURE_EPOCH: return err("Exit: validator has exited") diff --git a/beacon_chain/validator_duties.nim b/beacon_chain/validator_duties.nim index c20e34c01..4174adbe0 100644 --- a/beacon_chain/validator_duties.nim +++ b/beacon_chain/validator_duties.nim @@ -323,7 +323,7 @@ proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) = # We need to run attestations exactly for the slot that we're attesting to. # In case blocks went missing, this means advancing past the latest block # using empty slots as fillers. - # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/validator/0_beacon-chain-validator.md#validator-assignments + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#validator-assignments # TODO we could cache the validator assignment since it's valid for the entire # epoch since it doesn't change, but that has to be weighed against # the complexity of handling forks correctly - instead, we use an adapted diff --git a/beacon_chain/validator_pool.nim b/beacon_chain/validator_pool.nim index 43ece331c..cd0df2216 100644 --- a/beacon_chain/validator_pool.nim +++ b/beacon_chain/validator_pool.nim @@ -24,7 +24,7 @@ func getValidator*(pool: ValidatorPool, validatorKey: ValidatorPubKey): AttachedValidator = pool.validators.getOrDefault(validatorKey) -# TODO: Honest validator - https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md +# TODO: Honest validator - https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md proc signBlockProposal*(v: AttachedValidator, fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, blockRoot: Eth2Digest): Future[ValidatorSig] {.async.} = From 3cdae9f6be3132853e589b6431d0be06cfb73e90 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Thu, 9 Jul 2020 11:29:32 +0200 Subject: [PATCH 2/3] Dual headed fork choice [Revolution] (#1238) * Dual headed fork choice * fix finalizedEpoch not moving * reduce fork choice verbosity * Add failing tests due to pruning * Properly handle duplicate blocks in sync * test_block_pool also add a test for duplicate blocks * comments addressing review * Fix fork choice v2, was missing integrating block proposed * remove a spurious debug writeStackTrace * update block_sim * Use OrderedTable to ensure that we always load parents before children in fork choice * Load the DAG data in fork choice at init if there is some (can sync witti) * Cluster of quarantined blocks were not properly added to the fork choice * Workaround async gcsafe warnings * Update blockpoool tests * Do the callback before clearing the quarantine * Revert OrderedTable, implement topological sort of DAG, allow forkChoice to be initialized from arbitrary finalized heads * Make it work with latest devel - Altona readyness * Add a recovery mechanism when forkchoice desyncs with blockpool * add the current problematic node to the stack * Fix rebase indentation bug (but still producing invalid block) * Fix cache at epoch boundaries and lateBlock addition --- AllTests-mainnet.md | 6 +- beacon_chain/attestation_pool.nim | 194 +++++++++++++++++- beacon_chain/beacon_node.nim | 18 +- beacon_chain/beacon_node_common.nim | 3 + beacon_chain/beacon_node_types.nim | 6 +- beacon_chain/block_pool.nim | 24 ++- .../block_pools/block_pools_types.nim | 10 +- beacon_chain/block_pools/candidate_chains.nim | 24 ++- beacon_chain/block_pools/clearance.nim | 64 ++++-- beacon_chain/fork_choice/fork_choice.nim | 8 +- beacon_chain/fork_choice/proto_array.nim | 20 ++ beacon_chain/spec/beaconstate.nim | 4 +- beacon_chain/validator_duties.nim | 7 +- research/block_sim.nim | 10 +- tests/test_attestation_pool.nim | 176 +++++++++++++--- tests/test_block_pool.nim | 185 ++++++++++------- 16 files changed, 601 insertions(+), 158 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index b6e5b2079..73236ef24 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -9,8 +9,10 @@ AllTests-mainnet + Can add and retrieve simple attestation [Preset: mainnet] OK + Fork choice returns block with attestation OK + Fork choice returns latest block with no attestations OK ++ Trying to add a block twice tags the second as an error OK ++ Trying to add a duplicate block from an old pruned epoch is tagged as an error OK ``` -OK: 7/7 Fail: 0/7 Skip: 0/7 +OK: 9/9 Fail: 0/9 Skip: 0/9 ## Beacon chain DB [Preset: mainnet] ```diff + empty database [Preset: mainnet] OK @@ -32,7 +34,7 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 1/1 Fail: 0/1 Skip: 0/1 ## Block pool processing [Preset: mainnet] ```diff -+ Can add same block twice [Preset: mainnet] OK ++ Adding the same block twice returns a Duplicate error [Preset: mainnet] OK + Reverse order block add & get [Preset: mainnet] OK + Simple block add&get [Preset: mainnet] OK + getRef returns nil for missing blocks OK diff --git a/beacon_chain/attestation_pool.nim b/beacon_chain/attestation_pool.nim index 171cadcf9..e6de5b04f 100644 --- a/beacon_chain/attestation_pool.nim +++ b/beacon_chain/attestation_pool.nim @@ -8,24 +8,87 @@ {.push raises: [Defect].} import - deques, sequtils, tables, options, + # Standard libraries + deques, sequtils, tables, options, algorithm, + # Status libraries chronicles, stew/[byteutils], json_serialization/std/sets, + # Internal ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], - ./extras, ./block_pool, ./block_pools/candidate_chains, ./beacon_node_types + ./extras, ./block_pool, ./block_pools/candidate_chains, ./beacon_node_types, + ./fork_choice/fork_choice logScope: topics = "attpool" -func init*(T: type AttestationPool, blockPool: BlockPool): T = +proc init*(T: type AttestationPool, blockPool: BlockPool): T = ## Initialize an AttestationPool from the blockPool `headState` ## The `finalized_root` works around the finalized_checkpoint of the genesis block ## holding a zero_root. # TODO blockPool is only used when resolving orphaned attestations - it should # probably be removed as a dependency of AttestationPool (or some other # smart refactoring) + + # TODO: Return Value Optimization + + # TODO: In tests, on blockpool.init the finalized root + # from the `headState` and `justifiedState` is zero + var forkChoice = initForkChoice( + finalized_block_slot = default(Slot), # This is unnecessary for fork choice but may help external components for example logging/debugging + finalized_block_state_root = default(Eth2Digest), # This is unnecessary for fork choice but may help external components for example logging/debugging + justified_epoch = blockPool.headState.data.data.current_justified_checkpoint.epoch, + finalized_epoch = blockPool.headState.data.data.finalized_checkpoint.epoch, + # We should use the checkpoint, but at genesis the headState finalized checkpoint is 0x0000...0000 + # finalized_root = blockPool.headState.data.data.finalized_checkpoint.root + finalized_root = blockPool.finalizedHead.blck.root + ).get() + + # Load all blocks since finalized head - TODO a proper test + for blck in blockPool.dag.topoSortedSinceLastFinalization(): + if blck.root == blockPool.finalizedHead.blck.root: + continue + + # BlockRef + # should ideally contain the justified_epoch and finalized_epoch + # so that we can pass them directly to `process_block` without having to + # redo "updateStateData" + # + # In any case, `updateStateData` should shortcut + # to `getStateDataCached` + + updateStateData( + blockPool, + blockPool.tmpState, + BlockSlot(blck: blck, slot: blck.slot) + ) + + debug "Preloading fork choice with block", + block_root = shortlog(blck.root), + parent_root = shortlog(blck.parent.root), + justified_epoch = $blockPool.tmpState.data.data.current_justified_checkpoint.epoch, + finalized_epoch = $blockPool.tmpState.data.data.finalized_checkpoint.epoch, + slot = $blck.slot + + let status = forkChoice.process_block( + block_root = blck.root, + parent_root = blck.parent.root, + justified_epoch = blockPool.tmpState.data.data.current_justified_checkpoint.epoch, + finalized_epoch = blockPool.tmpState.data.data.finalized_checkpoint.epoch, + # Unused in fork choice - i.e. for logging or caching extra metadata + slot = blck.slot, + state_root = default(Eth2Digest) + ) + + doAssert status.isOk(), "Error in preloading the fork choice: " & $status.error + + info "Fork choice initialized", + justified_epoch = $blockPool.headState.data.data.current_justified_checkpoint.epoch, + finalized_epoch = $blockPool.headState.data.data.finalized_checkpoint.epoch, + finalized_root = shortlog(blockPool.finalizedHead.blck.root) + T( mapSlotsToAttestations: initDeque[AttestationsSeen](), blockPool: blockPool, unresolved: initTable[Eth2Digest, UnresolvedAttestation](), + forkChoice_v2: forkChoice ) proc combine*(tgt: var Attestation, src: Attestation, flags: UpdateFlags) = @@ -107,13 +170,21 @@ proc slotIndex( func updateLatestVotes( pool: var AttestationPool, state: BeaconState, attestationSlot: Slot, participants: seq[ValidatorIndex], blck: BlockRef) = + + # ForkChoice v2 + let target_epoch = compute_epoch_at_slot(attestationSlot) + for validator in participants: + # ForkChoice v1 let pubKey = state.validators[validator].pubkey current = pool.latestAttestations.getOrDefault(pubKey) if current.isNil or current.slot < attestationSlot: pool.latestAttestations[pubKey] = blck + # ForkChoice v2 + pool.forkChoice_v2.process_attestation(validator, blck.root, target_epoch) + func get_attesting_indices_seq(state: BeaconState, attestation_data: AttestationData, bits: CommitteeValidatorsBits, @@ -254,7 +325,7 @@ proc addResolved(pool: var AttestationPool, blck: BlockRef, attestation: Attesta blockSlot = shortLog(blck.slot), cat = "filtering" -proc add*(pool: var AttestationPool, attestation: Attestation) = +proc addAttestation*(pool: var AttestationPool, attestation: Attestation) = ## Add a verified attestation to the fork choice context logScope: pcs = "atp_add_attestation" @@ -269,6 +340,68 @@ proc add*(pool: var AttestationPool, attestation: Attestation) = pool.addResolved(blck, attestation) +proc addForkChoice_v2*(pool: var AttestationPool, blck: BlockRef) = + ## Add a verified block to the fork choice context + ## The current justifiedState of the block pool is used as reference + + # TODO: add(BlockPool, blockRoot: Eth2Digest, SignedBeaconBlock): BlockRef + # should ideally return the justified_epoch and finalized_epoch + # so that we can pass them directly to this proc without having to + # redo "updateStateData" + # + # In any case, `updateStateData` should shortcut + # to `getStateDataCached` + + var state: Result[void, string] + # A stack of block to add in case recovery is needed + var blockStack: seq[BlockSlot] + var current = BlockSlot(blck: blck, slot: blck.slot) + + while true: # The while loop should not be needed but it seems a block addition + # scenario is unaccounted for + updateStateData( + pool.blockPool, + pool.blockPool.tmpState, + current + ) + + let blockData = pool.blockPool.get(current.blck) + state = pool.forkChoice_v2.process_block( + slot = current.blck.slot, + block_root = current.blck.root, + parent_root = if not current.blck.parent.isNil: current.blck.parent.root else: default(Eth2Digest), + state_root = default(Eth2Digest), # This is unnecessary for fork choice but may help external components + justified_epoch = pool.blockPool.tmpState.data.data.current_justified_checkpoint.epoch, + finalized_epoch = pool.blockPool.tmpState.data.data.finalized_checkpoint.epoch, + ) + + # This should not happen and might lead to unresponsive networking while processing occurs + if state.isErr: + # TODO investigate, potential sources: + # - Pruning + # - Quarantine adding multiple blocks at once + # - Own block proposal + error "Desync between fork_choice and blockpool services, trying to recover.", + msg = state.error, + blck = shortlog(current.blck), + parent = shortlog(current.blck.parent), + finalizedHead = shortLog(pool.blockPool.finalizedHead), + justifiedHead = shortLog(pool.blockPool.head.justified), + head = shortLog(pool.blockPool.head.blck) + blockStack.add(current) + current = BlockSlot(blck: blck.parent, slot: blck.parent.slot) + elif blockStack.len == 0: + break + else: + info "Re-added missing or pruned block to fork choice", + msg = state.error, + blck = shortlog(current.blck), + parent = shortlog(current.blck.parent), + finalizedHead = shortLog(pool.blockPool.finalizedHead), + justifiedHead = shortLog(pool.blockPool.head.justified), + head = shortLog(pool.blockPool.head.blck) + current = blockStack.pop() + proc getAttestationsForSlot*(pool: AttestationPool, newBlockSlot: Slot): Option[AttestationsSeen] = if newBlockSlot < (GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY): @@ -403,7 +536,10 @@ proc resolve*(pool: var AttestationPool) = for a in resolved: pool.addResolved(a.blck, a.attestation) -func latestAttestation*( +# Fork choice v1 +# --------------------------------------------------------------- + +func latestAttestation( pool: AttestationPool, pubKey: ValidatorPubKey): BlockRef = pool.latestAttestations.getOrDefault(pubKey) @@ -411,7 +547,7 @@ func latestAttestation*( # The structure of this code differs from the spec since we use a different # strategy for storing states and justification points - it should nonetheless # be close in terms of functionality. -func lmdGhost*( +func lmdGhost( pool: AttestationPool, start_state: BeaconState, start_block: BlockRef): BlockRef = # TODO: a Fenwick Tree datastructure to keep track of cumulated votes @@ -462,7 +598,7 @@ func lmdGhost*( winCount = candCount head = winner -proc selectHead*(pool: AttestationPool): BlockRef = +proc selectHead_v1(pool: AttestationPool): BlockRef = let justifiedHead = pool.blockPool.latestJustifiedBlock() @@ -470,3 +606,47 @@ proc selectHead*(pool: AttestationPool): BlockRef = lmdGhost(pool, pool.blockPool.justifiedState.data.data, justifiedHead.blck) newHead + +# Fork choice v2 +# --------------------------------------------------------------- + +func getAttesterBalances(state: StateData): seq[Gwei] {.noInit.}= + ## Get the balances from a state + result.newSeq(state.data.data.validators.len) # zero-init + + let epoch = state.data.data.slot.compute_epoch_at_slot() + + for i in 0 ..< result.len: + # All non-active validators have a 0 balance + template validator: Validator = state.data.data.validators[i] + if validator.is_active_validator(epoch): + result[i] = validator.effective_balance + +proc selectHead_v2(pool: var AttestationPool): BlockRef = + let attesterBalances = pool.blockPool.justifiedState.getAttesterBalances() + + let newHead = pool.forkChoice_v2.find_head( + justified_epoch = pool.blockPool.justifiedState.data.data.slot.compute_epoch_at_slot(), + justified_root = pool.blockPool.head.justified.blck.root, + finalized_epoch = pool.blockPool.headState.data.data.finalized_checkpoint.epoch, + justified_state_balances = attesterBalances + ).get() + + pool.blockPool.getRef(newHead) + +proc pruneBefore*(pool: var AttestationPool, finalizedhead: BlockSlot) = + pool.forkChoice_v2.maybe_prune(finalizedHead.blck.root).get() + +# Dual-Headed Fork choice +# --------------------------------------------------------------- + +proc selectHead*(pool: var AttestationPool): BlockRef = + let head_v1 = pool.selectHead_v1() + let head_v2 = pool.selectHead_v2() + + if head_v1 != head_v2: + error "Fork choice engines in disagreement, using block from v1.", + v1_block = shortlog(head_v1), + v2_block = shortlog(head_v2) + + return head_v1 diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index ca3987e07..7723af3ba 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -305,7 +305,7 @@ proc onAttestation(node: BeaconNode, attestation: Attestation) = attestationSlot = attestation.data.slot, headSlot = head.blck.slot return - node.attestationPool.add(attestation) + node.attestationPool.addAttestation(attestation) proc dumpBlock[T]( node: BeaconNode, signedBlock: SignedBeaconBlock, @@ -333,10 +333,17 @@ proc storeBlock( pcs = "receive_block" beacon_blocks_received.inc() - let blck = node.blockPool.add(blockRoot, signedBlock) + + {.gcsafe.}: # TODO: fork choice and blockpool should sync via messages instead of callbacks + let blck = node.blockPool.addRawBlock(blockRoot, signedBlock) do (validBlock: BlockRef): + # Callback add to fork choice if valid + node.attestationPool.addForkChoice_v2(validBlock) node.dumpBlock(signedBlock, blck) + # There can be a scenario where we receive a block we already received. + # However this block was before the last finalized epoch and so its parent + # was pruned from the ForkChoice. if blck.isErr: return err(blck.error) @@ -347,7 +354,7 @@ proc storeBlock( attestation = shortLog(attestation), cat = "consensus" # Tag "consensus|attestation"? - node.attestationPool.add(attestation) + node.attestationPool.addAttestation(attestation) ok() proc onBeaconBlock(node: BeaconNode, signedBlock: SignedBeaconBlock) = @@ -565,7 +572,10 @@ proc runForwardSyncLoop(node: BeaconNode) {.async.} = # We going to ignore `BlockError.Unviable` errors because we have working # backward sync and it can happens that we can perform overlapping # requests. - if res.isErr and res.error != BlockError.Unviable: + # For the same reason we ignore Duplicate blocks as if they are duplicate + # from before the current finalized epoch, we can drop them + # (and they may have no parents anymore in the fork choice if it was pruned) + if res.isErr and res.error notin {BlockError.Unviable, BlockError.Old, BLockError.Duplicate}: return res discard node.updateHead() diff --git a/beacon_chain/beacon_node_common.nim b/beacon_chain/beacon_node_common.nim index fbeed6b14..f31654402 100644 --- a/beacon_chain/beacon_node_common.nim +++ b/beacon_chain/beacon_node_common.nim @@ -69,6 +69,9 @@ proc updateHead*(node: BeaconNode): BlockRef = node.blockPool.updateHead(newHead) beacon_head_root.set newHead.root.toGaugeValue + # Cleanup the fork choice v2 if we have a finalized head + node.attestationPool.pruneBefore(node.blockPool.finalizedHead) + newHead template findIt*(s: openarray, predicate: untyped): int64 = diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index 4015c3d60..03cf4767d 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -5,7 +5,8 @@ import stew/endians2, spec/[datatypes, crypto, digest], block_pools/block_pools_types, - block_pool # TODO: refactoring compat shim + block_pool, # TODO: refactoring compat shim + fork_choice/fork_choice_types export block_pools_types @@ -74,6 +75,9 @@ type latestAttestations*: Table[ValidatorPubKey, BlockRef] ##\ ## Map that keeps track of the most recent vote of each attester - see ## fork_choice + forkChoice_v2*: ForkChoice ##\ + ## The alternative fork choice "proto_array" that will ultimately + ## replace the original one # ############################################# # diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim index 227cb2393..0f4261548 100644 --- a/beacon_chain/block_pool.nim +++ b/beacon_chain/block_pool.nim @@ -26,7 +26,7 @@ type BlockPools* = object # TODO: Rename BlockPools quarantine: Quarantine - dag: CandidateChains + dag*: CandidateChains BlockPool* = BlockPools @@ -53,9 +53,19 @@ template head*(pool: BlockPool): Head = template finalizedHead*(pool: BlockPool): BlockSlot = pool.dag.finalizedHead -proc add*(pool: var BlockPool, blockRoot: Eth2Digest, - signedBlock: SignedBeaconBlock): Result[BlockRef, BlockError] {.gcsafe.} = - add(pool.dag, pool.quarantine, blockRoot, signedBlock) +proc addRawBlock*(pool: var BlockPool, blockRoot: Eth2Digest, + signedBlock: SignedBeaconBlock, + callback: proc(blck: BlockRef) + ): Result[BlockRef, BlockError] = + ## Add a raw block to the blockpool + ## Trigger "callback" on success + ## Adding a rawblock might unlock a consequent amount of blocks in quarantine + # TODO: `addRawBlock` is accumulating significant cruft + # and is in dire need of refactoring + # - the ugly `inAdd` field + # - the callback + # - callback may be problematic as it's called in async validator duties + result = addRawBlock(pool.dag, pool.quarantine, blockRoot, signedBlock, callback) export parent # func parent*(bs: BlockSlot): BlockSlot export isAncestorOf # func isAncestorOf*(a, b: BlockRef): bool @@ -68,7 +78,13 @@ proc init*(T: type BlockPools, db: BeaconChainDB, updateFlags: UpdateFlags = {}): BlockPools = result.dag = init(CandidateChains, db, updateFlags) +func addFlags*(pool: BlockPool, flags: UpdateFlags) = + ## Add a flag to the block processing + ## This is destined for testing to add skipBLSValidation flag + pool.dag.updateFlags.incl flags + export init # func init*(T: type BlockRef, root: Eth2Digest, blck: BeaconBlock): BlockRef +export addFlags func getRef*(pool: BlockPool, root: Eth2Digest): BlockRef = ## Retrieve a resolved block reference, if available diff --git a/beacon_chain/block_pools/block_pools_types.nim b/beacon_chain/block_pools/block_pools_types.nim index 8cb912cf5..4f2c05257 100644 --- a/beacon_chain/block_pools/block_pools_types.nim +++ b/beacon_chain/block_pools/block_pools_types.nim @@ -6,8 +6,11 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - deques, tables, + # Standard library + deques, tables, hashes, + # Status libraries stew/[endians2, byteutils], chronicles, + # Internals ../spec/[datatypes, crypto, digest], ../beacon_chain_db, ../extras @@ -36,6 +39,8 @@ type Invalid ##\ ## Block is broken / doesn't apply cleanly - whoever sent it is fishy (or ## we're buggy) + Old + Duplicate Quarantine* = object ## Keeps track of unsafe blocks coming from the network @@ -188,3 +193,6 @@ proc shortLog*(v: BlockRef): string = chronicles.formatIt BlockSlot: shortLog(it) chronicles.formatIt BlockRef: shortLog(it) + +func hash*(blockRef: BlockRef): Hash = + hash(blockRef.root) diff --git a/beacon_chain/block_pools/candidate_chains.nim b/beacon_chain/block_pools/candidate_chains.nim index c9b1d14aa..d27ecc396 100644 --- a/beacon_chain/block_pools/candidate_chains.nim +++ b/beacon_chain/block_pools/candidate_chains.nim @@ -8,8 +8,11 @@ {.push raises: [Defect].} import - chronicles, options, sequtils, tables, + # Standard libraries + chronicles, options, sequtils, tables, sets, + # Status libraries metrics, + # Internals ../ssz/merkleization, ../beacon_chain_db, ../extras, ../spec/[crypto, datatypes, digest, helpers, validator, state_transition], block_pools_types @@ -305,6 +308,25 @@ proc init*(T: type CandidateChains, db: BeaconChainDB, res +iterator topoSortedSinceLastFinalization*(dag: CandidateChains): BlockRef = + ## Iterate on the dag in topological order + # TODO: this uses "children" for simplicity + # but "children" should be deleted as it introduces cycles + # that causes significant overhead at least and leaks at worst + # for the GC. + # This is not perf critical, it is only used to bootstrap the fork choice. + var visited: HashSet[BlockRef] + var stack: seq[BlockRef] + + stack.add dag.finalizedHead.blck + + while stack.len != 0: + let node = stack.pop() + if node notin visited: + visited.incl node + stack.add node.children + yield node + proc getState( dag: CandidateChains, db: BeaconChainDB, stateRoot: Eth2Digest, blck: BlockRef, output: var StateData): bool = diff --git a/beacon_chain/block_pools/clearance.nim b/beacon_chain/block_pools/clearance.nim index ce43ed18d..15bf2593e 100644 --- a/beacon_chain/block_pools/clearance.nim +++ b/beacon_chain/block_pools/clearance.nim @@ -34,15 +34,24 @@ func getOrResolve*(dag: CandidateChains, quarantine: var Quarantine, root: Eth2D if result.isNil: quarantine.missing[root] = MissingBlock() -proc add*( - dag: var CandidateChains, quarantine: var Quarantine, - blockRoot: Eth2Digest, - signedBlock: SignedBeaconBlock): Result[BlockRef, BlockError] {.gcsafe.} +proc addRawBlock*( + dag: var CandidateChains, quarantine: var Quarantine, + blockRoot: Eth2Digest, + signedBlock: SignedBeaconBlock, + callback: proc(blck: BlockRef) + ): Result[BlockRef, BlockError] proc addResolvedBlock( - dag: var CandidateChains, quarantine: var Quarantine, - state: BeaconState, blockRoot: Eth2Digest, - signedBlock: SignedBeaconBlock, parent: BlockRef): BlockRef = + dag: var CandidateChains, quarantine: var Quarantine, + state: BeaconState, blockRoot: Eth2Digest, + signedBlock: SignedBeaconBlock, parent: BlockRef, + callback: proc(blck: BlockRef) + ): BlockRef = + # TODO: `addResolvedBlock` is accumulating significant cruft + # and is in dire need of refactoring + # - the ugly `quarantine.inAdd` field + # - the callback + # - callback may be problematic as it's called in async validator duties logScope: pcs = "block_resolution" doAssert state.slot == signedBlock.message.slot, "state must match block" @@ -86,6 +95,9 @@ proc addResolvedBlock( heads = dag.heads.len(), cat = "filtering" + # This MUST be added before the quarantine + callback(blockRef) + # Now that we have the new block, we should see if any of the previously # unresolved blocks magically become resolved # TODO there are more efficient ways of doing this that don't risk @@ -94,6 +106,7 @@ proc addResolvedBlock( # blocks being synced, there's a stack overflow as `add` gets called # for the whole chain of blocks. Instead we use this ugly field in `dag` # which could be avoided by refactoring the code + # TODO unit test the logic, in particular interaction with fork choice block parents if not quarantine.inAdd: quarantine.inAdd = true defer: quarantine.inAdd = false @@ -101,20 +114,26 @@ proc addResolvedBlock( while keepGoing: let retries = quarantine.orphans for k, v in retries: - discard add(dag, quarantine, k, v) + discard addRawBlock(dag, quarantine, k, v, callback) # Keep going for as long as the pending dag is shrinking # TODO inefficient! so what? keepGoing = quarantine.orphans.len < retries.len + blockRef -proc add*( - dag: var CandidateChains, quarantine: var Quarantine, - blockRoot: Eth2Digest, - signedBlock: SignedBeaconBlock): Result[BlockRef, BlockError] {.gcsafe.} = +proc addRawBlock*( + dag: var CandidateChains, quarantine: var Quarantine, + blockRoot: Eth2Digest, + signedBlock: SignedBeaconBlock, + callback: proc(blck: BlockRef) + ): Result[BlockRef, BlockError] = ## return the block, if resolved... - ## the state parameter may be updated to include the given block, if - ## everything checks out - # TODO reevaluate passing the state in like this + + # TODO: `addRawBlock` is accumulating significant cruft + # and is in dire need of refactoring + # - the ugly `quarantine.inAdd` field + # - the callback + # - callback may be problematic as it's called in async validator duties # TODO: to facilitate adding the block to the attestation pool # this should also return justified and finalized epoch corresponding @@ -124,18 +143,22 @@ proc add*( let blck = signedBlock.message - doAssert blockRoot == hash_tree_root(blck) + doAssert blockRoot == hash_tree_root(blck), "blockRoot: 0x" & shortLog(blockRoot) & ", signedBlock: 0x" & shortLog(hash_tree_root(blck)) logScope: pcs = "block_addition" # Already seen this block?? - dag.blocks.withValue(blockRoot, blockRef): + if blockRoot in dag.blocks: debug "Block already exists", blck = shortLog(blck), blockRoot = shortLog(blockRoot), cat = "filtering" - return ok blockRef[] + # There can be a scenario where we receive a block we already received. + # However this block was before the last finalized epoch and so its parent + # was pruned from the ForkChoice. Trying to add it again, even if the fork choice + # supports duplicate will lead to a crash. + return err Duplicate quarantine.missing.del(blockRoot) @@ -220,9 +243,12 @@ proc add*( # the BlockRef first! dag.tmpState.blck = addResolvedBlock( dag, quarantine, - dag.tmpState.data.data, blockRoot, signedBlock, parent) + dag.tmpState.data.data, blockRoot, signedBlock, parent, + callback + ) dag.putState(dag.tmpState.data, dag.tmpState.blck) + callback(dag.tmpState.blck) return ok dag.tmpState.blck # TODO already checked hash though? main reason to keep this is because diff --git a/beacon_chain/fork_choice/fork_choice.nim b/beacon_chain/fork_choice/fork_choice.nim index c9fc6db77..7b4f828bb 100644 --- a/beacon_chain/fork_choice/fork_choice.nim +++ b/beacon_chain/fork_choice/fork_choice.nim @@ -117,7 +117,7 @@ func process_attestation*( vote.next_epoch = target_epoch {.noSideEffect.}: - info "Integrating vote in fork choice", + trace "Integrating vote in fork choice", validator_index = $validator_index, new_vote = shortlog(vote) else: @@ -129,7 +129,7 @@ func process_attestation*( ignored_block_root = shortlog(block_root), ignored_target_epoch = $target_epoch else: - info "Ignoring double-vote for fork choice", + trace "Ignoring double-vote for fork choice", validator_index = $validator_index, current_vote = shortlog(vote), ignored_block_root = shortlog(block_root), @@ -159,7 +159,7 @@ func process_block*( return err("process_block_error: " & $err) {.noSideEffect.}: - info "Integrating block in fork choice", + trace "Integrating block in fork choice", block_root = $shortlog(block_root), parent_root = $shortlog(parent_root), justified_epoch = $justified_epoch, @@ -205,7 +205,7 @@ func find_head*( return err("find_head failed: " & $ghost_err) {.noSideEffect.}: - info "Fork choice requested", + debug "Fork choice requested", justified_epoch = $justified_epoch, justified_root = shortlog(justified_root), finalized_epoch = $finalized_epoch, diff --git a/beacon_chain/fork_choice/proto_array.nim b/beacon_chain/fork_choice/proto_array.nim index 79276bfee..8e8693405 100644 --- a/beacon_chain/fork_choice/proto_array.nim +++ b/beacon_chain/fork_choice/proto_array.nim @@ -10,11 +10,17 @@ import # Standard library std/tables, std/options, std/typetraits, + # Status libraries + chronicles, # Internal ../spec/[datatypes, digest], # Fork choice ./fork_choice_types +logScope: + topics = "fork_choice" + cat = "fork_choice" + # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/fork-choice.md # This is a port of https://github.com/sigp/lighthouse/pull/804 # which is a port of "Proto-Array": https://github.com/protolambda/lmd-ghost @@ -176,6 +182,14 @@ func on_block*( # Genesis (but Genesis might not be default(Eth2Digest)) parent_index = none(int) elif parent notin self.indices: + {.noSideEffect.}: + error "Trying to add block with unknown parent", + child_root = shortLog(root), + parent_root = shortLog(parent), + justified_epoch = $justified_epoch, + finalized_epoch = $finalized_epoch, + slot_optional = $slot + return ForkChoiceError( kind: fcErrUnknownParent, child_root: root, @@ -297,6 +311,12 @@ func maybe_prune*( kind: fcErrInvalidNodeIndex, index: finalized_index ) + + {.noSideEffect.}: + debug "Pruning blocks from fork choice", + finalizedRoot = shortlog(finalized_root), + pcs = "prune" + for node_index in 0 ..< finalized_index: self.indices.del(self.nodes[node_index].root) diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 76a1b915a..c5e05b969 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -689,7 +689,9 @@ func makeAttestationData*( if start_slot == state.slot: beacon_block_root else: get_block_root_at_slot(state, start_slot) - doAssert slot.compute_epoch_at_slot == current_epoch + doAssert slot.compute_epoch_at_slot == current_epoch, + "Computed epoch was " & $slot.compute_epoch_at_slot & + " while the state current_epoch was " & $current_epoch # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#attestation-data AttestationData( diff --git a/beacon_chain/validator_duties.nim b/beacon_chain/validator_duties.nim index 4174adbe0..145e7bfee 100644 --- a/beacon_chain/validator_duties.nim +++ b/beacon_chain/validator_duties.nim @@ -223,7 +223,12 @@ proc proposeSignedBlock*(node: BeaconNode, validator: AttachedValidator, newBlock: SignedBeaconBlock, blockRoot: Eth2Digest): Future[BlockRef] {.async.} = - let newBlockRef = node.blockPool.add(blockRoot, newBlock) + + {.gcsafe.}: # TODO: fork choice and blockpool should sync via messages instead of callbacks + let newBlockRef = node.blockPool.addRawBlock(blockRoot, newBlock) do (validBlock: BlockRef): + # Callback Add to fork choice + node.attestationPool.addForkChoice_v2(validBlock) + if newBlockRef.isErr: warn "Unable to add proposed block to block pool", newBlock = shortLog(newBlock.message), diff --git a/research/block_sim.nim b/research/block_sim.nim index 02ac9d063..5adac2a5a 100644 --- a/research/block_sim.nim +++ b/research/block_sim.nim @@ -86,7 +86,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6, var aggregation_bits = CommitteeValidatorsBits.init(committee.len) aggregation_bits.setBit index_in_committee - attPool.add( + attPool.addAttestation( Attestation( data: data, aggregation_bits: aggregation_bits, @@ -134,9 +134,11 @@ cli do(slots = SLOTS_PER_EPOCH * 6, state.fork, state.genesis_validators_root, newBlock.message.slot, blockRoot, privKey) - let added = blockPool.add(blockRoot, newBlock).tryGet() - blck() = added - blockPool.updateHead(added) + let added = blockPool.addRawBlock(blockRoot, newBlock) do (validBlock: BlockRef): + # Callback Add to fork choice + attPool.addForkChoice_v2(validBlock) + blck() = added[] + blockPool.updateHead(added[]) for i in 0.. Date: Thu, 9 Jul 2020 11:43:27 +0000 Subject: [PATCH 3/3] update most remaining non-fork-choice spec refs, updating code where necessary (#1292) * update most of the remaining non-fork-choice spec refs, updating code where necessary * revert presumably harmless compute_signing_root() change, but this way, keep things really unchanged outside inspector --- AllTests-mainnet.md | 5 +++-- beacon_chain/attestation_aggregation.nim | 2 +- beacon_chain/attestation_pool.nim | 2 +- beacon_chain/block_pools/candidate_chains.nim | 1 - beacon_chain/inspector.nim | 2 +- beacon_chain/spec/beaconstate.nim | 4 ++-- beacon_chain/spec/network.nim | 3 --- beacon_chain/spec/state_transition.nim | 2 +- beacon_chain/spec/state_transition_epoch.nim | 6 +++--- beacon_chain/spec/state_transition_helpers.nim | 2 +- tests/mocking/mock_genesis.nim | 2 +- 11 files changed, 14 insertions(+), 17 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 73236ef24..b64663acc 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -245,9 +245,10 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + Deposit at MAX_EFFECTIVE_BALANCE balance (32 ETH) OK + Deposit over MAX_EFFECTIVE_BALANCE balance (32 ETH) OK + Deposit under MAX_EFFECTIVE_BALANCE balance (32 ETH) OK ++ Invalid deposit at MAX_EFFECTIVE_BALANCE balance (32 ETH) OK + Validator top-up OK ``` -OK: 4/4 Fail: 0/4 Skip: 0/4 +OK: 5/5 Fail: 0/5 Skip: 0/5 ## [Unit - Spec - Epoch processing] Justification and Finalization [Preset: mainnet] ```diff + Rule I - 234 finalization with enough support OK @@ -267,4 +268,4 @@ OK: 8/8 Fail: 0/8 Skip: 0/8 OK: 1/1 Fail: 0/1 Skip: 0/1 ---TOTAL--- -OK: 160/163 Fail: 0/163 Skip: 3/163 +OK: 161/164 Fail: 0/164 Skip: 3/164 diff --git a/beacon_chain/attestation_aggregation.nim b/beacon_chain/attestation_aggregation.nim index f406c1cc4..d5e325ebd 100644 --- a/beacon_chain/attestation_aggregation.nim +++ b/beacon_chain/attestation_aggregation.nim @@ -52,7 +52,7 @@ proc aggregate_attestations*( if not is_aggregator(state, slot, index, slot_signature, cache): return none(AggregateAndProof) - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#attestation-data + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#attestation-data # describes how to construct an attestation, which applies for makeAttestationData(...) # TODO this won't actually match anything let attestation_data = AttestationData( diff --git a/beacon_chain/attestation_pool.nim b/beacon_chain/attestation_pool.nim index e6de5b04f..64a1a666a 100644 --- a/beacon_chain/attestation_pool.nim +++ b/beacon_chain/attestation_pool.nim @@ -473,7 +473,7 @@ proc getAttestationsForBlock*(pool: AttestationPool, var cache = get_empty_per_epoch_cache() for a in attestations: var - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#construct-attestation + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#construct-attestation attestation = Attestation( aggregation_bits: a.validations[0].aggregation_bits, data: a.data, diff --git a/beacon_chain/block_pools/candidate_chains.nim b/beacon_chain/block_pools/candidate_chains.nim index d27ecc396..6c44273e1 100644 --- a/beacon_chain/block_pools/candidate_chains.nim +++ b/beacon_chain/block_pools/candidate_chains.nim @@ -920,7 +920,6 @@ proc getProposer*( dag.withState(dag.tmpState, head.atSlot(slot)): var cache = get_empty_per_epoch_cache() - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/validator.md#validator-assignments let proposerIdx = get_beacon_proposer_index(state, cache) if proposerIdx.isNone: warn "Missing proposer index", diff --git a/beacon_chain/inspector.nim b/beacon_chain/inspector.nim index 2e07e058c..b816030cf 100644 --- a/beacon_chain/inspector.nim +++ b/beacon_chain/inspector.nim @@ -509,7 +509,7 @@ proc pubsubLogger(conf: InspectorConf, switch: Switch, try: if topic.endsWith(topicBeaconBlocksSuffix & "_snappy"): info "SignedBeaconBlock", msg = SSZ.decode(buffer, SignedBeaconBlock) - elif topic.endsWith(topicMainnetAttestationsSuffix & "_snappy"): + elif topic.endsWith("_snappy") and topic.contains("/beacon_attestation_"): info "Attestation", msg = SSZ.decode(buffer, Attestation) elif topic.endsWith(topicVoluntaryExitsSuffix & "_snappy"): info "SignedVoluntaryExit", msg = SSZ.decode(buffer, diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index c5e05b969..45dca7d43 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -453,7 +453,7 @@ func get_attesting_indices*(state: BeaconState, # This shouldn't happen if one begins with a valid BeaconState and applies # valid updates, but one can construct a BeaconState where it does. Do not # do anything here since the PendingAttestation wouldn't have made it past - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#attestations + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#attestations # which checks len(attestation.aggregation_bits) == len(committee) that in # nim-beacon-chain lives in check_attestation(...). # Addresses https://github.com/status-im/nim-beacon-chain/issues/922 @@ -568,7 +568,7 @@ proc isValidAttestationTargetEpoch*( true -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.2/specs/phase0/beacon-chain.md#attestations +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#attestations proc check_attestation*( state: BeaconState, attestation: SomeAttestation, flags: UpdateFlags, stateCache: var StateCache): bool = diff --git a/beacon_chain/spec/network.nim b/beacon_chain/spec/network.nim index ba451075c..302194e54 100644 --- a/beacon_chain/spec/network.nim +++ b/beacon_chain/spec/network.nim @@ -19,9 +19,6 @@ const topicAttesterSlashingsSuffix* = "attester_slashing/ssz" topicAggregateAndProofsSuffix* = "beacon_aggregate_and_proof/ssz" - # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/p2p-interface.md#topics-and-messages - topicMainnetAttestationsSuffix* = "_beacon_attestation/ssz" - # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#misc ATTESTATION_SUBNET_COUNT* = 64 diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index 0f66eb78f..d4da6f02c 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -262,7 +262,7 @@ proc state_transition*( state.data, state.data.slot.compute_epoch_at_slot) state_transition(state, signedBlock, cache, flags, rollback) -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#preparing-for-a-beaconblock # TODO There's more to do here - the spec has helpers that deal set up some of # the fields in here! proc makeBeaconBlock*( diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 81f032221..e9289c19d 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -70,7 +70,7 @@ func get_total_active_balance*(state: BeaconState, cache: var StateCache): Gwei except KeyError: raiseAssert("get_total_active_balance(): cache always filled before usage") -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#helper-functions-1 +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#helper-functions-1 func get_matching_source_attestations(state: BeaconState, epoch: Epoch): seq[PendingAttestation] = doAssert epoch in [get_current_epoch(state), get_previous_epoch(state)] @@ -104,7 +104,7 @@ func get_attesting_balance( get_total_balance(state, get_unslashed_attesting_indices( state, attestations, stateCache)) -# https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#justification-and-finalization +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#justification-and-finalization proc process_justification_and_finalization*(state: var BeaconState, stateCache: var StateCache, updateFlags: UpdateFlags = {}) {.nbench.} = @@ -437,7 +437,7 @@ func process_rewards_and_penalties( increase_balance(state, i.ValidatorIndex, rewards[i]) decrease_balance(state, i.ValidatorIndex, penalties[i]) -# https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#slashings +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#slashings func process_slashings*(state: var BeaconState, cache: var StateCache) {.nbench.}= let epoch = get_current_epoch(state) diff --git a/beacon_chain/spec/state_transition_helpers.nim b/beacon_chain/spec/state_transition_helpers.nim index 5e3935d00..001109c83 100644 --- a/beacon_chain/spec/state_transition_helpers.nim +++ b/beacon_chain/spec/state_transition_helpers.nim @@ -24,7 +24,7 @@ func shortLog*(x: Checkpoint): string = # Helpers used in epoch transition and trace-level block transition # -------------------------------------------------------- -# https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#helper-functions-1 +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#helper-functions-1 func get_attesting_indices*( state: BeaconState, attestations: openarray[PendingAttestation], stateCache: var StateCache): HashSet[ValidatorIndex] = diff --git a/tests/mocking/mock_genesis.nim b/tests/mocking/mock_genesis.nim index 9c0b3db07..75c80929b 100644 --- a/tests/mocking/mock_genesis.nim +++ b/tests/mocking/mock_genesis.nim @@ -12,7 +12,7 @@ import # Specs ../../beacon_chain/spec/[datatypes, beaconstate], # Internals - ../../beacon_chain/[extras, interop], + ../../beacon_chain/interop, # Mocking procs ./mock_deposits