fork choice proposer boosting support (#3349)

* fork choice proposer boosting support

* detect nodeDelta underflow/overflow

ConsensusSpecPreset-mainnet.md

@@ -952,7 +952,7 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
 ```diff
 + ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/chain_no_attestations        OK
 + ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/genesis                      OK
-  ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head  Skip
++ ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/proposer_boost_correct_head  OK
 + ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/shorter_chain_but_heavier_we OK
 + ForkChoice - mainnet/phase0/fork_choice/get_head/pyspec_tests/split_tie_breaker_no_attesta OK
 + ForkChoice - mainnet/phase0/fork_choice/on_block/pyspec_tests/basic                        OK
@@ -961,7 +961,7 @@ OK: 38/38 Fail: 0/38 Skip: 0/38
 + ForkChoice - mainnet/phase0/fork_choice/on_block/pyspec_tests/proposer_boost               OK
 + ForkChoice - mainnet/phase0/fork_choice/on_block/pyspec_tests/proposer_boost_root_same_slo OK
 ```
-OK: 8/10 Fail: 0/10 Skip: 2/10
+OK: 9/10 Fail: 0/10 Skip: 1/10
 ## EF - Phase 0 - Epoch Processing - Effective balance updates [Preset: mainnet]
 ```diff
 + Effective balance updates - effective_balance_hysteresis [Preset: mainnet]                 OK
@@ -1209,4 +1209,4 @@ OK: 44/44 Fail: 0/44 Skip: 0/44
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 1033/1035 Fail: 0/1035 Skip: 2/1035
+OK: 1034/1035 Fail: 0/1035 Skip: 1/1035

beacon_chain/conf.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -443,6 +443,12 @@ type
         defaultValue: false
         name: "validator-monitor-totals" }: bool
 
+      proposerBoosting* {.
+        hidden
+        desc: "Enable proposer boosting; temporary option feature gate (debugging; option will be removed)",
+        defaultValue: false
+        name: "proposer-boosting-debug" }: bool
+
     of BNStartUpCmd.createTestnet:
       testnetDepositsFile* {.
         desc: "A LaunchPad deposits file for the genesis state validators"

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -86,7 +86,8 @@ declareGauge attestation_pool_block_attestation_packing_time,
 
 proc init*(T: type AttestationPool, dag: ChainDAGRef,
            quarantine: ref Quarantine,
-           onAttestation: OnAttestationCallback = nil): T =
+           onAttestation: OnAttestationCallback = nil,
+           proposerBoosting: bool = false): T =
   ## Initialize an AttestationPool from the dag `headState`
   ## The `finalized_root` works around the finalized_checkpoint of the genesis block
   ## holding a zero_root.
@@ -94,7 +95,8 @@ proc init*(T: type AttestationPool, dag: ChainDAGRef,
 
   var forkChoice = ForkChoice.init(
     finalizedEpochRef,
-    dag.finalizedHead.blck)
+    dag.finalizedHead.blck,
+    proposerBoosting)
 
   # Feed fork choice with unfinalized history - during startup, block pool only
   # keeps track of a single history so we just need to follow it

beacon_chain/fork_choice/fork_choice.nim

@@ -49,17 +49,19 @@ logScope:
 
 proc init*(T: type ForkChoiceBackend,
            justifiedCheckpoint: Checkpoint,
-           finalizedCheckpoint: Checkpoint): T =
+           finalizedCheckpoint: Checkpoint,
+           useProposerBoosting: bool): T =
   T(
     proto_array: ProtoArray.init(
       justifiedCheckpoint,
-      finalizedCheckpoint
+      finalizedCheckpoint),
-    )
+    proposer_boosting: useProposerBoosting
   )
 
 proc init*(T: type ForkChoice,
            epochRef: EpochRef,
-           blck: BlockRef): T =
+           blck: BlockRef,
+           proposerBoosting: bool): T =
   ## Initialize a fork choice context for a finalized state - in the finalized
   ## state, the justified and finalized checkpoints are the same, so only one
   ## is used here
@@ -75,11 +77,12 @@ proc init*(T: type ForkChoice,
       root: blck.root, epoch: epochRef.epoch)
 
   ForkChoice(
-    backend: ForkChoiceBackend.init(best_justified, finalized),
+    backend: ForkChoiceBackend.init(
+      best_justified, finalized, proposerBoosting),
     checkpoints: Checkpoints(
       justified: justified,
       finalized: finalized,
-      best_justified: best_justified)
+      best_justified: best_justified),
   )
 
 func extend[T](s: var seq[T], minLen: int) =
@@ -344,7 +347,7 @@ proc process_block*(self: var ForkChoice,
   let committees_per_slot = get_committee_count_per_slot(epochRef)
 
   for attestation in blck.body.attestations:
-    let targetBlck = dag.getBlockRef(attestation.data.target.root).valueOr:
+    let _ = dag.getBlockRef(attestation.data.target.root).valueOr:
       continue
 
     let committee_index = block:
@@ -389,7 +392,8 @@ func find_head*(
        self: var ForkChoiceBackend,
        justifiedCheckpoint: Checkpoint,
        finalizedCheckpoint: Checkpoint,
-       justified_state_balances: seq[Gwei]
+       justified_state_balances: seq[Gwei],
+       proposer_boost_root: Eth2Digest
      ): FcResult[Eth2Digest] =
   ## Returns the new blockchain head
 
@@ -406,7 +410,8 @@ func find_head*(
 
   # Apply score changes
   ? self.proto_array.applyScoreChanges(
-    deltas, justifiedCheckpoint, finalizedCheckpoint
+    deltas, justifiedCheckpoint, finalizedCheckpoint,
+    justified_state_balances, proposer_boost_root, self.proposer_boosting
   )
 
   self.balances = justified_state_balances
@@ -433,6 +438,7 @@ proc get_head*(self: var ForkChoice,
     self.checkpoints.justified.checkpoint,
     self.checkpoints.finalized,
     self.checkpoints.justified.balances,
+    self.checkpoints.proposer_boost_root
   )
 
 func prune*(

beacon_chain/fork_choice/fork_choice_types.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -38,6 +38,7 @@ type
     fcInvalidParentDelta
     fcInvalidNodeDelta
     fcDeltaUnderflow
+    fcDeltaOverflow
     fcInvalidDeltaLen
     fcInvalidBestNode
     fcInconsistentTick
@@ -61,7 +62,8 @@ type
        fcInvalidBestDescendant,
        fcInvalidParentDelta,
        fcInvalidNodeDelta,
-       fcDeltaUnderflow:
+       fcDeltaUnderflow,
+       fcDeltaOverflow:
          index*: Index
     of fcInvalidDeltaLen:
       deltasLen*: int
@@ -92,6 +94,8 @@ type
     finalizedCheckpoint*: Checkpoint
     nodes*: ProtoNodes
     indices*: Table[Eth2Digest, Index]
+    previousProposerBoostRoot*: Eth2Digest
+    previousProposerBoostScore*: int64
 
   ProtoNode* = object
     root*: Eth2Digest
@@ -126,6 +130,7 @@ type
     proto_array*: ProtoArray
     votes*: seq[VoteTracker]
     balances*: seq[Gwei]
+    proposer_boosting*: bool
 
   QueuedAttestation* = object
     slot*: Slot

beacon_chain/fork_choice/proto_array.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -9,10 +9,10 @@
 
 import
   # Standard library
-  std/tables, std/options, std/typetraits,
+  std/[options, tables, typetraits],
   # Status libraries
   chronicles,
-  stew/results,
+  stew/[objects, results],
   # Internal
   ../spec/datatypes/base,
   # Fork choice
@@ -103,23 +103,52 @@ func init*(T: type ProtoArray,
     indices: {node.root: 0}.toTable()
   )
 
+# https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/phase0/fork-choice.md#configuration
+# https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/phase0/fork-choice.md#get_latest_attesting_balance
+func calculateProposerBoost(validatorBalances: openArray[Gwei]): int64 =
+  const PROPOSER_SCORE_BOOST = 70
+  var
+    total_balance: uint64
+    num_validators: int64
+  for balance in validatorBalances:
+    # We need to filter zero balances here to get an accurate active validator
+    # count. This is because we default inactive validator balances to zero
+    # when creating this balances array.
+    if balance != 0:
+      total_balance += balance
+      num_validators += 1
+  if num_validators == 0:
+    return 0
+  let
+    average_balance = int64(total_balance div num_validators.uint64)
+    committee_size = num_validators div SLOTS_PER_EPOCH.int64
+    committee_weight = committee_size * average_balance
+  (committee_weight * PROPOSER_SCORE_BOOST) div 100
+
 func applyScoreChanges*(self: var ProtoArray,
                         deltas: var openArray[Delta],
                         justifiedCheckpoint: Checkpoint,
-                        finalizedCheckpoint: Checkpoint): FcResult[void] =
+                        finalizedCheckpoint: Checkpoint,
+                        newBalances: openArray[Gwei],
+                        proposerBoostRoot: Eth2Digest,
+                        useProposerBoost: bool): FcResult[void] =
   ## Iterate backwards through the array, touching all nodes and their parents
   ## and potentially the best-child of each parent.
-  ##
+  #
-  ## The structure of `self.nodes` array ensures that the child of each node
+  # The structure of `self.nodes` array ensures that the child of each node
-  ## is always touched before it's aprent.
+  # is always touched before its parent.
-  ##
+  #
-  ## For each node the following is done:
+  # For each node the following is done:
-  ##
+  #
-  ## 1. Update the node's weight with the corresponding delta.
+  # 1. Update the node's weight with the corresponding delta.
-  ## 2. Backpropagate each node's delta to its parent's delta.
+  # 2. Backpropagate each node's delta to its parent's delta.
-  ## 3. Compare the current node with the parent's best-child,
+  # 3. Compare the current node with the parent's best-child,
-  ##    updating if the current node should become the best-child
+  #    updating if the current node should become the best-child
-  ## 4. If required, update the parent's best-descendant with the current node or its best-descendant
+  # 4. If required, update the parent's best-descendant with the current node
+  #    or its best-descendant
+  #
+  # useProposerBoost is temporary, until it can be either permanently enabled
+  # or is removed from the Eth2 spec.
   doAssert self.indices.len == self.nodes.len # By construction
   if deltas.len != self.indices.len:
     return err ForkChoiceError(
@@ -135,12 +164,42 @@ func applyScoreChanges*(self: var ProtoArray,
   template node: untyped {.dirty.} =
     self.nodes.buf[nodePhysicalIdx]
 
+  # Default value, if not otherwise set in first node loop
+  var proposerBoostScore: int64
+
   # Iterate backwards through all the indices in `self.nodes`
   for nodePhysicalIdx in countdown(self.nodes.len - 1, 0):
-    if node.root == default(Eth2Digest):
+    if node.root.isZeroMemory:
       continue
 
-    let nodeDelta = deltas[nodePhysicalIdx]
+    var nodeDelta = deltas[nodePhysicalIdx]
+
+    # If we find the node for which the proposer boost was previously applied,
+    # decrease the delta by the previous score amount.
+    if  useProposerBoost and
+        (not self.previousProposerBoostRoot.isZeroMemory) and
+        self.previousProposerBoostRoot == node.root:
+          if  nodeDelta < 0 and
+              nodeDelta - low(Delta) < self.previousProposerBoostScore:
+            return err ForkChoiceError(
+                kind: fcDeltaUnderflow,
+                index: nodePhysicalIdx)
+          nodeDelta -= self.previousProposerBoostScore
+
+    # If we find the node matching the current proposer boost root, increase
+    # the delta by the new score amount.
+    #
+    # https://github.com/ethereum/consensus-specs/blob/v1.1.9/specs/phase0/fork-choice.md#get_latest_attesting_balance
+    if  useProposerBoost and
+        (not proposer_boost_root.isZeroMemory) and
+        proposer_boost_root == node.root:
+      proposerBoostScore = calculateProposerBoost(newBalances)
+      if  nodeDelta >= 0 and
+          high(Delta) - nodeDelta < self.previousProposerBoostScore:
+        return err ForkChoiceError(
+            kind: fcDeltaOverflow,
+            index: nodePhysicalIdx)
+      nodeDelta += proposerBoostScore.int64
 
     # Apply the delta to the node
     # We fail fast if underflow, which shouldn't happen.
@@ -152,7 +211,7 @@ func applyScoreChanges*(self: var ProtoArray,
         index: nodePhysicalIdx)
     node.weight = weight
 
-    # If the node has a parent, try to update its best-child and best-descendant
+    # If the node has a parent, try to update its best-child and best-descendent
     if node.parent.isSome():
       let parentLogicalIdx = node.parent.unsafeGet()
       let parentPhysicalIdx = parentLogicalIdx - self.nodes.offset
@@ -186,8 +245,13 @@ func applyScoreChanges*(self: var ProtoArray,
       # Back-propagate the nodes delta to its parent.
       deltas[parentPhysicalIdx] += nodeDelta
 
+  # After applying all deltas, update the `previous_proposer_boost`.
+  if useProposerBoost:
+    self.previousProposerBoostRoot = proposerBoostRoot
+    self.previousProposerBoostScore = proposerBoostScore
+
   for nodePhysicalIdx in countdown(self.nodes.len - 1, 0):
-    if node.root == default(Eth2Digest):
+    if node.root.isZeroMemory:
       continue
 
     if node.parent.isSome():

beacon_chain/nimbus_beacon_node.nim

@@ -444,8 +444,8 @@ proc init*(T: type BeaconNode,
       rng, config, netKeys, cfg, dag.forkDigests, getBeaconTime,
       getStateField(dag.headState.data, genesis_validators_root))
     attestationPool = newClone(
-      AttestationPool.init(dag, quarantine, onAttestationReceived)
+      AttestationPool.init(
-    )
+        dag, quarantine, onAttestationReceived, config.proposerBoosting))
     syncCommitteeMsgPool = newClone(
       SyncCommitteeMsgPool.init(rng, onSyncContribution)
     )

tests/consensus_spec/test_fixture_fork_choice.nim

@@ -99,7 +99,8 @@ proc initialLoad(
       defaultRuntimeConfig, db, validatorMonitor, {})
     fkChoice = newClone(ForkChoice.init(
       dag.getFinalizedEpochRef(),
-      dag.finalizedHead.blck
+      dag.finalizedHead.blck,
+      true
     ))
 
   (dag, fkChoice)
@@ -329,9 +330,6 @@ suite "EF - ForkChoice" & preset():
     # test: tests/fork_choice/scenarios/no_votes.nim
     #       "Ensure the head is still 4 whilst the justified epoch is 0."
     "on_block_future_block",
-
-    # TODO needs the actual proposer boost enabled
-    "proposer_boost_correct_head"
   ]
 
   for fork in [BeaconBlockFork.Phase0]: # TODO: init ChainDAG from Merge/Altair

tests/fork_choice/interpreter.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -7,7 +7,7 @@
 
 import
   # Standard library
-  std/strformat, std/tables, std/options,
+  std/[options, strformat, tables],
   # Status libraries
   stew/[results, endians2],
   # Internals
@@ -68,7 +68,9 @@ func apply(ctx: var ForkChoiceBackend, id: int, op: Operation) =
     let r = ctx.find_head(
       op.justified_checkpoint,
       op.finalized_checkpoint,
-      op.justified_state_balances
+      op.justified_state_balances,
+      # Don't use proposer boosting
+      default(Eth2Digest)
     )
     if op.kind == FindHead:
       doAssert r.isOk(), &"find_head (op #{id}) returned an error: {r.error}"

tests/fork_choice/scenarios/ffg_01.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -14,7 +14,8 @@ func setup_finality_01(): tuple[fork_choice: ForkChoiceBackend, ops: seq[Operati
   # Initialize the fork choice context
   result.fork_choice = ForkChoiceBackend.init(
     justifiedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(0)),
-    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(0))
+    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(0)),
+    true # use proposer boosting, though the proposer boost root not set
   )
 
   # ----------------------------------

tests/fork_choice/scenarios/ffg_02.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -14,7 +14,8 @@ func setup_finality_02(): tuple[fork_choice: ForkChoiceBackend, ops: seq[Operati
   # Initialize the fork choice context
   result.fork_choice = ForkChoiceBackend.init(
     justifiedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
-    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1))
+    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
+    true # use proposer boosting, though the proposer boost root not set
   )
 
   # ----------------------------------

tests/fork_choice/scenarios/no_votes.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -15,7 +15,8 @@ func setup_no_votes(): tuple[fork_choice: ForkChoiceBackend, ops: seq[Operation]
   # We start with epoch 0 fully finalized to avoid epoch 0 special cases.
   result.fork_choice = ForkChoiceBackend.init(
     justifiedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
-    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1))
+    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
+    true # use proposer boosting, though the proposer boost root not set
   )
 
   # ----------------------------------

tests/fork_choice/scenarios/votes.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2021 Status Research & Development GmbH
+# Copyright (c) 2018-2022 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -15,7 +15,8 @@ func setup_votes(): tuple[fork_choice: ForkChoiceBackend, ops: seq[Operation]] =
   # We start with epoch 0 fully finalized to avoid epoch 0 special cases.
   result.fork_choice = ForkChoiceBackend.init(
     justifiedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
-    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1))
+    finalizedCheckpoint = Checkpoint(root: GenesisRoot, epoch: Epoch(1)),
+    true # use proposer boosting, though the proposer boost root not set
   )
 
   # ----------------------------------