Dual headed fork choice (#1163)

* Dual headed fork choice

* fix finalizedEpoch not moving

* reduce fork choice verbosity

.gitignore

@@ -32,9 +32,9 @@ build/
 *.sqlite3
 
 /local_testnet_data*/
+/local_testnet*_data*/
 
 # Prometheus db
 /data
 # Grafana dashboards
 /docker/*.json
-

beacon_chain/attestation_pool.nim

@@ -11,7 +11,8 @@ import
   deques, sequtils, tables, options,
   chronicles, stew/[byteutils], json_serialization/std/sets,
   ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator],
-  ./extras, ./block_pool, ./block_pools/candidate_chains, ./beacon_node_types
+  ./extras, ./block_pool, ./block_pools/candidate_chains, ./beacon_node_types,
+  ./fork_choice/[fork_choice_types, fork_choice]
 
 logScope: topics = "attpool"
 
@@ -22,10 +23,23 @@ func init*(T: type AttestationPool, blockPool: BlockPool): T =
   # TODO blockPool is only used when resolving orphaned attestations - it should
   #      probably be removed as a dependency of AttestationPool (or some other
   #      smart refactoring)
+
+  # TODO: In tests, on blockpool.init the finalized root
+  #       from the `headState` and `justifiedState` is zero
+  let forkChoice = initForkChoice(
+    finalized_block_slot = default(Slot),             # This is unnecessary for fork choice but may help external components
+    finalized_block_state_root = default(Eth2Digest), # This is unnecessary for fork choice but may help external components
+    justified_epoch = blockPool.headState.data.data.current_justified_checkpoint.epoch,
+    finalized_epoch = blockPool.headState.data.data.finalized_checkpoint.epoch,
+    # finalized_root = blockPool.headState.data.data.finalized_checkpoint.root
+    finalized_root = blockPool.finalizedHead.blck.root
+  ).get()
+
   T(
     mapSlotsToAttestations: initDeque[AttestationsSeen](),
     blockPool: blockPool,
     unresolved: initTable[Eth2Digest, UnresolvedAttestation](),
+    forkChoice_v2: forkChoice
   )
 
 proc combine*(tgt: var Attestation, src: Attestation, flags: UpdateFlags) =
@@ -107,13 +121,21 @@ proc slotIndex(
 func updateLatestVotes(
     pool: var AttestationPool, state: BeaconState, attestationSlot: Slot,
     participants: seq[ValidatorIndex], blck: BlockRef) =
+
+  # ForkChoice v2
+  let target_epoch = compute_epoch_at_slot(attestationSlot)
+
   for validator in participants:
+    # ForkChoice v1
     let
       pubKey = state.validators[validator].pubkey
       current = pool.latestAttestations.getOrDefault(pubKey)
     if current.isNil or current.slot < attestationSlot:
       pool.latestAttestations[pubKey] = blck
 
+    # ForkChoice v2
+    pool.forkChoice_v2.process_attestation(validator, blck.root, target_epoch)
+
 func get_attesting_indices_seq(state: BeaconState,
                                attestation_data: AttestationData,
                                bits: CommitteeValidatorsBits,
@@ -261,6 +283,34 @@ proc add*(pool: var AttestationPool, attestation: Attestation) =
 
   pool.addResolved(blck, attestation)
 
+proc addForkChoice_v2*(pool: var AttestationPool, blck: BlockRef) =
+  ## Add a verified block to the fork choice context
+  ## The current justifiedState of the block pool is used as reference
+
+  # TODO: add(BlockPool, blockRoot: Eth2Digest, SignedBeaconBlock): BlockRef
+  # should ideally return the justified_epoch and finalized_epoch
+  # so that we can pass them directly to this proc without having to
+  # redo "updateStateData"
+  #
+  # In any case, `updateStateData` should shortcut
+  # to `getStateDataCached`
+
+  updateStateData(
+    pool.blockPool,
+    pool.blockPool.tmpState,
+    BlockSlot(blck: blck, slot: blck.slot)
+  )
+
+  let blockData = pool.blockPool.get(blck)
+  pool.forkChoice_v2.process_block(
+    slot = blck.slot,
+    block_root = blck.root,
+    parent_root = if not blck.parent.isNil: blck.parent.root else: default(Eth2Digest),
+    state_root = default(Eth2Digest), # This is unnecessary for fork choice but may help external components
+    justified_epoch = pool.blockPool.tmpState.data.data.current_justified_checkpoint.epoch,
+    finalized_epoch = pool.blockPool.tmpState.data.data.finalized_checkpoint.epoch,
+  ).get()
+
 proc getAttestationsForSlot*(pool: AttestationPool, newBlockSlot: Slot):
     Option[AttestationsSeen] =
   if newBlockSlot < (GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY):
@@ -395,7 +445,10 @@ proc resolve*(pool: var AttestationPool) =
   for a in resolved:
     pool.addResolved(a.blck, a.attestation)
 
-func latestAttestation*(
+# Fork choice v1
+# ---------------------------------------------------------------
+
+func latestAttestation(
     pool: AttestationPool, pubKey: ValidatorPubKey): BlockRef =
   pool.latestAttestations.getOrDefault(pubKey)
 
@@ -403,7 +456,7 @@ func latestAttestation*(
 # The structure of this code differs from the spec since we use a different
 # strategy for storing states and justification points - it should nonetheless
 # be close in terms of functionality.
-func lmdGhost*(
+func lmdGhost(
     pool: AttestationPool, start_state: BeaconState,
     start_block: BlockRef): BlockRef =
   # TODO: a Fenwick Tree datastructure to keep track of cumulated votes
@@ -454,7 +507,7 @@ func lmdGhost*(
           winCount = candCount
       head = winner
 
-proc selectHead*(pool: AttestationPool): BlockRef =
+proc selectHead_v1(pool: AttestationPool): BlockRef =
   let
     justifiedHead = pool.blockPool.latestJustifiedBlock()
 
@@ -462,3 +515,47 @@ proc selectHead*(pool: AttestationPool): BlockRef =
     lmdGhost(pool, pool.blockPool.justifiedState.data.data, justifiedHead.blck)
 
   newHead
+
+# Fork choice v2
+# ---------------------------------------------------------------
+
+func getAttesterBalances(state: StateData): seq[Gwei] {.noInit.}=
+  ## Get the balances from a state
+  result.newSeq(state.data.data.validators.len) # zero-init
+
+  let epoch = state.data.data.slot.compute_epoch_at_slot()
+
+  for i in 0 ..< result.len:
+    # All non-active validators have a 0 balance
+    template validator: Validator = state.data.data.validators[i]
+    if validator.is_active_validator(epoch):
+      result[i] = validator.effective_balance
+
+proc selectHead_v2(pool: var AttestationPool): BlockRef =
+  let attesterBalances = pool.blockPool.justifiedState.getAttesterBalances()
+
+  let newHead = pool.forkChoice_v2.find_head(
+    justified_epoch = pool.blockPool.justifiedState.data.data.slot.compute_epoch_at_slot(),
+    justified_root = pool.blockPool.head.justified.blck.root,
+    finalized_epoch = pool.blockPool.headState.data.data.finalized_checkpoint.epoch,
+    justified_state_balances = attesterBalances
+  ).get()
+
+  pool.blockPool.getRef(newHead)
+
+proc pruneBefore*(pool: var AttestationPool, finalizedhead: BlockSlot) =
+  pool.forkChoice_v2.maybe_prune(finalizedHead.blck.root).get()
+
+# Dual-Headed Fork choice
+# ---------------------------------------------------------------
+
+proc selectHead*(pool: var AttestationPool): BlockRef =
+  let head_v1 = pool.selectHead_v1()
+  let head_v2 = pool.selectHead_v2()
+
+  if head_v1 != head_v2:
+    error "Fork choice engines in disagreement, using block from v1.",
+      v1_block = shortlog(head_v1),
+      v2_block = shortlog(head_v2)
+
+  return head_v1

beacon_chain/beacon_node.nim

@@ -313,6 +313,10 @@ proc storeBlock(
 
     return err(blck.error)
 
+  # Still here? This means we received a valid block and we need to add it
+  # to the fork choice
+  node.attestationPool.addForkChoice_v2(blck.get())
+
   # The block we received contains attestations, and we might not yet know about
   # all of them. Let's add them to the attestation pool - in case the block
   # is not yet resolved, neither will the attestations be!
@@ -1134,4 +1138,3 @@ programMain:
         config.depositContractAddress,
         config.depositPrivateKey,
         delayGenerator)
-

beacon_chain/beacon_node_common.nim

@@ -20,7 +20,8 @@ import
   conf, time, beacon_chain_db,
   attestation_pool, block_pool, eth2_network,
   beacon_node_types, mainchain_monitor, request_manager,
-  sync_manager
+  sync_manager,
+  fork_choice/fork_choice
 
 # This removes an invalid Nim warning that the digest module is unused here
 # It's currently used for `shortLog(head.blck.root)`
@@ -68,6 +69,9 @@ proc updateHead*(node: BeaconNode): BlockRef =
   node.blockPool.updateHead(newHead)
   beacon_head_root.set newHead.root.toGaugeValue
 
+  # Cleanup the fork choice v2 if we have a finalized head
+  node.attestationPool.pruneBefore(node.blockPool.finalizedHead)
+
   newHead
 
 template findIt*(s: openarray, predicate: untyped): int64 =

beacon_chain/beacon_node_types.nim

@@ -5,7 +5,8 @@ import
   stew/endians2,
   spec/[datatypes, crypto, digest],
   block_pools/block_pools_types,
-  block_pool # TODO: refactoring compat shim
+  block_pool, # TODO: refactoring compat shim
+  fork_choice/fork_choice_types
 
 export block_pools_types
 
@@ -74,6 +75,9 @@ type
     latestAttestations*: Table[ValidatorPubKey, BlockRef] ##\
     ## Map that keeps track of the most recent vote of each attester - see
     ## fork_choice
+    forkChoice_v2*: ForkChoice ##\
+    ## The alternative fork choice "proto_array" that will ultimately
+    ## replace the original one
 
   # #############################################
   #

beacon_chain/fork_choice/fork_choice.nim

@@ -117,7 +117,7 @@ func process_attestation*(
     vote.next_epoch = target_epoch
 
     {.noSideEffect.}:
-      info "Integrating vote in fork choice",
+      trace "Integrating vote in fork choice",
         validator_index = $validator_index,
         new_vote = shortlog(vote)
   else:
@@ -129,7 +129,7 @@ func process_attestation*(
           ignored_block_root = shortlog(block_root),
           ignored_target_epoch = $target_epoch
       else:
-        info "Ignoring double-vote for fork choice",
+        trace "Ignoring double-vote for fork choice",
           validator_index = $validator_index,
           current_vote = shortlog(vote),
           ignored_block_root = shortlog(block_root),
@@ -159,7 +159,7 @@ func process_block*(
     return err("process_block_error: " & $err)
 
   {.noSideEffect.}:
-    info "Integrating block in fork choice",
+    trace "Integrating block in fork choice",
       block_root = $shortlog(block_root),
       parent_root = $shortlog(parent_root),
       justified_epoch = $justified_epoch,
@@ -205,7 +205,7 @@ func find_head*(
     return err("find_head failed: " & $ghost_err)
 
   {.noSideEffect.}:
-    info "Fork choice requested",
+    debug "Fork choice requested",
       justified_epoch = $justified_epoch,
       justified_root = shortlog(justified_root),
       finalized_epoch = $finalized_epoch,

tests/test_attestation_pool.nim

@@ -33,7 +33,7 @@ suiteReport "Attestation pool processing" & preset():
     check:
       process_slots(state.data, state.data.data.slot + 1)
 
-    # pool[].add(blockPool[].tail) # Make the tail known to fork choice
+    pool[].addForkChoice_v2(blockPool[].tail) # Make the tail known to fork choice
 
   timedTest "Can add and retrieve simple attestation" & preset():
     var cache = get_empty_per_epoch_cache()
@@ -161,7 +161,7 @@ suiteReport "Attestation pool processing" & preset():
       b1Root = hash_tree_root(b1.message)
       b1Add = blockpool[].add(b1Root, b1)[]
 
-    # pool[].add(b1Add) - make a block known to the future fork choice
+    pool[].addForkChoice_v2(b1Add)
     let head = pool[].selectHead()
 
     check:
@@ -172,7 +172,7 @@ suiteReport "Attestation pool processing" & preset():
       b2Root = hash_tree_root(b2.message)
       b2Add = blockpool[].add(b2Root, b2)[]
 
-    # pool[].add(b2Add) - make a block known to the future fork choice
+    pool[].addForkChoice_v2(b2Add)
     let head2 = pool[].selectHead()
 
     check:
@@ -185,7 +185,7 @@ suiteReport "Attestation pool processing" & preset():
       b10Root = hash_tree_root(b10.message)
       b10Add = blockpool[].add(b10Root, b10)[]
 
-    # pool[].add(b10Add) - make a block known to the future fork choice
+    pool[].addForkChoice_v2(b10Add)
     let head = pool[].selectHead()
 
     check:
@@ -202,7 +202,7 @@ suiteReport "Attestation pool processing" & preset():
         state.data.data, state.data.data.slot, 1.CommitteeIndex, cache)
       attestation0 = makeAttestation(state.data.data, b10Root, bc1[0], cache)
 
-    # pool[].add(b11Add) - make a block known to the future fork choice
+    pool[].addForkChoice_v2(b11Add)
     pool[].add(attestation0)
 
     let head2 = pool[].selectHead()