keep cache of per-epoch items in block pool (#1068)

* plumbing between block pool and state transition functions around active validator indices and committees

* have shared epochrefs followed by blockref tree while allowing for skipped slots

* factor out the epoch info extraction; document how the EpochRef follows forks

beacon_chain/block_pools/block_pools_types.nim

@@ -116,6 +116,10 @@ type
 
     updateFlags*: UpdateFlags
 
+  EpochRef* = ref object
+    shuffled_active_validator_indices*: seq[ValidatorIndex]
+    epoch*: Epoch
+
   BlockRef* = ref object
     ## Node in object graph guaranteed to lead back to tail block, and to have
     ## a corresponding entry in database.
@@ -132,6 +136,11 @@ type
 
     slot*: Slot # TODO could calculate this by walking to root, but..
 
+    epochsInfo*: seq[EpochRef]
+    ## Could be multiple, since blocks could skip slots, but usually, not many
+    ## Even if competing forks happen later during this epoch, potential empty
+    ## slots beforehand must all be from this fork.
+
   BlockData* = object
     ## Body and graph in one
 

beacon_chain/block_pools/candidate_chains.nim

@@ -8,7 +8,7 @@
 {.push raises: [Defect].}
 
 import
-  chronicles, options, tables,
+  chronicles, options, sequtils, tables,
   metrics,
   ../ssz, ../beacon_chain_db, ../state_transition, ../extras,
   ../spec/[crypto, datatypes, digest, helpers, validator],
@@ -53,6 +53,12 @@ func parent*(bs: BlockSlot): BlockSlot =
       slot: bs.slot - 1
     )
 
+func populateEpochCache*(state: BeaconState, epoch: Epoch): EpochRef =
+  result = (EpochRef)(
+    epoch: state.slot.compute_epoch_at_slot,
+    shuffled_active_validator_indices:
+      get_shuffled_active_validator_indices(state, epoch))
+
 func link*(parent, child: BlockRef) =
   doAssert (not (parent.root == Eth2Digest() or child.root == Eth2Digest())),
     "blocks missing root!"
@@ -133,6 +139,23 @@ func atSlot*(blck: BlockRef, slot: Slot): BlockSlot =
   ## block proposal)
   BlockSlot(blck: blck.getAncestorAt(slot), slot: slot)
 
+func getEpochInfo*(blck: BlockRef, state: BeaconState): EpochRef =
+  # This is the only intended mechanism by which to get an EpochRef
+  let
+    state_epoch = state.slot.compute_epoch_at_slot
+    matching_epochinfo = blck.epochsInfo.filterIt(it.epoch == state_epoch)
+
+  if matching_epochinfo.len == 0:
+    let cache = populateEpochCache(state, state_epoch)
+    blck.epochsInfo.add(cache)
+    trace "candidate_chains.skipAndUpdateState(): back-filling parent.epochInfo",
+      state_slot = state.slot
+    cache
+  elif matching_epochinfo.len == 1:
+    matching_epochinfo[0]
+  else:
+    raiseAssert "multiple EpochRefs per epoch per BlockRef invalid"
+
 func init(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
   BlockRef(
     root: root,
@@ -441,8 +464,17 @@ proc skipAndUpdateState(
     doAssert (addr(statePtr.data) == addr v)
     statePtr[] = dag.headState
 
+  # TODO it's probably not the right way to convey this, but for now, avoids
+  # death-by-dozens-of-pointless-changes in developing this
+  let epochInfo = getEpochInfo(blck.refs, state.data.data)
+  var stateCache = get_empty_per_epoch_cache()
+  stateCache.shuffled_active_validator_indices[
+    state.data.data.slot.compute_epoch_at_slot] =
+      epochInfo.shuffled_active_validator_indices
+
   let ok = state_transition(
-    state.data, blck.data, flags + dag.updateFlags, restore)
+    state.data, blck.data, stateCache, flags + dag.updateFlags, restore)
+
   if ok and save:
     dag.putState(state.data, blck.refs)
 

beacon_chain/block_pools/clearance.nim

@@ -9,7 +9,7 @@ import
   chronicles, tables,
   metrics, stew/results,
   ../ssz, ../state_transition, ../extras,
-  ../spec/[crypto, datatypes, digest, helpers],
+  ../spec/[crypto, datatypes, digest, helpers, validator],
 
   block_pools_types, candidate_chains
 
@@ -46,6 +46,8 @@ proc addResolvedBlock(
   doAssert state.slot == signedBlock.message.slot, "state must match block"
 
   let blockRef = BlockRef.init(blockRoot, signedBlock.message)
+  parent.epochsInfo =
+    @[populateEpochCache(state, state.slot.compute_epoch_at_slot)]
   link(parent, blockRef)
 
   dag.blocks[blockRoot] = blockRef
@@ -175,8 +177,19 @@ proc add*(
       doAssert v.addr == addr poolPtr.tmpState.data
       poolPtr.tmpState = poolPtr.headState
 
+    # TODO it's probably not the right way to convey this, but for now, avoids
+    # death-by-dozens-of-pointless-changes in developing this
+    # TODO rename these, since now, the two "state cache"s are juxtaposed
+    # directly
+    let epochInfo = getEpochInfo(parent, dag.tmpState.data.data)
+    var stateCache = get_empty_per_epoch_cache()
+    stateCache.shuffled_active_validator_indices[
+      dag.tmpState.data.data.slot.compute_epoch_at_slot] =
+        epochInfo.shuffled_active_validator_indices
+    # End of section to refactor/combine
+
     if not state_transition(
-        dag.tmpState.data, signedBlock, dag.updateFlags, restore):
+        dag.tmpState.data, signedBlock, stateCache, dag.updateFlags, restore):
       # TODO find a better way to log all this block data
       notice "Invalid block",
         blck = shortLog(blck),

beacon_chain/spec/datatypes.nim

@@ -399,11 +399,14 @@ type
     data*: BeaconState
     root*: Eth2Digest # hash_tree_root(data)
 
+  # TODO remove some of these, or otherwise coordinate with EpochRef
   StateCache* = object
     beacon_committee_cache*:
       Table[tuple[a: int, b: Eth2Digest], seq[ValidatorIndex]]
     active_validator_indices_cache*:
       Table[Epoch, seq[ValidatorIndex]]
+    shuffled_active_validator_indices*:
+      Table[Epoch, seq[ValidatorIndex]]
     committee_count_cache*: Table[Epoch, uint64]
 
 func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =

beacon_chain/spec/validator.nim

@@ -79,6 +79,17 @@ func get_shuffled_seq*(seed: Eth2Digest,
 
   result = shuffled_active_validator_indices
 
+func get_shuffled_active_validator_indices*(state: BeaconState, epoch: Epoch):
+    seq[ValidatorIndex] =
+  # Non-spec function, to cache a data structure from which one can cheaply
+  # compute both get_active_validator_indexes() and get_beacon_committee().
+  let active_validator_indices = get_active_validator_indices(state, epoch)
+  mapIt(
+    get_shuffled_seq(
+      get_seed(state, epoch, DOMAIN_BEACON_ATTESTER),
+      active_validator_indices.len.uint64),
+    active_validator_indices[it])
+
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#get_previous_epoch
 func get_previous_epoch*(state: BeaconState): Epoch =
   # Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``).
@@ -153,6 +164,8 @@ func get_empty_per_epoch_cache*(): StateCache =
     initTable[tuple[a: int, b: Eth2Digest], seq[ValidatorIndex]]()
   result.active_validator_indices_cache =
     initTable[Epoch, seq[ValidatorIndex]]()
+  result.shuffled_active_validator_indices =
+    initTable[Epoch, seq[ValidatorIndex]]()
   result.committee_count_cache = initTable[Epoch, uint64]()
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#compute_proposer_index

beacon_chain/state_transition.nim

@@ -28,6 +28,7 @@
 {.push raises: [Defect].}
 
 import
+  tables,
   chronicles,
   stew/results,
   ./extras, ./ssz, metrics,
@@ -177,6 +178,8 @@ proc noRollback*(state: var HashedBeaconState) =
 
 proc state_transition*(
     state: var HashedBeaconState, signedBlock: SignedBeaconBlock,
+    # TODO this is ... okay, but not perfect; align with EpochRef
+    stateCache: var StateCache,
     flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} =
   ## Time in the beacon chain moves by slots. Every time (haha.) that happens,
   ## we will update the beacon state. Normally, the state updates will be driven
@@ -209,6 +212,7 @@ proc state_transition*(
   #      the changes in case of failure (look out for `var BeaconState` and
   #      bool return values...)
   doAssert not rollback.isNil, "use noRollback if it's ok to mess up state"
+  doAssert stateCache.shuffled_active_validator_indices.hasKey(state.data.slot.compute_epoch_at_slot)
 
   if not process_slots(state, signedBlock.message.slot, flags):
     rollback(state)
@@ -223,11 +227,11 @@ proc state_transition*(
   if skipBLSValidation in flags or
       verify_block_signature(state.data, signedBlock):
 
-    var per_epoch_cache = get_empty_per_epoch_cache()
+    # TODO after checking scaffolding, remove this
     trace "in state_transition: processing block, signature passed",
       signature = signedBlock.signature,
       blockRoot = hash_tree_root(signedBlock.message)
-    if processBlock(state.data, signedBlock.message, flags, per_epoch_cache):
+    if processBlock(state.data, signedBlock.message, flags, stateCache):
       if skipStateRootValidation in flags or verifyStateRoot(state.data, signedBlock.message):
         # State root is what it should be - we're done!
 
@@ -245,6 +249,18 @@ proc state_transition*(
 
   false
 
+proc state_transition*(
+    state: var HashedBeaconState, signedBlock: SignedBeaconBlock,
+    flags: UpdateFlags, rollback: RollbackHashedProc): bool {.nbench.} =
+  # TODO consider moving this to testutils or similar, since non-testing
+  # and fuzzing code should always be coming from blockpool which should
+  # always be providing cache or equivalent
+  var cache = get_empty_per_epoch_cache()
+  cache.shuffled_active_validator_indices[state.data.slot.compute_epoch_at_slot] =
+    get_shuffled_active_validator_indices(
+      state.data, state.data.slot.compute_epoch_at_slot)
+  state_transition(state, signedBlock, cache, flags, rollback)
+
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md
 # TODO There's more to do here - the spec has helpers that deal set up some of
 #      the fields in here!