rework epoch cache referencing

* collect all epochrefs in specific blocks to make them easier to find
and to avoid lots of small seqs
* reuse validator key databases more aggressively by comparing keys
* make state cache available from within `withState`
* make epochRef available from within onBlockAdded callback
* integrate getEpochInfo into block resolution and epoch ref logic such
that epochrefs are created when blocks are added to pool or lazily when
needed by a getEpochRef
* fill state cache better from EpochRef, speeding up replay and
validation
* store epochRef in specific blocks to make them easier to find and
reuse
* fix database corruption when state is saved while replaying quarantine
* replay slots fully from block pool before processing state
* compare bls values more smartly
* store epoch state without block applied in database - it's recommended
to resync the node!

this branch will drastically speed up processing in times of long
non-finality, as well as cut memory usage by 10x during the recent
medalla madness.

AllTests-mainnet.md

@@ -54,10 +54,11 @@ OK: 7/7 Fail: 0/7 Skip: 0/7
 OK: 5/5 Fail: 0/5 Skip: 0/5
 ## BlockRef and helpers [Preset: mainnet]
 ```diff
++ epochAncestor sanity [Preset: mainnet]                                                     OK
 + get_ancestor sanity [Preset: mainnet]                                                      OK
 + isAncestorOf sanity [Preset: mainnet]                                                      OK
 ```
-OK: 2/2 Fail: 0/2 Skip: 0/2
+OK: 3/3 Fail: 0/3 Skip: 0/3
 ## BlockSlot and helpers [Preset: mainnet]
 ```diff
 + atSlot sanity [Preset: mainnet]                                                            OK
@@ -247,4 +248,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 134/141 Fail: 0/141 Skip: 7/141
+OK: 135/142 Fail: 0/142 Skip: 7/142

beacon_chain/beacon_node.nim

@@ -351,9 +351,8 @@ proc storeBlock(
   {.gcsafe.}: # TODO: fork choice and quarantine should sync via messages instead of callbacks
     let blck = node.chainDag.addRawBlock(node.quarantine, signedBlock) do (
         blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-        state: HashedBeaconState):
+        epochRef: EpochRef, state: HashedBeaconState):
       # Callback add to fork choice if valid
-      let epochRef = getEpochInfo(blckRef, state.data)
       node.attestationPool.addForkChoice(
         epochRef, blckRef, signedBlock.message,
         node.beaconClock.now().slotOrZero())

beacon_chain/block_pools/block_pools_types.nim

@@ -157,12 +157,15 @@ type
 
     slot*: Slot # TODO could calculate this by walking to root, but..
 
-    epochsInfo*: seq[EpochRef] ##\
+    epochRefs*: seq[EpochRef] ##\
     ## Cached information about the epochs starting at this block.
     ## Could be multiple, since blocks could skip slots, but usually, not many
     ## Even if competing forks happen later during this epoch, potential empty
-    ## slots beforehand must all be from this fork. getEpochInfo() is the only
+    ## slots beforehand must all be from this fork. find/getEpochRef() are the
-    ## supported way of accesssing these.
+    ## only supported way of accesssing these.
+    ## In particular, epoch refs are only stored with the last block of the
+    ## parent epoch - this way, it's easy to find them from any block in the
+    ## epoch - including when there are forks that skip the epoch slot.
 
   BlockData* = object
     ## Body and graph in one
@@ -190,7 +193,7 @@ type
 
   OnBlockAdded* = proc(
     blckRef: BlockRef, blck: SignedBeaconBlock,
-    state: HashedBeaconState) {.raises: [Defect], gcsafe.}
+    epochRef: EpochRef, state: HashedBeaconState) {.raises: [Defect], gcsafe.}
 
 template validator_keys*(e: EpochRef): untyped = e.validator_key_store[1][]
 

beacon_chain/block_pools/chain_dag.nim

@@ -33,21 +33,24 @@ proc putBlock*(
   dag.db.putBlock(signedBlock)
 
 proc updateStateData*(
-  dag: ChainDAGRef, state: var StateData, bs: BlockSlot) {.gcsafe.}
+  dag: ChainDAGRef, state: var StateData, bs: BlockSlot,
+  cache: var StateCache) {.gcsafe.}
 
 template withState*(
-    dag: ChainDAGRef, cache: var StateData, blockSlot: BlockSlot, body: untyped): untyped =
+    dag: ChainDAGRef, stateData: var StateData, blockSlot: BlockSlot,
-  ## Helper template that updates state to a particular BlockSlot - usage of
+    body: untyped): untyped =
-  ## cache is unsafe outside of block.
+  ## Helper template that updates stateData to a particular BlockSlot - usage of
-  ## TODO async transformations will lead to a race where cache gets updated
+  ## stateData is unsafe outside of block.
+  ## TODO async transformations will lead to a race where stateData gets updated
   ##      while waiting for future to complete - catch this here somehow?
 
-  updateStateData(dag, cache, blockSlot)
+  var cache {.inject.} = blockSlot.blck.getStateCache(blockSlot.slot.epoch())
+  updateStateData(dag, stateData, blockSlot, cache)
 
-  template hashedState(): HashedBeaconState {.inject, used.} = cache.data
+  template hashedState(): HashedBeaconState {.inject, used.} = stateData.data
-  template state(): BeaconState {.inject, used.} = cache.data.data
+  template state(): BeaconState {.inject, used.} = stateData.data.data
-  template blck(): BlockRef {.inject, used.} = cache.blck
+  template blck(): BlockRef {.inject, used.} = stateData.blck
-  template root(): Eth2Digest {.inject, used.} = cache.data.root
+  template root(): Eth2Digest {.inject, used.} = stateData.data.root
 
   body
 
@@ -74,7 +77,9 @@ func get_effective_balances*(state: BeaconState): seq[Gwei] =
     if validator.is_active_validator(epoch):
       result[i] = validator.effective_balance
 
-proc init*(T: type EpochRef, state: BeaconState, cache: var StateCache, prevEpoch: EpochRef): T =
+proc init*(
+    T: type EpochRef, state: BeaconState, cache: var StateCache,
+    prevEpoch: EpochRef): T =
   let
     epoch = state.get_current_epoch()
     epochRef = EpochRef(
@@ -90,13 +95,35 @@ proc init*(T: type EpochRef, state: BeaconState, cache: var StateCache, prevEpoc
       epochRef.beacon_proposers[i] =
         some((idx.get(), state.validators[idx.get].pubkey))
 
-  if prevEpoch != nil and
+  # Validator sets typically don't change between epochs - a more efficient
-      (prevEpoch.validator_key_store[0] == hash_tree_root(state.validators)):
+  # scheme could be devised where parts of the validator key set is reused
-    # Validator sets typically don't change between epochs - a more efficient
+  # between epochs because in a single history, the validator set only
-    # scheme could be devised where parts of the validator key set is reused
+  # grows - this however is a trivially implementable compromise.
-    # between epochs because in a single history, the validator set only
+
-    # grows - this however is a trivially implementable compromise.
+  # The validators root is cached in the state, so we can quickly compare
-    epochRef.validator_key_store = prevEpoch.validator_key_store
+  # it to see if it remains unchanged - effective balances in the validator
+  # information may however result in a different root, even if the public
+  # keys are the same
+
+  let validators_root = hash_tree_root(state.validators)
+
+  template sameKeys(a: openArray[ValidatorPubKey], b: openArray[Validator]): bool =
+    if a.len != b.len:
+      false
+    else:
+      block:
+        var ret = true
+        for i, key in a:
+          if key != b[i].pubkey:
+            ret = false
+            break
+        ret
+
+  if prevEpoch != nil and (
+    prevEpoch.validator_key_store[0] == hash_tree_root(state.validators) or
+      sameKeys(prevEpoch.validator_key_store[1][], state.validators.asSeq)):
+    epochRef.validator_key_store =
+      (validators_root, prevEpoch.validator_key_store[1])
   else:
     epochRef.validator_key_store = (
       hash_tree_root(state.validators),
@@ -175,70 +202,51 @@ func atEpochEnd*(blck: BlockRef, epoch: Epoch): BlockSlot =
   ## Return the BlockSlot corresponding to the last slot in the given epoch
   atSlot(blck, (epoch + 1).compute_start_slot_at_epoch - 1)
 
-proc getEpochInfo*(blck: BlockRef, state: BeaconState, cache: var StateCache): EpochRef =
+func epochAncestor*(blck: BlockRef, epoch: Epoch): BlockSlot =
-  # This is the only intended mechanism by which to get an EpochRef
+  ## The state transition works by storing information from blocks in a
-  let
+  ## "working" area until the epoch transition, then batching work collected
-    state_epoch = state.get_current_epoch()
+  ## during the epoch. Thus, last block in the ancestor epochs is the block
-    matching_epochinfo = blck.epochsInfo.filterIt(it.epoch == state_epoch)
+  ## that has an impact on epoch currently considered.
+  ##
+  ## This function returns a BlockSlot pointing to that epoch boundary, ie the
+  ## boundary where the last block has been applied to the state and epoch
+  ## processing has been done - we will store epoch caches in that particular
+  ## block so that any block in the dag that needs it can find it easily. In
+  ## particular, if empty slot processing is done, there may be multiple epoch
+  ## caches found there.
+  var blck = blck
+  while blck.slot.epoch >= epoch and not blck.parent.isNil:
+    blck = blck.parent
 
-  if matching_epochinfo.len == 0:
+  blck.atEpochStart(epoch)
-    # When creating an epochref, we can somtimes reuse some of the information
-    # from an earlier epoch in the same history - if we're processing slots
-    # only, the epochref of an earlier slot of the same block will be the most
-    # similar
 
-    var prevEpochRefs = blck.epochsInfo.filterIt(it.epoch < state_epoch)
+proc getStateCache*(blck: BlockRef, epoch: Epoch): StateCache =
-    var prevEpochRef: EpochRef = nil # nil ok
+  # When creating a state cache, we want the current and the previous epoch
-    if prevEpochRefs.len > 0:
+  # information to be preloaded as both of these are used in state transition
-      prevEpochRef = prevEpochRefs[^1]
+  # functions
-    elif state_epoch > 0:
-      let parent = blck.atEpochEnd((state_epoch - 1))
-      if parent.blck != nil and parent.blck.epochsInfo.len > 0:
-        prevEpochRef = parent.blck.epochsInfo[0]
 
-    let epochInfo = EpochRef.init(state, cache, prevEpochRef)
+  var res = StateCache()
+  template load(e: Epoch) =
+    let ancestor = blck.epochAncestor(epoch)
+    for epochRef in ancestor.blck.epochRefs:
+      if epochRef.epoch == e:
+        res.shuffled_active_validator_indices[epochRef.epoch] =
+          epochRef.shuffled_active_validator_indices
 
-    # Don't use BlockRef caching as far as the epoch where the active
+        if epochRef.epoch == epoch:
-    # validator indices can diverge.
+          for i, idx in epochRef.beacon_proposers:
-    if (compute_activation_exit_epoch(blck.slot.compute_epoch_at_slot) >
+            res.beacon_proposer_indices[
-        state_epoch):
+              epoch.compute_start_slot_at_epoch + i] =
-      blck.epochsInfo.add(epochInfo)
+                if idx.isSome: some(idx.get()[0]) else: none(ValidatorIndex)
-    trace "chain_dag.getEpochInfo: back-filling parent.epochInfo",
-      state_slot = state.slot
-    epochInfo
-  elif matching_epochinfo.len == 1:
-    matching_epochinfo[0]
-  else:
-    raiseAssert "multiple EpochRefs per epoch per BlockRef invalid"
 
-proc getEpochInfo*(blck: BlockRef, state: BeaconState): EpochRef =
+        break
-  # This is the only intended mechanism by which to get an EpochRef
-  var cache = StateCache()
-  getEpochInfo(blck, state, cache)
 
-proc getEpochCache*(blck: BlockRef, state: BeaconState): StateCache =
+  load(epoch)
-  var tmp = StateCache() # TODO Resolve circular init issue
-  let epochInfo = getEpochInfo(blck, state, tmp)
-  if epochInfo.epoch > 0:
-    # When doing state transitioning, both the current and previous epochs are
-    # useful from a cache perspective since attestations may come from either -
-    # we'll use the last slot from the epoch because it is more likely to
-    # be filled in already, compared to the first slot where the block might
-    # be from the epoch before.
-    let
-      prevEpochBlck = blck.atEpochEnd(epochInfo.epoch - 1).blck
 
-    for ei in prevEpochBlck.epochsInfo:
+  if epoch > 0:
-      if ei.epoch == epochInfo.epoch - 1:
+    load(epoch - 1)
-        result.shuffled_active_validator_indices[ei.epoch] =
-          ei.shuffled_active_validator_indices
 
-  result.shuffled_active_validator_indices[state.get_current_epoch()] =
+  res
-      epochInfo.shuffled_active_validator_indices
-  for i, idx in epochInfo.beacon_proposers:
-    result.beacon_proposer_indices[
-      epochInfo.epoch.compute_start_slot_at_epoch + i] =
-        if idx.isSome: some(idx.get()[0]) else: none(ValidatorIndex)
 
 func init(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
   BlockRef(
@@ -299,24 +307,31 @@ proc init*(T: type ChainDAGRef,
     headRef = tailRef
 
   var
-    bs = headRef.atSlot(headRef.slot)
+    cur = headRef.atSlot(headRef.slot)
     tmpState = (ref StateData)()
 
   # Now that we have a head block, we need to find the most recent state that
   # we have saved in the database
-  while bs.blck != nil:
+  while cur.blck != nil:
-    let root = db.getStateRoot(bs.blck.root, bs.slot)
+    let root = db.getStateRoot(cur.blck.root, cur.slot)
     if root.isSome():
       # TODO load StateData from BeaconChainDB
       # We save state root separately for empty slots which means we might
       # sometimes not find a state even though we saved its state root
       if db.getState(root.get(), tmpState.data.data, noRollback):
         tmpState.data.root = root.get()
-        tmpState.blck = bs.blck
+        tmpState.blck = cur.blck
 
         break
 
-    bs = bs.parent() # Iterate slot by slot in case there's a gap!
+    if cur.blck.parent != nil and
+        cur.blck.slot.epoch != epoch(cur.blck.parent.slot):
+      # We store the state of the parent block with the epoch processing applied
+      # in the database!
+      cur = cur.blck.parent.atEpochStart(cur.blck.slot.epoch)
+    else:
+      # Moves back slot by slot, in case a state for an empty slot was saved
+      cur = cur.parent
 
   if tmpState.blck == nil:
     warn "No state found in head history, database corrupt?"
@@ -324,19 +339,10 @@ proc init*(T: type ChainDAGRef,
     #      would be a good recovery model?
     raiseAssert "No state found in head history, database corrupt?"
 
-  # We presently save states on the epoch boundary - it means that the latest
-  # state we loaded might be older than head block - nonetheless, it will be
-  # from the same epoch as the head, thus the finalized and justified slots are
-  # the same - these only change on epoch boundaries.
-  let
-    finalizedHead = headRef.atEpochStart(
-      tmpState.data.data.finalized_checkpoint.epoch)
-
   let res = ChainDAGRef(
     blocks: blocks,
     tail: tailRef,
     head: headRef,
-    finalizedHead: finalizedHead,
     db: db,
     heads: @[headRef],
     headState: tmpState[],
@@ -351,36 +357,50 @@ proc init*(T: type ChainDAGRef,
 
   doAssert res.updateFlags in [{}, {verifyFinalization}]
 
-  res.updateStateData(res.headState, headRef.atSlot(headRef.slot))
+  var cache: StateCache
+  res.updateStateData(res.headState, headRef.atSlot(headRef.slot), cache)
+    # We presently save states on the epoch boundary - it means that the latest
+  # state we loaded might be older than head block - nonetheless, it will be
+  # from the same epoch as the head, thus the finalized and justified slots are
+  # the same - these only change on epoch boundaries.
+  res.finalizedHead = headRef.atEpochStart(
+      res.headState.data.data.finalized_checkpoint.epoch)
+
   res.clearanceState = res.headState
 
   info "Block dag initialized",
     head = shortLog(headRef),
-    finalizedHead = shortLog(finalizedHead),
+    finalizedHead = shortLog(res.finalizedHead),
     tail = shortLog(tailRef),
     totalBlocks = blocks.len
 
   res
 
-proc getEpochRef*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef =
+proc findEpochRef*(blck: BlockRef, epoch: Epoch): EpochRef = # may return nil!
-  var bs = blck.atEpochEnd(epoch)
+  let ancestor = blck.epochAncestor(epoch)
+  for epochRef in ancestor.blck.epochRefs:
+    if epochRef.epoch == epoch:
+      return epochRef
 
-  while true:
+proc getEpochRef*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef =
-    # Any block from within the same epoch will carry the same epochinfo, so
+  let epochRef = blck.findEpochRef(epoch)
-    # we start at the most recent one
+  if epochRef != nil:
-    for e in bs.blck.epochsInfo:
+      beacon_state_data_cache_hits.inc
-      if e.epoch == epoch:
+      return epochRef
-        beacon_state_data_cache_hits.inc
-        return e
-    if bs.slot == epoch.compute_start_slot_at_epoch:
-      break
-    bs = bs.parent
 
   beacon_state_data_cache_misses.inc
 
-  dag.withState(dag.tmpState, bs):
+  let
-    var cache = StateCache()
+    ancestor = blck.epochAncestor(epoch)
-    getEpochInfo(blck, state, cache)
+
+  dag.withState(dag.tmpState, ancestor):
+    let
+      prevEpochRef = blck.findEpochRef(epoch - 1)
+      newEpochRef = EpochRef.init(state, cache, prevEpochRef)
+
+    # TODO consider constraining the number of epochrefs per state
+    ancestor.blck.epochRefs.add newEpochRef
+    newEpochRef
 
 proc getState(
     dag: ChainDAGRef, state: var StateData, stateRoot: Eth2Digest,
@@ -412,6 +432,10 @@ proc getState(dag: ChainDAGRef, state: var StateData, bs: BlockSlot): bool =
   if not bs.slot.isEpoch:
     return false # We only ever save epoch states - no need to hit database
 
+  # TODO earlier versions would store the epoch state with a the epoch block
+  #      applied - we generally shouldn't hit the database for such states but
+  #      will do so in a transitionary upgrade period!
+
   if (let stateRoot = dag.db.getStateRoot(bs.blck.root, bs.slot);
       stateRoot.isSome()):
     return dag.getState(state, stateRoot.get(), bs.blck)
@@ -425,10 +449,15 @@ proc putState*(dag: ChainDAGRef, state: StateData) =
   #      we could easily see a state explosion
   logScope: pcs = "save_state_at_epoch_start"
 
+  # As a policy, we only store epoch boundary states without the epoch block
+  # (if it exists) applied - the rest can be reconstructed by loading an epoch
+  # boundary state and applying the missing blocks
   if not state.data.data.slot.isEpoch:
-    # As a policy, we only store epoch boundary states - the rest can be
+    trace "Not storing non-epoch state"
-    # reconstructed by loading an epoch boundary state and applying the
+    return
-    # missing blocks
+
+  if state.data.data.slot <= state.blck.slot:
+    trace "Not storing epoch state with block already applied"
     return
 
   if dag.db.containsState(state.data.root):
@@ -522,9 +551,9 @@ proc advanceSlots(
   # processing
   doAssert state.data.data.slot <= slot
 
+  var cache = getStateCache(state.blck, state.data.data.slot.epoch)
   while state.data.data.slot < slot:
     # Process slots one at a time in case afterUpdate needs to see empty states
-    var cache = getEpochCache(state.blck, state.data.data)
     advance_slot(state.data, dag.updateFlags, cache)
 
     if save:
@@ -540,18 +569,17 @@ proc applyBlock(
 
   # `state_transition` can handle empty slots, but we want to potentially save
   # some of the empty slot states
-  dag.advanceSlots(state, blck.data.message.slot - 1, save)
+  dag.advanceSlots(state, blck.data.message.slot, save)
 
   var statePtr = unsafeAddr state # safe because `restore` is locally scoped
   func restore(v: var HashedBeaconState) =
     doAssert (addr(statePtr.data) == addr v)
     statePtr[] = dag.headState
 
-  var cache = getEpochCache(blck.refs, state.data.data)
+  var cache = getStateCache(state.blck, state.data.data.slot.epoch)
-
   let ok = state_transition(
     dag.runtimePreset, state.data, blck.data,
-    cache, flags + dag.updateFlags, restore)
+    cache, flags + dag.updateFlags + {slotProcessed}, restore)
   if ok:
     state.blck = blck.refs
     dag.putState(state)
@@ -559,7 +587,8 @@ proc applyBlock(
   ok
 
 proc updateStateData*(
-    dag: ChainDAGRef, state: var StateData, bs: BlockSlot) =
+    dag: ChainDAGRef, state: var StateData, bs: BlockSlot,
+    cache: var StateCache) =
   ## Rewind or advance state such that it matches the given block and slot -
   ## this may include replaying from an earlier snapshot if blck is on a
   ## different branch or has advanced to a higher slot number than slot
@@ -590,14 +619,24 @@ proc updateStateData*(
   while not dag.getState(state, cur):
     # There's no state saved for this particular BlockSlot combination, keep
     # looking...
-    if cur.slot == cur.blck.slot:
+    if cur.blck.parent != nil and
-      # This is not an empty slot, so the block will need to be applied to
+        cur.blck.slot.epoch != epoch(cur.blck.parent.slot):
-      # eventually reach bs
+      # We store the state of the parent block with the epoch processing applied
+      # in the database - we'll need to apply the block however!
       ancestors.add(cur.blck)
+      cur = cur.blck.parent.atEpochStart(cur.blck.slot.epoch)
+    else:
+      if cur.slot == cur.blck.slot:
+        # This is not an empty slot, so the block will need to be applied to
+        # eventually reach bs
+        ancestors.add(cur.blck)
 
-    # Moves back slot by slot, in case a state for an empty slot was saved
+      # Moves back slot by slot, in case a state for an empty slot was saved
-    cur = cur.parent
+      cur = cur.parent
 
+  let
+    startSlot = state.data.data.slot
+    startRoot = state.data.root
   # Time to replay all the blocks between then and now
   for i in countdown(ancestors.len - 1, 0):
     # Because the ancestors are in the database, there's no need to persist them
@@ -615,7 +654,11 @@ proc updateStateData*(
   beacon_state_rewinds.inc()
 
   debug "State reloaded from database",
-    blocks = ancestors.len, stateRoot = shortLog(state.data.root),
+    blocks = ancestors.len,
+    slots = state.data.data.slot - startSlot,
+    stateRoot = shortLog(state.data.root),
+    stateSlot = state.data.data.slot,
+    stateRoot = shortLog(startRoot),
     blck = shortLog(bs)
 
 proc loadTailState*(dag: ChainDAGRef): StateData =
@@ -662,8 +705,9 @@ proc updateHead*(dag: ChainDAGRef, newHead: BlockRef) =
       dag.clearanceState.data.data.slot == newHead.slot:
     assign(dag.headState, dag.clearanceState)
   else:
+    var cache = getStateCache(newHead, newHead.slot.epoch())
     updateStateData(
-      dag, dag.headState, newHead.atSlot(newHead.slot))
+      dag, dag.headState, newHead.atSlot(newHead.slot), cache)
 
   dag.head = newHead
 
@@ -744,7 +788,7 @@ proc updateHead*(dag: ChainDAGRef, newHead: BlockRef) =
       while tmp != dag.finalizedHead.blck:
         # leave the epoch cache in the last block of the epoch..
         tmp = tmp.parent
-        tmp.epochsInfo = @[]
+        tmp.epochRefs = @[]
 
     dag.finalizedHead = finalizedHead
 

beacon_chain/block_pools/clearance.nim

@@ -8,7 +8,7 @@
 {.push raises: [Defect].}
 
 import
-  std/[sequtils, tables],
+  std/[tables],
   chronicles,
   metrics, stew/results,
   ../extras,
@@ -42,13 +42,16 @@ proc addRawBlock*(
 
 proc addResolvedBlock(
        dag: var ChainDAGRef, quarantine: var QuarantineRef,
-       state: HashedBeaconState, signedBlock: SignedBeaconBlock,
+       state: var StateData, signedBlock: SignedBeaconBlock,
        parent: BlockRef, cache: var StateCache,
        onBlockAdded: OnBlockAdded
-     ): BlockRef =
+     ) =
   # TODO move quarantine processing out of here
   logScope: pcs = "block_resolution"
-  doAssert state.data.slot == signedBlock.message.slot, "state must match block"
+  doAssert state.data.data.slot == signedBlock.message.slot,
+    "state must match block"
+  doAssert state.blck.root == signedBlock.message.parent_root,
+    "the StateData passed into the addResolved function not yet updated!"
 
   let
     blockRoot = signedBlock.root
@@ -57,14 +60,12 @@ proc addResolvedBlock(
 
   link(parent, blockRef)
 
-  if parent.slot.compute_epoch_at_slot() == blockEpoch:
+  var epochRef = blockRef.findEpochRef(blockEpoch)
-    # If the parent and child blocks are from the same epoch, we can reuse
+  if epochRef == nil:
-    # the epoch cache - but we'll only use the current epoch because the new
+    let prevEpochRef = blockRef.findEpochRef(blockEpoch - 1)
-    # block might have affected what the next epoch looks like
+
-    blockRef.epochsInfo = filterIt(parent.epochsInfo, it.epoch == blockEpoch)
+    epochRef = EpochRef.init(state.data.data, cache, prevEpochRef)
-  else:
+    blockRef.epochAncestor(blockEpoch).blck.epochRefs.add epochRef
-    # Ensure we collect the epoch info if it's missing
-    discard getEpochInfo(blockRef, state.data, cache)
 
   dag.blocks[blockRoot] = blockRef
   trace "Populating block dag", key = blockRoot, val = blockRef
@@ -90,10 +91,12 @@ proc addResolvedBlock(
     blockRoot = shortLog(blockRoot),
     heads = dag.heads.len()
 
+  state.blck = blockRef
+
   # Notify others of the new block before processing the quarantine, such that
   # notifications for parents happens before those of the children
   if onBlockAdded != nil:
-    onBlockAdded(blockRef, signedBlock, state)
+    onBlockAdded(blockRef, signedBlock, epochRef, state.data)
 
   # Now that we have the new block, we should see if any of the previously
   # unresolved blocks magically become resolved
@@ -115,8 +118,6 @@ proc addResolvedBlock(
       for v in resolved:
         discard addRawBlock(dag, quarantine, v, onBlockAdded)
 
-  blockRef
-
 proc addRawBlock*(
        dag: var ChainDAGRef, quarantine: var QuarantineRef,
        signedBlock: SignedBeaconBlock,
@@ -190,8 +191,9 @@ proc addRawBlock*(
 
     # TODO if the block is from the future, we should not be resolving it (yet),
     #      but maybe we should use it as a hint that our clock is wrong?
+    var cache = getStateCache(parent, blck.slot.epoch)
     updateStateData(
-      dag, dag.clearanceState, BlockSlot(blck: parent, slot: blck.slot - 1))
+      dag, dag.clearanceState, parent.atSlot(blck.slot), cache)
 
     let
       poolPtr = unsafeAddr dag # safe because restore is short-lived
@@ -202,21 +204,17 @@ proc addRawBlock*(
       doAssert v.addr == addr poolPtr.clearanceState.data
       assign(poolPtr.clearanceState, poolPtr.headState)
 
-    var cache = getEpochCache(parent, dag.clearanceState.data.data)
     if not state_transition(dag.runtimePreset, dag.clearanceState.data, signedBlock,
-                            cache, dag.updateFlags, restore):
+                            cache, dag.updateFlags + {slotProcessed}, restore):
       notice "Invalid block"
 
       return err Invalid
 
     # Careful, clearanceState.data has been updated but not blck - we need to
     # create the BlockRef first!
-    dag.clearanceState.blck = addResolvedBlock(
+    addResolvedBlock(
-      dag, quarantine, dag.clearanceState.data, signedBlock, parent, cache,
+      dag, quarantine, dag.clearanceState, signedBlock, parent, cache,
-      onBlockAdded
+      onBlockAdded)
-    )
-
-    dag.putState(dag.clearanceState)
 
     return ok dag.clearanceState.blck
 

beacon_chain/extras.nim

@@ -27,5 +27,8 @@ type
     skipStateRootValidation ##\
     ## Skip verification of block state root.
     verifyFinalization
+    slotProcessed ##\
+    ## Allow blocks to be applied to states with the same slot number as the
+    ## block which is what happens when `process_block` is called separately
 
   UpdateFlags* = set[UpdateFlag]

beacon_chain/spec/crypto.nim

@@ -79,19 +79,6 @@ type
 
 export AggregateSignature
 
-func `==`*(a, b: BlsValue): bool =
-  if a.kind != b.kind: return false
-  if a.kind == Real:
-    return a.blsValue == b.blsValue
-  else:
-    return a.blob == b.blob
-
-template `==`*[N, T](a: BlsValue[N, T], b: T): bool =
-  a.blsValue == b
-
-template `==`*[N, T](a: T, b: BlsValue[N, T]): bool =
-  a == b.blsValue
-
 # API
 # ----------------------------------------------------------------------
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/beacon-chain.md#bls-signatures
@@ -225,10 +212,9 @@ func `$`*(x: ValidatorPrivKey): string =
 func `$`*(x: BlsValue): string =
   # The prefix must be short
   # due to the mechanics of the `shortLog` function.
-  if x.kind == Real:
+  case x.kind
-    x.blsValue.toHex()
+  of Real: x.blsValue.toHex()
-  else:
+  of OpaqueBlob: "r:" & x.blob.toHex()
-    "raw: " & x.blob.toHex()
 
 func toRaw*(x: ValidatorPrivKey): array[32, byte] =
   # TODO: distinct type - see https://github.com/status-im/nim-blscurve/pull/67
@@ -278,6 +264,20 @@ func fromHex*(T: type BlsCurveType, hexStr: string): BlsResult[T] {.inline.} =
   except ValueError:
     err "bls: cannot parse value"
 
+func `==`*(a, b: BlsValue): bool =
+  # The assumption here is that converting to raw is mostly fast!
+  case a.kind
+  of Real:
+    if a.kind == b.kind:
+      a.blsValue == b.blsValue
+    else:
+      a.toRaw() == b.blob
+  of OpaqueBlob:
+    if a.kind == b.kind:
+      a.blob == b.blob
+    else:
+      a.blob == b.toRaw()
+
 # Hashing
 # ----------------------------------------------------------------------
 
@@ -348,7 +348,7 @@ func shortLog*(x: BlsValue): string =
   if x.kind == Real:
     x.blsValue.exportRaw()[0..3].toHex()
   else:
-    "raw: " & x.blob[0..3].toHex()
+    "r:" & x.blob[0..3].toHex()
 
 func shortLog*(x: ValidatorPrivKey): string =
   ## Logging for raw unwrapped BLS types
@@ -369,7 +369,6 @@ func init*(T: typedesc[ValidatorPrivKey], hex: string): T {.noInit, raises: [Val
     raise (ref ValueError)(msg: $v.error)
   v[]
 
-
 # For mainchain monitor
 func init*(T: typedesc[ValidatorPubKey], data: array[RawPubKeySize, byte]): T {.noInit, raises: [ValueError, Defect].} =
   let v = T.fromRaw(data)

beacon_chain/spec/state_transition.nim

@@ -163,13 +163,14 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot,
   #      slots "automatically" in `state_transition`, perhaps it would be better
   #      to keep a pre-condition that state must be at the right slot already?
   if not (state.data.slot < slot):
-    notice(
+    if slotProcessed notin updateFlags or state.data.slot != slot:
-      "Unusual request for a slot in the past",
+      notice(
-      state_root = shortLog(state.root),
+        "Unusual request for a slot in the past",
-      current_slot = state.data.slot,
+        state_root = shortLog(state.root),
-      target_slot = slot
+        current_slot = state.data.slot,
-    )
+        target_slot = slot
-    return false
+      )
+      return false
 
   # Catch up to the target slot
   var cache = StateCache()

beacon_chain/validator_api.nim

@@ -229,8 +229,6 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
       stateId: string, epoch: uint64, index: uint64, slot: uint64) ->
       seq[BeaconStatesCommitteesTuple]:
     withStateForStateId(stateId):
-      var cache = StateCache() # TODO is this OK?
-
       proc getCommittee(slot: Slot, index: CommitteeIndex): BeaconStatesCommitteesTuple =
         let vals = get_beacon_committee(state, slot, index, cache).mapIt(it.uint64)
         return (index: index.uint64, slot: slot.uint64, validators: vals)

beacon_chain/validator_duties.nim

@@ -205,7 +205,6 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
       doAssert v.addr == addr poolPtr.tmpState.data
       assign(poolPtr.tmpState, poolPtr.headState)
 
-    var cache = StateCache()
     let message = makeBeaconBlock(
       node.config.runtimePreset,
       hashedState,
@@ -237,9 +236,8 @@ proc proposeSignedBlock*(node: BeaconNode,
     let newBlockRef = node.chainDag.addRawBlock(node.quarantine,
                                                      newBlock) do (
         blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-        state: HashedBeaconState):
+        epochRef: EpochRef, state: HashedBeaconState):
       # Callback add to fork choice if valid
-      let epochRef = getEpochInfo(blckRef, state.data)
       node.attestationPool.addForkChoice(
         epochRef, blckRef, signedBlock.message,
         node.beaconClock.now().slotOrZero())
@@ -402,7 +400,6 @@ proc broadcastAggregatedAttestations(
 
   let bs = BlockSlot(blck: aggregationHead, slot: aggregationSlot)
   node.chainDag.withState(node.chainDag.tmpState, bs):
-    var cache = getEpochCache(aggregationHead, state)
     let
       committees_per_slot =
         get_committee_count_per_slot(state, aggregationSlot.epoch, cache)

research/block_sim.nim

@@ -70,7 +70,6 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
       attestationHead = chainDag.head.atSlot(slot)
 
     chainDag.withState(chainDag.tmpState, attestationHead):
-      var cache = getEpochCache(attestationHead.blck, state)
       let committees_per_slot =
         get_committee_count_per_slot(state, slot.epoch, cache)
 
@@ -104,8 +103,6 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
       head = chainDag.head
 
     chainDag.withState(chainDag.tmpState, head.atSlot(slot)):
-      var cache = StateCache()
-
       let
         proposerIdx = get_beacon_proposer_index(state, cache).get()
         privKey = hackPrivKey(state.validators[proposerIdx])
@@ -140,9 +137,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
 
       let added = chainDag.addRawBlock(quarantine, newBlock) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         attPool.addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
       blck() = added[]
@@ -175,8 +171,9 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
 
   if replay:
     withTimer(timers[tReplay]):
+      var cache = StateCache()
       chainDag.updateStateData(
-        replayState[], chainDag.head.atSlot(Slot(slots)))
+        replayState[], chainDag.head.atSlot(Slot(slots)), cache)
 
   echo "Done!"
 

tests/test_attestation_pool.nim

@@ -185,9 +185,8 @@ suiteReport "Attestation pool processing" & preset():
       b1 = addTestBlock(state.data, chainDag.tail.root, cache)
       b1Add = chainDag.addRawBlock(quarantine, b1) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     let head = pool[].selectHead(b1Add[].slot)
@@ -199,9 +198,8 @@ suiteReport "Attestation pool processing" & preset():
       b2 = addTestBlock(state.data, b1.root, cache)
       b2Add = chainDag.addRawBlock(quarantine, b2) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     let head2 = pool[].selectHead(b2Add[].slot)
@@ -215,9 +213,8 @@ suiteReport "Attestation pool processing" & preset():
       b10 = makeTestBlock(state.data, chainDag.tail.root, cache)
       b10Add = chainDag.addRawBlock(quarantine, b10) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     let head = pool[].selectHead(b10Add[].slot)
@@ -231,9 +228,8 @@ suiteReport "Attestation pool processing" & preset():
       )
       b11Add = chainDag.addRawBlock(quarantine, b11) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
       bc1 = get_beacon_committee(
@@ -274,9 +270,8 @@ suiteReport "Attestation pool processing" & preset():
       b10 = makeTestBlock(state.data, chainDag.tail.root, cache)
       b10Add = chainDag.addRawBlock(quarantine, b10) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     let head = pool[].selectHead(b10Add[].slot)
@@ -289,9 +284,8 @@ suiteReport "Attestation pool processing" & preset():
     let b10_clone = b10 # Assumes deep copy
     let b10Add_clone = chainDag.addRawBlock(quarantine, b10_clone) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     doAssert: b10Add_clone.error == Duplicate
@@ -304,9 +298,8 @@ suiteReport "Attestation pool processing" & preset():
       b10 = makeTestBlock(state.data, chainDag.tail.root, cache)
       b10Add = chainDag.addRawBlock(quarantine, b10) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     let head = pool[].selectHead(b10Add[].slot)
@@ -339,9 +332,8 @@ suiteReport "Attestation pool processing" & preset():
         block_root = new_block.root
         let blockRef = chainDag.addRawBlock(quarantine, new_block) do (
             blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-            state: HashedBeaconState):
+            epochRef: EpochRef, state: HashedBeaconState):
           # Callback add to fork choice if valid
-          let epochRef = getEpochInfo(blckRef, state.data)
           pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
         let head = pool[].selectHead(blockRef[].slot)
@@ -381,9 +373,8 @@ suiteReport "Attestation pool processing" & preset():
     # Add back the old block to ensure we have a duplicate error
     let b10Add_clone = chainDag.addRawBlock(quarantine, b10_clone) do (
           blckRef: BlockRef, signedBlock: SignedBeaconBlock,
-          state: HashedBeaconState):
+          epochRef: EpochRef, state: HashedBeaconState):
         # Callback add to fork choice if valid
-        let epochRef = getEpochInfo(blckRef, state.data)
         pool[].addForkChoice(epochRef, blckRef, signedBlock.message, blckRef.slot)
 
     doAssert: b10Add_clone.error == Duplicate

tests/test_block_pool.nim

@@ -55,6 +55,22 @@ suiteReport "BlockRef and helpers" & preset():
       s4.get_ancestor(Slot(3)) == s2
       s4.get_ancestor(Slot(4)) == s4
 
+  timedTest "epochAncestor sanity" & preset():
+    let
+      s0 = BlockRef(slot: Slot(0))
+    var cur = s0
+    for i in 1..SLOTS_PER_EPOCH * 2:
+      cur = BlockRef(slot: Slot(i), parent: cur)
+
+    let ancestor = cur.epochAncestor(cur.slot.epoch)
+
+    check:
+      ancestor.slot.epoch == cur.slot.epoch
+      ancestor.blck != cur # should have selected a parent
+
+      ancestor.blck.epochAncestor(cur.slot.epoch) == ancestor
+      ancestor.blck.epochAncestor(ancestor.blck.slot.epoch) != ancestor
+
 suiteReport "BlockSlot and helpers" & preset():
   timedTest "atSlot sanity" & preset():
     let
@@ -98,7 +114,6 @@ suiteReport "Block pool processing" & preset():
       b1Root = hash_tree_root(b1.message)
       b2 = addTestBlock(stateData.data, b1Root, cache)
       b2Root {.used.} = hash_tree_root(b2.message)
-
   timedTest "getRef returns nil for missing blocks":
     check:
       dag.getRef(default Eth2Digest) == nil
@@ -132,9 +147,10 @@ suiteReport "Block pool processing" & preset():
       b2Add[].root == b2Get.get().refs.root
       dag.heads.len == 1
       dag.heads[0] == b2Add[]
-      # both should have the same epoch ref instance because they're from the
+      not b1Add[].findEpochRef(b1Add[].slot.epoch).isNil
-      # same epoch
+      b1Add[].findEpochRef(b1Add[].slot.epoch) ==
-      addr(b2Add[].epochsInfo[0][]) == addr(b1Add[].epochsInfo[0][])
+        b2Add[].findEpochRef(b2Add[].slot.epoch)
+      b1Add[].findEpochRef(b1Add[].slot.epoch + 1).isNil
 
     # Skip one slot to get a gap
     check:
@@ -249,39 +265,40 @@ suiteReport "Block pool processing" & preset():
     var tmpState = assignClone(dag.headState)
 
     # move to specific block
-    dag.updateStateData(tmpState[], bs1)
+    var cache = StateCache()
+    dag.updateStateData(tmpState[], bs1, cache)
 
     check:
       tmpState.blck == b1Add[]
       tmpState.data.data.slot == bs1.slot
 
     # Skip slots
-    dag.updateStateData(tmpState[], bs1_3) # skip slots
+    dag.updateStateData(tmpState[], bs1_3, cache) # skip slots
 
     check:
       tmpState.blck == b1Add[]
       tmpState.data.data.slot == bs1_3.slot
 
     # Move back slots, but not blocks
-    dag.updateStateData(tmpState[], bs1_3.parent())
+    dag.updateStateData(tmpState[], bs1_3.parent(), cache)
     check:
       tmpState.blck == b1Add[]
       tmpState.data.data.slot == bs1_3.parent().slot
 
     # Move to different block and slot
-    dag.updateStateData(tmpState[], bs2_3)
+    dag.updateStateData(tmpState[], bs2_3, cache)
     check:
       tmpState.blck == b2Add[]
       tmpState.data.data.slot == bs2_3.slot
 
     # Move back slot and block
-    dag.updateStateData(tmpState[], bs1)
+    dag.updateStateData(tmpState[], bs1, cache)
     check:
       tmpState.blck == b1Add[]
       tmpState.data.data.slot == bs1.slot
 
     # Move back to genesis
-    dag.updateStateData(tmpState[], bs1.parent())
+    dag.updateStateData(tmpState[], bs1.parent(), cache)
     check:
       tmpState.blck == b1Add[].parent
       tmpState.data.data.slot == bs1.parent.slot
@@ -328,10 +345,12 @@ suiteReport "chain DAG finalization tests" & preset():
 
       # Epochrefs should share validator key set when the validator set is
       # stable
-      addr(dag.heads[0].epochsInfo[0].validator_key_store[1][]) ==
+      not dag.heads[0].findEpochRef(dag.heads[0].slot.epoch).isNil
-        addr(dag.heads[0].atEpochEnd(
+      not dag.heads[0].findEpochRef(dag.heads[0].slot.epoch - 1).isNil
-          dag.heads[0].slot.compute_epoch_at_slot() - 1).
+      dag.heads[0].findEpochRef(dag.heads[0].slot.epoch) !=
-            blck.epochsInfo[0].validator_key_store[1][])
+        dag.heads[0].findEpochRef(dag.heads[0].slot.epoch - 1)
+      dag.heads[0].findEpochRef(dag.heads[0].slot.epoch).validator_key_store[1] ==
+        dag.heads[0].findEpochRef(dag.heads[0].slot.epoch - 1).validator_key_store[1]
 
     block:
       # The late block is a block whose parent was finalized long ago and thus