era: Capella+ support (fixes #4752) (#4853)

Post-Capella, historical roots are computed from historical summaries
instead of being directly stored in the beacon state.

Slightly messy to pass both lists around - this is done to avoid
computing the historical root unnecessarily.

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -238,6 +238,7 @@ proc containsBlock(dag: ChainDAGRef, bid: BlockId): bool =
   (bid.slot <= dag.finalizedHead.slot and
     getBlockSZ(
       dag.era, getStateField(dag.headState, historical_roots).asSeq,
+      dag.headState.historical_summaries().asSeq,
       bid.slot, bytes).isOk and bytes.len > 0)
 
 proc getBlock*(
@@ -246,6 +247,7 @@ proc getBlock*(
   dag.db.getBlock(bid.root, T) or
     getBlock(
       dag.era, getStateField(dag.headState, historical_roots).asSeq,
+      dag.headState.historical_summaries().asSeq,
       bid.slot, Opt[Eth2Digest].ok(bid.root), T)
 
 proc getBlockSSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool =
@@ -256,6 +258,7 @@ proc getBlockSSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool =
     (bid.slot <= dag.finalizedHead.slot and
       getBlockSSZ(
         dag.era, getStateField(dag.headState, historical_roots).asSeq,
+        dag.headState.historical_summaries().asSeq,
         bid.slot, bytes).isOk)
 
 proc getBlockSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool =
@@ -268,6 +271,7 @@ proc getBlockSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool =
     (bid.slot <= dag.finalizedHead.slot and
       getBlockSZ(
         dag.era, getStateField(dag.headState, historical_roots).asSeq,
+        dag.headState.historical_summaries().asSeq,
         bid.slot, bytes).isOk)
 
 proc getForkedBlock*(
@@ -280,6 +284,7 @@ proc getForkedBlock*(
     blck = getBlock(dag, bid, T).valueOr:
         getBlock(
             dag.era, getStateField(dag.headState, historical_roots).asSeq,
+            dag.headState.historical_summaries().asSeq,
             bid.slot, Opt[Eth2Digest].ok(bid.root), T).valueOr:
           result.err()
           return
@@ -1152,13 +1157,15 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
 
     let
       historical_roots = getStateField(dag.headState, historical_roots).asSeq()
+      historical_summaries = dag.headState.historical_summaries.asSeq()
 
     var
       blocks = 0
 
     # Here, we'll build up the slot->root mapping in memory for the range of
     # blocks from genesis to backfill, if possible.
-    for bid in dag.era.getBlockIds(historical_roots, Slot(0), Eth2Digest()):
+    for bid in dag.era.getBlockIds(
+        historical_roots, historical_summaries, Slot(0), Eth2Digest()):
       if bid.slot >= dag.backfill.slot:
         # If we end up in here, we failed the root comparison just below in
         # an earlier iteration
@@ -2394,6 +2401,7 @@ proc rebuildIndex*(dag: ChainDAGRef) =
   let
     roots = dag.db.loadStateRoots()
     historicalRoots = getStateField(dag.headState, historical_roots).asSeq()
+    historicalSummaries = dag.headState.historical_summaries.asSeq()
 
   var
     canonical = newSeq[Eth2Digest](
@@ -2456,7 +2464,8 @@ proc rebuildIndex*(dag: ChainDAGRef) =
         # If we can find an era file with this state, use it as an alternative
         # starting point - ignore failures for now
         var bytes: seq[byte]
-        if dag.era.getState(historicalRoots, slot, state[]).isOk():
+        if dag.era.getState(
+            historicalRoots, historicalSummaries, slot, state[]).isOk():
           state_root = getStateRoot(state[])
 
           withState(state[]): dag.db.putState(forkyState)

beacon_chain/era_db.nim

@@ -234,7 +234,8 @@ proc verify*(f: EraFile, cfg: RuntimeConfig): Result[Eth2Digest, string] =
   ok(getStateRoot(state[]))
 
 proc getEraFile(
-    db: EraDB, historical_roots: openArray[Eth2Digest], era: Era):
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], era: Era):
     Result[EraFile, string] =
   for f in db.files:
     if f.stateIdx.startSlot.era == era:
@@ -242,7 +243,8 @@ proc getEraFile(
 
   let
     eraRoot = eraRoot(
-        db.genesis_validators_root, historical_roots, era).valueOr:
+        db.genesis_validators_root, historical_roots, historical_summaries,
+        era).valueOr:
       return err("Era outside of known history")
     name = eraFileName(db.cfg, era, eraRoot)
     path = db.path / name
@@ -265,7 +267,8 @@ proc getEraFile(
   ok(f)
 
 proc getBlockSZ*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     bytes: var seq[byte]): Result[void, string] =
   ## Get a snappy-frame-compressed version of the block data - may overwrite
   ## `bytes` on error
@@ -276,23 +279,26 @@ proc getBlockSZ*(
   # Block content for the blocks of an era is found in the file for the _next_
   # era
   let
-    f = ? db.getEraFile(historical_roots, slot.era + 1)
+    f = ? db.getEraFile(historical_roots, historical_summaries, slot.era + 1)
 
   f.getBlockSZ(slot, bytes)
 
 proc getBlockSSZ*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     bytes: var seq[byte]): Result[void, string] =
   let
-    f = ? db.getEraFile(historical_roots, slot.era + 1)
+    f = ? db.getEraFile(historical_roots, historical_summaries, slot.era + 1)
 
   f.getBlockSSZ(slot, bytes)
 
 proc getBlock*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     root: Opt[Eth2Digest], T: type ForkyTrustedSignedBeaconBlock): Opt[T] =
   var tmp: seq[byte]
-  ? db.getBlockSSZ(historical_roots, slot, tmp).mapErr(proc(x: auto) = discard)
+  ? db.getBlockSSZ(
+    historical_roots, historical_summaries, slot, tmp).mapErr(proc(x: auto) = discard)
 
   result.ok(default(T))
   try:
@@ -303,7 +309,8 @@ proc getBlock*(
     result.err()
 
 proc getStateSZ*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     bytes: var seq[byte]):
     Result[void, string] =
   ## Get a snappy-frame-compressed version of the state data - may overwrite
@@ -313,23 +320,25 @@ proc getStateSZ*(
   # Block content for the blocks of an era is found in the file for the _next_
   # era
   let
-    f = ? db.getEraFile(historical_roots, slot.era)
+    f = ? db.getEraFile(historical_roots, historical_summaries, slot.era)
 
   f.getStateSZ(slot, bytes)
 
 proc getStateSSZ*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     bytes: var seq[byte], partial = Opt.none(int)): Result[void, string] =
   let
-    f = ? db.getEraFile(historical_roots, slot.era)
+    f = ? db.getEraFile(historical_roots, historical_summaries, slot.era)
 
   f.getStateSSZ(slot, bytes, partial)
 
 proc getState*(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     state: var ForkedHashedBeaconState): Result[void, string] =
   var bytes: seq[byte]
-  ? db.getStateSSZ(historical_roots, slot, bytes)
+  ? db.getStateSSZ(historical_roots, historical_summaries, slot, bytes)
 
   try:
     state = readSszForkedHashedBeaconState(db.cfg, slot, bytes)
@@ -356,7 +365,8 @@ type
     ## Needed to process attestations, older to newer
 
 proc getPartialState(
-    db: EraDB, historical_roots: openArray[Eth2Digest], slot: Slot,
+    db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary], slot: Slot,
     output: var PartialBeaconState): bool =
   static: doAssert isFixedSize(PartialBeaconState)
   const partialBytes = fixedPortionSize(PartialBeaconState)
@@ -366,7 +376,8 @@ proc getPartialState(
   # of reading the minimal number of bytes from disk
   var tmp: seq[byte]
   if (let e = db.getStateSSZ(
-      historical_roots, slot, tmp, Opt[int].ok(partialBytes));
+      historical_roots, historical_summaries, slot, tmp,
+      Opt[int].ok(partialBytes));
       e.isErr):
     return false
 
@@ -379,6 +390,7 @@ proc getPartialState(
 
 iterator getBlockIds*(
     db: EraDB, historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary],
     start_slot: Slot, prev_root: Eth2Digest): BlockId =
   ## Iterate over block roots starting from the given slot - `prev_root` must
   ## point out the last block added to the chain before `start_slot` such that
@@ -394,7 +406,8 @@ iterator getBlockIds*(
     case db.cfg.consensusForkAtEpoch(slot.epoch)
     of ConsensusFork.Phase0 .. ConsensusFork.Deneb:
       let stateSlot = (slot.era() + 1).start_slot()
-      if not getPartialState(db, historical_roots, stateSlot, state[]):
+      if not getPartialState(
+          db, historical_roots, historical_summaries, stateSlot, state[]):
         state = nil # No `return` in iterators
 
     if state == nil:
@@ -445,7 +458,7 @@ when isMainModule:
     got8191 = false
     got8192 = false
     got8193 = false
-  for bid in db.getBlockIds(historical_roots, Slot(0), Eth2Digest()):
+  for bid in db.getBlockIds(historical_roots, [], Slot(0), Eth2Digest()):
     if bid.slot == Slot(0):
       doAssert bid.root == Eth2Digest.fromHex(
         "0x4d611d5b93fdab69013a7f0a2f961caca0c853f87cfe9595fe50038163079360")

beacon_chain/spec/forks.nim

@@ -1025,3 +1025,11 @@ func toBlockId*(blck: ForkedSignedBeaconBlock |
                       ForkedMsgTrustedSignedBeaconBlock |
                       ForkedTrustedSignedBeaconBlock): BlockId =
   withBlck(blck): BlockId(root: blck.root, slot: blck.message.slot)
+
+func historical_summaries*(state: ForkedHashedBeaconState):
+    HashList[HistoricalSummary, Limit HISTORICAL_ROOTS_LIMIT] =
+  withState(state):
+    when consensusFork >= ConsensusFork.Capella:
+      forkyState.data.historical_summaries
+    else:
+      HashList[HistoricalSummary, Limit HISTORICAL_ROOTS_LIMIT]()

docs/e2store.md

@@ -187,6 +187,7 @@ Each era is identified by when it ends. Thus, the genesis era is era `0`, follow
 * `era-number` is the number of the _first_ era stored in the file - for example, the genesis era file has number 0 - as a 5-digit 0-filled decimal integer
 * `short-era-root` is the first 4 bytes of the last historical root in the _last_ state in the era file, lower-case hex-encoded (8 characters), except the genesis era which instead uses the `genesis_validators_root` field from the genesis state.
   * The root is available as `state.historical_roots[era - 1]` except for genesis, which is `state.genesis_validators_root`
+  * Post-Capella, the root must be computed from `state.historical_summaries[era - state.historical_roots.len - 1]`
 
 Era files with multiple eras use the era number of the lowest era stored in the file, and the root of the highest era.
 

ncli/e2store.nim

@@ -54,9 +54,15 @@ proc toString(v: IoErrorCode): string =
 
 func eraRoot*(
     genesis_validators_root: Eth2Digest,
-    historical_roots: openArray[Eth2Digest], era: Era): Opt[Eth2Digest] =
+    historical_roots: openArray[Eth2Digest],
+    historical_summaries: openArray[HistoricalSummary],
+    era: Era): Opt[Eth2Digest] =
   if era == Era(0): ok(genesis_validators_root)
-  elif era <= historical_roots.lenu64(): ok(historical_roots[int(uint64(era) - 1)])
+  elif era <= historical_roots.lenu64():
+    ok(historical_roots[int(uint64(era) - 1)])
+  elif era <= historical_roots.lenu64() + historical_summaries.lenu64():
+    ok(hash_tree_root(
+      historical_summaries[int(uint64(era) - 1) - historical_roots.len()]))
   else: err()
 
 func eraFileName*(

ncli/ncli_db.nim

@@ -528,6 +528,7 @@ proc cmdExportEra(conf: DbConf, cfg: RuntimeConfig) =
         eraRoot(
           forkyState.data.genesis_validators_root,
           forkyState.data.historical_roots.asSeq,
+          dag.headState.historical_summaries().asSeq,
           era).expect("have era root since we checked slot")
       name = eraFileName(cfg, era, eraRoot)