update statediffs to support capella (#4852)

beacon_chain/spec/datatypes/altair.nim

@@ -529,60 +529,6 @@ type
   SyncSubcommitteeIndex* = distinct uint8
   IndexInSyncCommittee* = distinct uint16
 
-  BeaconStateDiff* = object
-    # Small and/or static; always include
-    slot*: Slot
-    latest_block_header*: BeaconBlockHeader
-
-    # Mod-increment/circular
-    block_roots*: array[SLOTS_PER_EPOCH, Eth2Digest]
-    state_roots*: array[SLOTS_PER_EPOCH, Eth2Digest]
-
-    # Append-only; either 0 or 1 per epoch
-    historical_root_added*: bool
-    historical_root*: Eth2Digest
-
-    # Replace
-    eth1_data*: Eth1Data
-
-    eth1_data_votes_replaced*: bool
-    eth1_data_votes*:
-      List[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)]
-
-    # Replace
-    eth1_deposit_index*: uint64
-
-    # Validators come in two parts, the immutable public key and mutable
-    # entrance/exit/slashed information about that validator.
-    validator_statuses*:
-      List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT]
-
-    # Represent in full
-    balances*: List[Gwei, Limit VALIDATOR_REGISTRY_LIMIT]
-
-    # Mod-increment
-    randao_mix*: Eth2Digest
-    slashing*: uint64
-
-    # Represent in full; for the next epoch, current_epoch_participation in
-    # epoch n is previous_epoch_participation in epoch n+1 but this doesn't
-    # generalize.
-    previous_epoch_participation*: EpochParticipationFlags
-    current_epoch_participation*: EpochParticipationFlags
-
-    justification_bits*: JustificationBits
-    previous_justified_checkpoint*: Checkpoint
-    current_justified_checkpoint*: Checkpoint
-    finalized_checkpoint*: Checkpoint
-
-    # Represent in full
-    inactivity_scores*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT]
-
-    # Represent in full; for the next epoch, next_sync_committee is
-    # current_sync_committee, but this doesn't generalize.
-    current_sync_committee*: SyncCommittee
-    next_sync_committee*: SyncCommittee
-
 chronicles.formatIt BeaconBlock: it.shortLog
 chronicles.formatIt SyncSubcommitteeIndex: uint8(it)
 

beacon_chain/spec/datatypes/capella.nim

@@ -495,6 +495,67 @@ type
     bls_to_execution_changes*:
       List[SignedBLSToExecutionChange, Limit MAX_BLS_TO_EXECUTION_CHANGES]
 
+  BeaconStateDiff* = object
+    # Small and/or static; always include
+    slot*: Slot
+    latest_block_header*: BeaconBlockHeader
+
+    # Mod-increment/circular
+    block_roots*: array[SLOTS_PER_EPOCH, Eth2Digest]
+    state_roots*: array[SLOTS_PER_EPOCH, Eth2Digest]
+
+    # Replace
+    eth1_data*: Eth1Data
+
+    eth1_data_votes_replaced*: bool
+    eth1_data_votes*:
+      List[Eth1Data, Limit(EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)]
+
+    # Replace
+    eth1_deposit_index*: uint64
+
+    # Validators come in two parts, the immutable public key and mutable
+    # entrance/exit/slashed information about that validator.
+    validator_statuses*:
+      List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT]
+
+    # Represent in full
+    balances*: List[Gwei, Limit VALIDATOR_REGISTRY_LIMIT]
+
+    # Mod-increment
+    randao_mix*: Eth2Digest
+    slashing*: uint64
+
+    # Represent in full; for the next epoch, current_epoch_participation in
+    # epoch n is previous_epoch_participation in epoch n+1 but this doesn't
+    # generalize.
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
+
+    justification_bits*: JustificationBits
+    previous_justified_checkpoint*: Checkpoint
+    current_justified_checkpoint*: Checkpoint
+    finalized_checkpoint*: Checkpoint
+
+    # Represent in full
+    inactivity_scores*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT]
+
+    # Represent in full; for the next epoch, next_sync_committee is
+    # current_sync_committee, but this doesn't generalize.
+    current_sync_committee*: SyncCommittee
+    next_sync_committee*: SyncCommittee
+
+    # Not tiny, but small and infeasible to reliably reduce much
+    latest_execution_payload_header*: ExecutionPayloadHeader
+
+    # Small, so represent in full
+    next_withdrawal_index*: WithdrawalIndex
+    next_withdrawal_validator_index*: uint64
+
+    # Append-only; either 0 or 1 per epoch
+    historical_summary_added*: bool
+    historical_summary*: HistoricalSummary
+
 # TODO: There should be only a single generic HashedBeaconState definition
 func initHashedBeaconState*(s: BeaconState): HashedBeaconState =
   HashedBeaconState(data: s)

beacon_chain/statediff.nim

@@ -9,8 +9,7 @@
 
 import
   stew/assign2,
-  ./spec/datatypes/bellatrix,
+  ./spec/forks
-  ./spec/helpers
 
 func diffModIncEpoch[T, U](hl: HashArray[U, T], startSlot: uint64):
     array[SLOTS_PER_EPOCH, T] =
@@ -82,7 +81,7 @@ func replaceOrAddDecodeEth1Votes[T, U](
     if not votes0.add item:
       raiseAssert "same limit"
 
-func getMutableValidatorStatuses(state: bellatrix.BeaconState):
+func getMutableValidatorStatuses(state: capella.BeaconState):
     List[ValidatorStatus, Limit VALIDATOR_REGISTRY_LIMIT] =
   if not result.setLen(state.validators.len):
     raiseAssert "same limt as validators"
@@ -97,7 +96,7 @@ func getMutableValidatorStatuses(state: bellatrix.BeaconState):
     assign(result[i].exit_epoch, validator.exit_epoch)
     assign(result[i].withdrawable_epoch, validator.withdrawable_epoch)
 
-func diffStates*(state0, state1: bellatrix.BeaconState): BeaconStateDiff =
+func diffStates*(state0, state1: capella.BeaconState): BeaconStateDiff =
   doAssert state1.slot > state0.slot
   doAssert state0.slot.is_epoch
   doAssert state1.slot == state0.slot + SLOTS_PER_EPOCH
@@ -106,11 +105,13 @@ func diffStates*(state0, state1: bellatrix.BeaconState): BeaconStateDiff =
   doAssert state0.genesis_time == state1.genesis_time
   doAssert state0.genesis_validators_root == state1.genesis_validators_root
   doAssert state0.fork == state1.fork
-  doAssert state1.historical_roots.len - state0.historical_roots.len in [0, 1]
+  doAssert state1.historical_roots == state0.historical_roots
+  doAssert state1.historical_summaries.len -
+    state0.historical_summaries.len in [0, 1]
 
   let
-    historical_root_added =
+    historical_summary_added =
-      state0.historical_roots.len != state1.historical_roots.len
+      state0.historical_summaries.len != state1.historical_summaries.len
     (eth1_data_votes_replaced, eth1_data_votes) =
       replaceOrAddEncodeEth1Votes(state0.eth1_data_votes, state1.eth1_data_votes)
 
@@ -120,12 +121,6 @@ func diffStates*(state0, state1: bellatrix.BeaconState): BeaconStateDiff =
 
     block_roots: diffModIncEpoch(state1.block_roots, state0.slot.uint64),
     state_roots: diffModIncEpoch(state1.state_roots, state0.slot.uint64),
-    historical_root_added: historical_root_added,
-    historical_root:
-      if historical_root_added:
-        state1.historical_roots[state0.historical_roots.len]
-      else:
-        default(Eth2Digest),
     eth1_data: state1.eth1_data,
     eth1_data_votes_replaced: eth1_data_votes_replaced,
     eth1_data_votes: eth1_data_votes,
@@ -151,11 +146,23 @@ func diffStates*(state0, state1: bellatrix.BeaconState): BeaconStateDiff =
     inactivity_scores: state1.inactivity_scores.data,
 
     current_sync_committee: state1.current_sync_committee,
-    next_sync_committee: state1.next_sync_committee
+    next_sync_committee: state1.next_sync_committee,
+
+    latest_execution_payload_header: state1.latest_execution_payload_header,
+
+    next_withdrawal_index: state1.next_withdrawal_index,
+    next_withdrawal_validator_index: state1.next_withdrawal_validator_index,
+
+    historical_summary_added: historical_summary_added,
+    historical_summary:
+      if historical_summary_added:
+        state1.historical_summaries[state0.historical_summaries.len]
+      else:
+        (static(default(HistoricalSummary)))
   )
 
 func applyDiff*(
-    state: var bellatrix.BeaconState,
+    state: var capella.BeaconState,
     immutableValidators: openArray[ImmutableValidatorData2],
     stateDiff: BeaconStateDiff) =
   template assign[T, U](tgt: var HashList[T, U], src: List[T, U]) =
@@ -167,9 +174,8 @@ func applyDiff*(
 
   applyModIncrement(state.block_roots, stateDiff.block_roots, state.slot.uint64)
   applyModIncrement(state.state_roots, stateDiff.state_roots, state.slot.uint64)
-  if stateDiff.historical_root_added:
+
-    if not state.historical_roots.add stateDiff.historical_root:
+  # Capella freezes historical_roots
-      raiseAssert "cannot readd historical state root"
 
   assign(state.eth1_data, stateDiff.eth1_data)
   replaceOrAddDecodeEth1Votes(
@@ -195,7 +201,8 @@ func applyDiff*(
 
   state.justification_bits = stateDiff.justification_bits
   assign(
-    state.previous_justified_checkpoint, stateDiff.previous_justified_checkpoint)
+    state.previous_justified_checkpoint,
+    stateDiff.previous_justified_checkpoint)
   assign(
     state.current_justified_checkpoint, stateDiff.current_justified_checkpoint)
   assign(state.finalized_checkpoint, stateDiff.finalized_checkpoint)
@@ -205,5 +212,18 @@ func applyDiff*(
   assign(state.current_sync_committee, stateDiff.current_sync_committee)
   assign(state.next_sync_committee, stateDiff.next_sync_committee)
 
+  assign(
+    state.latest_execution_payload_header,
+    stateDiff.latest_execution_payload_header)
+
+  assign(state.next_withdrawal_index, stateDiff.next_withdrawal_index)
+  assign(
+    state.next_withdrawal_validator_index,
+    stateDiff.next_withdrawal_validator_index)
+
+  if stateDiff.historical_summary_added:
+    if not state.historical_summaries.add stateDiff.historical_summary:
+      raiseAssert "cannot readd historical summary"
+
   # Don't update slot until the end, because various other updates depend on it
   state.slot = stateDiff.slot

tests/test_statediff.nim

@@ -8,14 +8,14 @@
 {.used.}
 
 import
-  std/sequtils,
   unittest2,
   ./testutil, ./testdbutil, ./teststateutil,
-  ../beacon_chain/spec/datatypes/bellatrix,
+  ../beacon_chain/spec/forks,
-  ../beacon_chain/spec/[forks, helpers],
-  ../beacon_chain/statediff,
   ../beacon_chain/consensus_object_pools/[blockchain_dag, block_quarantine]
 
+from std/sequtils import mapIt
+from ../beacon_chain/statediff import applyDiff, diffStates
+
 when isMainModule:
   import chronicles # or some random compile error happens...
 
@@ -27,17 +27,19 @@ suite "state diff tests" & preset():
       dag = init(ChainDAGRef, defaultRuntimeConfig, db, validatorMonitor, {})
 
   test "random slot differences" & preset():
-    let testStates = getTestStates(dag.headState, ConsensusFork.Bellatrix)
+    let testStates = getTestStates(dag.headState, ConsensusFork.Capella)
 
     for i in 0 ..< testStates.len:
       for j in (i+1) ..< testStates.len:
         doAssert getStateField(testStates[i][], slot) <
           getStateField(testStates[j][], slot)
-        if getStateField(testStates[i][], slot) + SLOTS_PER_EPOCH != getStateField(testStates[j][], slot):
+        if  getStateField(testStates[i][], slot) + SLOTS_PER_EPOCH !=
+            getStateField(testStates[j][], slot):
           continue
-        let tmpStateApplyBase = assignClone(testStates[i].bellatrixData.data)
+        let
-        let diff = diffStates(
+          tmpStateApplyBase = assignClone(testStates[i].capellaData.data)
-          testStates[i].bellatrixData.data, testStates[j].bellatrixData.data)
+          diff = diffStates(
+            testStates[i].capellaData.data, testStates[j].capellaData.data)
         # Immutable parts of validators stored separately, so aren't part of
         # the state diff. Synthesize required portion here for testing.
         applyDiff(
@@ -48,5 +50,5 @@ suite "state diff tests" & preset():
               getStateField(testStates[j][], validators).len - 1],
             it.getImmutableValidatorData),
           diff)
-        check hash_tree_root(testStates[j][].bellatrixData.data) ==
+        check hash_tree_root(testStates[j][].capellaData.data) ==
           hash_tree_root(tmpStateApplyBase[])