block_clearance, ncli, and ncli_db Altair state saving (#2672)

* block_clearance, ncli, and ncli_db Altair state saving

* avoid invalidating SSZ hash caches with every assignment

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -155,9 +155,7 @@ proc addResolvedBlock(
   dag.blocks.incl(KeyedBlockRef.init(blockRef))
 
   # Resolved blocks should be stored in database
-  when not (trustedBlock is altair.TrustedSignedBeaconBlock):
-    # TODO implement this for altair
-    dag.putBlock(trustedBlock)
+  dag.putBlock(trustedBlock)
   let putBlockTick = Moment.now()
 
   var foundHead: bool

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -17,7 +17,7 @@ import
     beaconstate, forkedbeaconstate_helpers],
   ../spec/datatypes/[phase0, altair],
   ../beacon_clock,
-  "."/[block_pools_types, block_quarantine]
+  "."/[block_pools_types, block_quarantine, forkedbeaconstate_dbhelpers]
 
 export block_pools_types, helpers, phase0
 
@@ -45,7 +45,8 @@ declareGauge beacon_processed_deposits_total, "Number of total deposits included
 logScope: topics = "chaindag"
 
 proc putBlock*(
-    dag: ChainDAGRef, signedBlock: phase0.TrustedSignedBeaconBlock) =
+    dag: ChainDAGRef, signedBlock:
+      phase0.TrustedSignedBeaconBlock | altair.TrustedSignedBeaconBlock) =
   dag.db.putBlock(signedBlock)
 
 proc updateStateData*(
@@ -540,7 +541,7 @@ proc getState(dag: ChainDAGRef, state: var StateData, bs: BlockSlot): bool =
 
   false
 
-proc putState(dag: ChainDAGRef, state: var StateData) =
+proc putState(dag: ChainDAGRef, state: StateData) =
   # Store a state and its root
   logScope:
     blck = shortLog(state.blck)
@@ -559,12 +560,7 @@ proc putState(dag: ChainDAGRef, state: var StateData) =
   # Ideally we would save the state and the root lookup cache in a single
   # transaction to prevent database inconsistencies, but the state loading code
   # is resilient against one or the other going missing
-  case state.data.beaconStateFork:
-  of forkPhase0:
-    dag.db.putState(getStateRoot(state.data), state.data.hbsPhase0.data)
-  of forkAltair:
-    dag.db.putState(getStateRoot(state.data), state.data.hbsAltair.data)
-
+  dag.db.putState(state.data)
   dag.db.putStateRoot(
     state.blck.root, getStateField(state.data, slot), getStateRoot(state.data))
 

beacon_chain/consensus_object_pools/forkedbeaconstate_dbhelpers.nim

@@ -0,0 +1,17 @@
+# beacon_chain
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [Defect].}
+
+import
+  ../spec/forkedbeaconstate_helpers,
+  ../beacon_chain_db
+
+proc putState*(db: BeaconChainDB, state: ForkedHashedBeaconState) =
+  case state.beaconStateFork:
+  of forkPhase0: db.putState(getStateRoot(state), state.hbsPhase0.data)
+  of forkAltair: db.putState(getStateRoot(state), state.hbsAltair.data)

beacon_chain/spec/state_transition_epoch.nim

@@ -691,7 +691,7 @@ func process_rewards_and_penalties(
     decrease_balance(state, ValidatorIndex(index), penalties[index])
 
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.1/specs/phase0/beacon-chain.md#slashings
-# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#slashings
+# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.7/specs/altair/beacon-chain.md#slashings
 func process_slashings*(state: var SomeBeaconState, total_balance: Gwei) {.nbench.}=
   let
     epoch = get_current_epoch(state)
@@ -784,14 +784,19 @@ func process_participation_record_updates*(state: var phase0.BeaconState) {.nben
   state.previous_epoch_attestations.clear()
   swap(state.previous_epoch_attestations, state.current_epoch_attestations)
 
-# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#participation-flags-updates
+# https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.7/specs/altair/beacon-chain.md#participation-flags-updates
 func process_participation_flag_updates*(state: var altair.BeaconState) =
   state.previous_epoch_participation = state.current_epoch_participation
 
-  # TODO more subtle clearing
-  state.current_epoch_participation.clear()
-  for _ in 0 ..< state.validators.len:
-    doAssert state.current_epoch_participation.add 0.ParticipationFlags
+  const zero = 0.ParticipationFlags
+  for i in 0 ..< state.current_epoch_participation.len:
+    state.current_epoch_participation.data[i] = zero
+
+  # Shouldn't be wasted zeroing, because state.current_epoch_participation only
+  # grows. New elements are automatically initialized to 0, as required.
+  doAssert state.current_epoch_participation.data.setLen(state.validators.len)
+
+  state.current_epoch_participation.resetCache()
 
 # https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.7/specs/altair/beacon-chain.md#sync-committee-updates
 proc process_sync_committee_updates*(state: var altair.BeaconState) =

ncli/ncli.nim

@@ -74,6 +74,11 @@ type
         argument
         desc: "Filename of state resulting from applying blck to preState"}: string
 
+template saveSSZFile(filename: string, value: ForkedHashedBeaconState) =
+  case value.beaconStateFork:
+  of forkPhase0: SSZ.saveFile(filename, value.hbsPhase0.data)
+  of forkAltair: SSZ.saveFile(filename, value.hbsAltair.data)
+
 proc doTransition(conf: NcliConf) =
   let
     stateY = (ref ForkedHashedBeaconState)(
@@ -95,7 +100,7 @@ proc doTransition(conf: NcliConf) =
     error "State transition failed"
     quit 1
   else:
-    SSZ.saveFile(conf.postState, stateY.hbsPhase0.data)
+    saveSSZFile(conf.postState, stateY[])
 
 proc doSlots(conf: NcliConf) =
   type
@@ -126,7 +131,7 @@ proc doSlots(conf: NcliConf) =
         FAR_FUTURE_SLOT)
 
   withTimer(timers[tSaveState]):
-    SSZ.saveFile(conf.postState, stateY.hbsPhase0.data)
+    saveSSZFile(conf.postState, stateY[])
 
   printTimers(false, timers)
 

ncli/ncli_db.nim

@@ -3,10 +3,11 @@ import
   chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
   ../beacon_chain/networking/network_metadata,
   ../beacon_chain/[beacon_chain_db, extras],
-  ../beacon_chain/consensus_object_pools/blockchain_dag,
-  ../beacon_chain/spec/[crypto, datatypes, digest, forkedbeaconstate_helpers,
-                        helpers, state_transition, state_transition_epoch,
-                        presets],
+  ../beacon_chain/consensus_object_pools/[
+    blockchain_dag, forkedbeaconstate_dbhelpers],
+  ../beacon_chain/spec/[
+    crypto, datatypes, digest, forkedbeaconstate_helpers, helpers,
+    state_transition, state_transition_epoch, presets],
   ../beacon_chain/ssz, ../beacon_chain/ssz/sszdump,
   ../research/simutils, ./e2store
 
@@ -222,11 +223,11 @@ proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
 
     if getStateField(state[].data, slot).isEpoch and conf.storeStates:
       if getStateField(state[].data, slot).epoch < 2:
-        dbBenchmark.putState(getStateRoot(state[].data), state[].data.hbsPhase0.data)
+        dbBenchmark.putState(state[].data)
         dbBenchmark.checkpoint()
       else:
         withTimer(timers[tDbStore]):
-          dbBenchmark.putState(getStateRoot(state[].data), state[].data.hbsPhase0.data)
+          dbBenchmark.putState(state[].data)
           dbBenchmark.checkpoint()
 
         withTimer(timers[tDbLoad]):