eip4844 light client tests; avoid case object out-of-bound array reads (#4404)

ConsensusSpecPreset-minimal.md

@@ -408,6 +408,9 @@ ConsensusSpecPreset-minimal
 + Light client - Sync - minimal/capella/light_client/sync/pyspec_tests/advance_finality_with OK
 + Light client - Sync - minimal/capella/light_client/sync/pyspec_tests/light_client_sync     OK
 + Light client - Sync - minimal/capella/light_client/sync/pyspec_tests/supply_sync_committee OK
++ Light client - Sync - minimal/eip4844/light_client/sync/pyspec_tests/advance_finality_with OK
++ Light client - Sync - minimal/eip4844/light_client/sync/pyspec_tests/light_client_sync     OK
++ Light client - Sync - minimal/eip4844/light_client/sync/pyspec_tests/supply_sync_committee OK
 + Light client - Update ranking - minimal/altair/light_client/update_ranking/pyspec_tests/up OK
 + Light client - Update ranking - minimal/bellatrix/light_client/update_ranking/pyspec_tests OK
 + Light client - Update ranking - minimal/capella/light_client/update_ranking/pyspec_tests/u OK
@@ -835,7 +838,7 @@ ConsensusSpecPreset-minimal
 + fork_random_low_balances                                                                   OK
 + fork_random_misc_balances                                                                  OK
 ```
-OK: 824/832 Fail: 0/832 Skip: 8/832
+OK: 827/835 Fail: 0/835 Skip: 8/835
 ## Attestation
 ```diff
 + [Invalid] EF - Altair - Operations - Attestation - after_epoch_slots                       OK
@@ -2588,4 +2591,4 @@ OK: 68/68 Fail: 0/68 Skip: 0/68
 OK: 52/52 Fail: 0/52 Skip: 0/52
 
 ---TOTAL---
-OK: 2278/2286 Fail: 0/2286 Skip: 8/2286
+OK: 2281/2289 Fail: 0/2289 Skip: 8/2289

beacon_chain/beacon_chain_db.nim

@@ -300,7 +300,7 @@ proc close*(s: var DbSeq) =
 
 proc add*[T](s: var DbSeq[T], val: T) =
   doAssert(distinctBase(s.insertStmt) != nil, "database closed or table not preset")
-  var bytes = SSZ.encode(val)
+  let bytes = SSZ.encode(val)
   s.insertStmt.exec(bytes).expectDb()
   inc s.recordCount
 
@@ -993,7 +993,8 @@ proc getBlockSSZ*(
   db.blocks[T.toFork].get(key.data, decode).expectDb() and success
 
 proc getBlockSSZ*[
-    X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock](
+    X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock |
+       eip4844.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type X): bool =
   let dataPtr = addr data # Short-lived
   var success = true
@@ -1015,7 +1016,7 @@ proc getBlockSSZ*(
   of BeaconBlockFork.Capella:
     getBlockSSZ(db, key, data, capella.TrustedSignedBeaconBlock)
   of BeaconBlockFork.EIP4844:
-    raiseAssert $eip4844ImplementationMissing
+    getBlockSSZ(db, key, data, eip4844.TrustedSignedBeaconBlock)
 
 
 proc getBlockSZ*(
@@ -1042,7 +1043,8 @@ proc getBlockSZ*(
   db.blocks[T.toFork].get(key.data, decode).expectDb() and success
 
 proc getBlockSZ*[
-    X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock](
+    X: bellatrix.TrustedSignedBeaconBlock | capella.TrustedSignedBeaconBlock |
+       eip4844.TrustedSignedBeaconBlock](
     db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type X): bool =
   let dataPtr = addr data # Short-lived
   var success = true
@@ -1063,7 +1065,7 @@ proc getBlockSZ*(
   of BeaconBlockFork.Capella:
     getBlockSZ(db, key, data, capella.TrustedSignedBeaconBlock)
   of BeaconBlockFork.EIP4844:
-    raiseAssert $eip4844ImplementationMissing
+    getBlockSZ(db, key, data, eip4844.TrustedSignedBeaconBlock)
 
 
 proc getStateOnlyMutableValidators(

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -146,9 +146,7 @@ proc init*(T: type AttestationPool, dag: ChainDAGRef,
           var unrealized: FinalityCheckpoints
           if enableTestFeatures in dag.updateFlags and blckRef == dag.head:
             unrealized = withState(dag.headState):
-              when stateFork == BeaconStateFork.EIP4844:
+              when stateFork >= BeaconStateFork.Altair:
-                raiseAssert $eip4844ImplementationMissing & ": attestation_pool.nim:init"
-              elif stateFork >= BeaconStateFork.Altair:
                 forkyState.data.compute_unrealized_finality()
               else:
                 var cache: StateCache
@@ -483,10 +481,12 @@ func init(T: type AttestationCache, state: phase0.HashedBeaconState): T =
       state.data.current_epoch_attestations[i].data,
       state.data.current_epoch_attestations[i].aggregation_bits)
 
+from ../spec/datatypes/eip4844 import HashedBeaconState
+
 func init(
     T: type AttestationCache,
     state: altair.HashedBeaconState | bellatrix.HashedBeaconState |
-           capella.HashedBeaconState,
+           capella.HashedBeaconState | eip4844.HashedBeaconState,
     cache: var StateCache): T =
   # Load attestations that are scheduled for being given rewards for
   let
@@ -560,8 +560,6 @@ proc check_attestation_compatible*(
 
   ok()
 
-from ../spec/datatypes/eip4844 import HashedBeaconState
-
 proc getAttestationsForBlock*(pool: var AttestationPool,
                               state: ForkyHashedBeaconState,
                               cache: var StateCache): seq[Attestation] =
@@ -586,12 +584,8 @@ proc getAttestationsForBlock*(pool: var AttestationPool,
       when state is phase0.HashedBeaconState:
         AttestationCache.init(state)
       elif state is altair.HashedBeaconState or state is bellatrix.HashedBeaconState or
-           state is capella.HashedBeaconState:
+           state is capella.HashedBeaconState or state is eip4844.HashedBeaconState:
         AttestationCache.init(state, cache)
-      elif state is eip4844.HashedBeaconState:
-        if true:
-          raiseAssert $eip4844ImplementationMissing & ": attestation_pool.nim:getAttestationsForBlock"
-        default(AttestationCache)
       else:
         static: doAssert false
 

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -18,8 +18,8 @@ import
     state_transition, state_transition_epoch],
   "."/[block_dag, blockchain_dag, blockchain_dag_light_client]
 
-# TODO remove when forks re-exports this
 from ../spec/datatypes/capella import asSigVerified, asTrusted, shortLog
+from ../spec/datatypes/eip4844 import asSigVerified, asTrusted, shortLog
 
 export results, signatures_batch, block_dag, blockchain_dag
 
@@ -108,9 +108,7 @@ proc addResolvedHeadBlock(
     if enableTestFeatures in dag.updateFlags:
       unrealized = withState(state):
         static: doAssert high(BeaconStateFork) == BeaconStateFork.EIP4844
-        when stateFork == BeaconStateFork.EIP4844:
+        when stateFork >= BeaconStateFork.Altair:
-          raiseAssert $eip4844ImplementationMissing & ": block_clearance.nim:addResolvedHeadBlock"
-        elif stateFork >= BeaconStateFork.Altair:
           forkyState.data.compute_unrealized_finality()
         else:
           forkyState.data.compute_unrealized_finality(cache)

beacon_chain/consensus_object_pools/block_dag.nim

@@ -15,8 +15,8 @@ import
   ../spec/datatypes/[phase0, altair, bellatrix],
   ../spec/forks
 
-# TODO remove once forks re-exports these
 from ../spec/datatypes/capella import SomeBeaconBlock, TrustedBeaconBlock
+from ../spec/datatypes/eip4844 import SomeBeaconBlock, TrustedBeaconBlock
 
 export chronicles, forks
 
@@ -71,7 +71,8 @@ func init*(
 func init*(
     T: type BlockRef, root: Eth2Digest,
     blck: bellatrix.SomeBeaconBlock | bellatrix.TrustedBeaconBlock |
-          capella.SomeBeaconBlock | capella.TrustedBeaconBlock): BlockRef =
+          capella.SomeBeaconBlock | capella.TrustedBeaconBlock |
+          eip4844.SomeBeaconBlock | eip4844.TrustedBeaconBlock): BlockRef =
   BlockRef.init(
     root, some Eth2Digest(blck.body.execution_payload.block_hash), blck.slot)
 

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -23,6 +23,7 @@ import
   ./block_dag, block_pools_types_light_client
 
 from ../spec/datatypes/capella import TrustedSignedBeaconBlock
+from ../spec/datatypes/eip4844 import TrustedSignedBeaconBlock
 
 from "."/vanity_logs/pandas import VanityLogs
 
@@ -311,9 +312,15 @@ type
     epochRef: EpochRef,
     unrealized: FinalityCheckpoints) {.gcsafe, raises: [Defect].}
 
+  OnEIP4844BlockAdded* = proc(
+    blckRef: BlockRef,
+    blck: eip4844.TrustedSignedBeaconBlock,
+    epochRef: EpochRef,
+    unrealized: FinalityCheckpoints) {.gcsafe, raises: [Defect].}
+
   OnForkyBlockAdded* =
     OnPhase0BlockAdded | OnAltairBlockAdded | OnBellatrixBlockAdded |
-    OnCapellaBlockAdded
+    OnCapellaBlockAdded | OnEIP4844BlockAdded
 
   HeadChangeInfoObject* = object
     slot*: Slot

beacon_chain/consensus_object_pools/blockchain_dag_light_client.nim

@@ -18,9 +18,9 @@ import
   ../beacon_chain_db_light_client,
   "."/[block_pools_types, blockchain_dag]
 
-from ../spec/datatypes/capella import TrustedSignedBeaconBlock
+from ../spec/datatypes/capella import TrustedSignedBeaconBlock, asSigned
 from ../spec/datatypes/eip4844 import
-  HashedBeaconState, TrustedSignedBeaconBlock
+  HashedBeaconState, TrustedSignedBeaconBlock, asSigned
 
 logScope: topics = "chaindag_lc"
 
@@ -221,9 +221,6 @@ proc initLightClientBootstrapForPeriod(
       dag.lcDataStore.db.putCurrentSyncCommitteeBranch(bid.slot, branch)
   res
 
-from ../spec/datatypes/capella import asSigned
-from ../spec/datatypes/eip4844 import asSigned
-
 proc initLightClientUpdateForPeriod(
     dag: ChainDAGRef, period: SyncCommitteePeriod): Opt[void] =
   ## Compute and cache the best `LightClientUpdate` within a given
@@ -439,7 +436,7 @@ proc cacheLightClientData(
   if dag.lcDataStore.cache.data.hasKeyOrPut(bid, cachedData):
     doAssert false, "Redundant `cacheLightClientData` call"
 
-func shouldImportLcData(dag: ChainDAGref): bool =
+func shouldImportLcData(dag: ChainDAGRef): bool =
   dag.lcDataStore.importMode != LightClientDataImportMode.None and
   dag.cfg.ALTAIR_FORK_EPOCH != FAR_FUTURE_EPOCH
 
@@ -723,7 +720,10 @@ proc processNewBlockForLightClient*(
   if signedBlock.message.slot < dag.lcDataStore.cache.tailSlot:
     return
 
-  when signedBlock is capella.TrustedSignedBeaconBlock:
+  when signedBlock is eip4844.TrustedSignedBeaconBlock:
+    dag.cacheLightClientData(state.eip4844Data, signedBlock.toBlockId())
+    dag.createLightClientUpdates(state.eip4844Data, signedBlock, parentBid)
+  elif signedBlock is capella.TrustedSignedBeaconBlock:
     dag.cacheLightClientData(state.capellaData, signedBlock.toBlockId())
     dag.createLightClientUpdates(state.capellaData, signedBlock, parentBid)
   elif signedBlock is bellatrix.TrustedSignedBeaconBlock:

beacon_chain/networking/network_metadata.nim

@@ -198,6 +198,8 @@ when defined(gnosisChainBinary) and const_preset == "mainnet":
   static:
     for network in [gnosisMetadata]:
       checkForkConsistency(network.cfg)
+    doAssert network.cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH
+    doAssert network.cfg.EIP4844_FORK_EPOCH == FAR_FUTURE_EPOCH
 
 elif const_preset == "mainnet":
   const
@@ -209,6 +211,7 @@ elif const_preset == "mainnet":
         mainnetMetadata, praterMetadata, sepoliaMetadata]:
       checkForkConsistency(network.cfg)
       doAssert network.cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH
+      doAssert network.cfg.EIP4844_FORK_EPOCH == FAR_FUTURE_EPOCH
 
 proc getMetadataForNetwork*(
     networkName: string): Eth2NetworkMetadata {.raises: [Defect, IOError].} =

beacon_chain/rpc/rest_utils.nim

@@ -270,9 +270,8 @@ proc getStateOptimistic*(node: BeaconNode,
     case state.kind
     of BeaconStateFork.Phase0, BeaconStateFork.Altair:
       some[bool](false)
-    of BeaconStateFork.EIP4844:
+    of  BeaconStateFork.Bellatrix, BeaconStateFork.Capella,
-      raiseAssert $eip4844ImplementationMissing & "rest_utils.nim:getStateOptimistic"
+        BeaconStateFork.EIP4844:
-    of BeaconStateFork.Bellatrix, BeaconStateFork.Capella:
       # A state is optimistic iff the block which created it is
       withState(state):
         # The block root which created the state at slot `n` is at slot `n-1`

beacon_chain/spec/beaconstate.nim

@@ -11,7 +11,6 @@ else:
   {.push raises: [].}
 
 import
-  std/[algorithm, collections/heapqueue, math, sequtils, tables],
   stew/assign2,
   json_serialization/std/sets,
   chronicles,
@@ -19,6 +18,10 @@ import
   ./datatypes/[phase0, altair, bellatrix],
   "."/[eth2_merkleization, forks, signatures, validator]
 
+from std/algorithm import fill
+from std/math import `^`
+from std/sequtils import anyIt, mapIt
+
 from ./datatypes/capella import BeaconState, ExecutionPayloadHeader, Withdrawal
 
 export extras, forks, validator

beacon_chain/spec/crypto.nim

@@ -28,7 +28,7 @@ else:
 
 import
   # Standard library
-  std/[options, hashes, sequtils, tables],
+  std/options,
   # Status
   stew/[endians2, objects, results, byteutils],
   blscurve,
@@ -36,6 +36,10 @@ import
   bearssl/rand,
   json_serialization
 
+from std/hashes import Hash
+from std/sequtils import mapIt
+from std/tables import Table, withValue, `[]=`
+
 from nimcrypto/utils import burnMem
 
 export options, results, blscurve, rand, json_serialization

beacon_chain/spec/datatypes/base.nim

@@ -953,7 +953,6 @@ func clear*(cache: var StateCache) =
   cache.sync_committees.clear
 
 func checkForkConsistency*(cfg: RuntimeConfig) =
-  doAssert cfg.EIP4844_FORK_EPOCH == FAR_FUTURE_EPOCH
   doAssert cfg.SHARDING_FORK_EPOCH == FAR_FUTURE_EPOCH
 
   let forkVersions =

beacon_chain/spec/datatypes/eip4844.nim

@@ -367,6 +367,30 @@ type
     parentHash*: string
     timestamp*: string
 
+func shortLog*(v: SomeBeaconBlock): auto =
+  (
+    slot: shortLog(v.slot),
+    proposer_index: v.proposer_index,
+    parent_root: shortLog(v.parent_root),
+    state_root: shortLog(v.state_root),
+    eth1data: v.body.eth1_data,
+    graffiti: $v.body.graffiti,
+    proposer_slashings_len: v.body.proposer_slashings.len(),
+    attester_slashings_len: v.body.attester_slashings.len(),
+    attestations_len: v.body.attestations.len(),
+    deposits_len: v.body.deposits.len(),
+    voluntary_exits_len: v.body.voluntary_exits.len(),
+    sync_committee_participants:
+      countOnes(v.body.sync_aggregate.sync_committee_bits),
+    block_number: 0'u64, # Bellatrix compat
+    fee_recipient: "",
+  )
+
+func shortLog*(v: SomeSignedBeaconBlock): auto =
+  (
+    blck: shortLog(v.message),
+    signature: shortLog(v.signature)
+  )
 template asSigned*(
     x: SigVerifiedSignedBeaconBlock |
        MsgTrustedSignedBeaconBlock |

beacon_chain/spec/engine_authentication.nim

@@ -6,12 +6,17 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  std/[base64, json, options, os, strutils],
   chronicles,
   bearssl/rand,
   nimcrypto/[hmac, utils],
   stew/[byteutils, results]
 
+from std/base64 import encode
+from std/json import JsonNode, `$`, `%*`
+from std/options import Option, get, isNone
+from std/os import `/`
+from std/strutils import replace
+
 export rand, results
 
 when (NimMajor, NimMinor) < (1, 4):

beacon_chain/spec/forks.nim

@@ -559,28 +559,33 @@ func setStateRoot*(x: var ForkedHashedBeaconState, root: Eth2Digest) =
 func stateForkAtEpoch*(cfg: RuntimeConfig, epoch: Epoch): BeaconStateFork =
   ## Return the current fork for the given epoch.
   static:
+    doAssert high(BeaconStateFork) == BeaconStateFork.EIP4844
+    doAssert BeaconStateFork.EIP4844   > BeaconStateFork.Capella
     doAssert BeaconStateFork.Capella   > BeaconStateFork.Bellatrix
     doAssert BeaconStateFork.Bellatrix > BeaconStateFork.Altair
     doAssert BeaconStateFork.Altair    > BeaconStateFork.Phase0
     doAssert GENESIS_EPOCH == 0
 
-  if   epoch >= cfg.CAPELLA_FORK_EPOCH:   BeaconStateFork.Capella
+  if   epoch >= cfg.EIP4844_FORK_EPOCH:   BeaconStateFork.EIP4844
+  elif epoch >= cfg.CAPELLA_FORK_EPOCH:   BeaconStateFork.Capella
   elif epoch >= cfg.BELLATRIX_FORK_EPOCH: BeaconStateFork.Bellatrix
   elif epoch >= cfg.ALTAIR_FORK_EPOCH:    BeaconStateFork.Altair
   else:                                   BeaconStateFork.Phase0
 
 func blockForkAtEpoch*(cfg: RuntimeConfig, epoch: Epoch): BeaconBlockFork =
   ## Return the current fork for the given epoch.
-  if   epoch >= cfg.CAPELLA_FORK_EPOCH:   BeaconBlockFork.Capella
+  static: doAssert high(BeaconBlockFork) == BeaconBlockFork.EIP4844
+  if   epoch >= cfg.EIP4844_FORK_EPOCH:   BeaconBlockFork.EIP4844
+  elif epoch >= cfg.CAPELLA_FORK_EPOCH:   BeaconBlockFork.Capella
   elif epoch >= cfg.BELLATRIX_FORK_EPOCH: BeaconBlockFork.Bellatrix
   elif epoch >= cfg.ALTAIR_FORK_EPOCH:    BeaconBlockFork.Altair
   else:                                   BeaconBlockFork.Phase0
 
 func stateForkForDigest*(
     forkDigests: ForkDigests, forkDigest: ForkDigest): Opt[BeaconStateFork] =
+  static: doAssert high(BeaconStateFork) == BeaconStateFork.EIP4844
   if   forkDigest == forkDigests.eip4844:
-    discard $eip4844ImplementationMissing & ": when forkedstate appears, use it here"
+    ok BeaconStateFork.EIP4844
-    ok BeaconStateFork.Capella
   elif forkDigest == forkDigests.capella:
     ok BeaconStateFork.Capella
   elif forkDigest == forkDigests.bellatrix:
@@ -834,13 +839,14 @@ func nextForkEpochAtEpoch*(cfg: RuntimeConfig, epoch: Epoch): Epoch =
   of BeaconStateFork.Altair:    cfg.BELLATRIX_FORK_EPOCH
   of BeaconStateFork.Phase0:    cfg.ALTAIR_FORK_EPOCH
 
-func getForkSchedule*(cfg: RuntimeConfig): array[3, Fork] =
+func getForkSchedule*(cfg: RuntimeConfig): array[5, Fork] =
   ## This procedure returns list of known and/or scheduled forks.
   ##
   ## This procedure is used by HTTP REST framework and validator client.
   ##
   ## NOTE: Update this procedure when new fork will be scheduled.
-  [cfg.genesisFork(), cfg.altairFork(), cfg.bellatrixFork()]
+  [cfg.genesisFork(), cfg.altairFork(), cfg.bellatrixFork(), cfg.capellaFork(),
+   cfg.eip4844Fork()]
 
 type
   # The first few fields of a state, shared across all forks
@@ -926,6 +932,7 @@ func compute_fork_digest*(current_version: Version,
 func init*(T: type ForkDigests,
            cfg: RuntimeConfig,
            genesis_validators_root: Eth2Digest): T =
+  static: doAssert high(BeaconStateFork) == BeaconStateFork.EIP4844
   T(
     phase0:
       compute_fork_digest(cfg.GENESIS_FORK_VERSION, genesis_validators_root),
@@ -934,7 +941,9 @@ func init*(T: type ForkDigests,
     bellatrix:
       compute_fork_digest(cfg.BELLATRIX_FORK_VERSION, genesis_validators_root),
     capella:
-      compute_fork_digest(cfg.CAPELLA_FORK_VERSION, genesis_validators_root)
+      compute_fork_digest(cfg.CAPELLA_FORK_VERSION, genesis_validators_root),
+    eip4844:
+      compute_fork_digest(cfg.EIP4844_FORK_VERSION, genesis_validators_root)
   )
 
 func toBlockId*(header: BeaconBlockHeader): BlockId =

beacon_chain/spec/keystore.nim

@@ -275,7 +275,7 @@ macro wordListArray*(filename: static string,
                      minWordLen: static int = 0,
                      maxWordLen: static int = high(int)): untyped =
   result = newTree(nnkBracket)
-  var words = slurp(filename.replace('\\', '/')).splitLines()
+  let words = slurp(filename.replace('\\', '/')).splitLines()
   for word in words:
     if word.len >= minWordLen and word.len <= maxWordLen:
       result.add newCall("cstring", newLit(word))
@@ -489,7 +489,7 @@ proc writeJsonHexString(s: OutputStream, data: openArray[byte])
 
 proc readValue*(r: var JsonReader, value: var Pbkdf2Salt)
                {.raises: [SerializationError, IOError, Defect].} =
-  var s = r.readValue(string)
+  let s = r.readValue(string)
 
   if s.len == 0 or s.len mod 16 != 0:
     r.raiseUnexpectedValue(
@@ -503,7 +503,7 @@ proc readValue*(r: var JsonReader, value: var Pbkdf2Salt)
 
 proc readValue*(r: var JsonReader, value: var Aes128CtrIv)
                {.raises: [SerializationError, IOError, Defect].} =
-  var s = r.readValue(string)
+  let s = r.readValue(string)
 
   if s.len != 32:
     r.raiseUnexpectedValue(

beacon_chain/spec/state_transition.nim

@@ -541,7 +541,12 @@ proc makeBeaconBlock*[T: bellatrix.ExecutionPayload | capella.ExecutionPayload](
       raiseAssert "Attempt to use Capella payload with non-Capella state"
     of BeaconStateFork.Capella:   makeBeaconBlock(capella)
   elif T is eip4844.ExecutionPayload:
-    raiseAssert $eip4844ImplementationMissing & ": state_transition"
+    case state.kind
+    of  BeaconStateFork.Phase0, BeaconStateFork.Altair,
+        BeaconStateFork.Bellatrix, BeaconStateFork.Capella:
+      raiseAssert "Attempt to use EIP4844 payload with non-EIP4844 state"
+    of BeaconStateFork.EIP4844:
+      raiseAssert $eip4844ImplementationMissing & ": state_transition"
 
 # workaround for https://github.com/nim-lang/Nim/issues/20900 rather than have
 # these be default arguments

beacon_chain/spec/state_transition_block.nim

@@ -23,12 +23,13 @@ else:
   {.push raises: [].}
 
 import
-  std/[algorithm, sequtils, sets, tables],
   chronicles, metrics,
   ../extras,
   ./datatypes/[phase0, altair, bellatrix],
   "."/[beaconstate, eth2_merkleization, helpers, validator, signatures]
 
+from std/algorithm import fill, sorted
+from std/sequtils import count, filterIt, mapIt
 from ./datatypes/capella import
   BeaconState, MAX_WITHDRAWALS_PER_PAYLOAD, SignedBLSToExecutionChange,
   Withdrawal
@@ -847,7 +848,8 @@ func verify_kzg_commitments_against_transactions(
 # https://github.com/ethereum/consensus-specs/blob/v1.3.0-alpha.1/specs/eip4844/beacon-chain.md#blob-kzg-commitments
 func process_blob_kzg_commitments(
     state: var eip4844.BeaconState,
-    body: eip4844.BeaconBlockBody | eip4844.TrustedBeaconBlockBody):
+    body: eip4844.BeaconBlockBody | eip4844.TrustedBeaconBlockBody |
+          eip4844.SigVerifiedBeaconBlockBody):
     Result[void, cstring] =
   if verify_kzg_commitments_against_transactions(
       body.execution_payload.transactions.asSeq,

beacon_chain/spec/state_transition_epoch.nim

@@ -24,12 +24,13 @@ else:
   {.push raises: [].}
 
 import
-  std/[algorithm, math],
   stew/bitops2, chronicles,
   ../extras,
   ./datatypes/[phase0, altair, bellatrix],
   "."/[beaconstate, eth2_merkleization, helpers, validator]
 
+from std/algorithm import sort
+from std/math import sum, `^`
 from ./datatypes/capella import BeaconState, Withdrawal, WithdrawalIndex
 
 export extras, phase0, altair
@@ -441,8 +442,8 @@ proc process_justification_and_finalization*(
     balances.current_epoch_TIMELY_TARGET, flags)
 
 proc compute_unrealized_finality*(
-    state: altair.BeaconState | bellatrix.BeaconState | capella.BeaconState):
+    state: altair.BeaconState | bellatrix.BeaconState | capella.BeaconState |
-    FinalityCheckpoints =
+           eip4844.BeaconState): FinalityCheckpoints =
   if get_current_epoch(state) <= GENESIS_EPOCH + 1:
     return FinalityCheckpoints(
       justified: state.current_justified_checkpoint,

beacon_chain/spec/validator.nim

@@ -12,7 +12,6 @@ else:
   {.push raises: [].}
 
 import
-  std/[options, math, tables],
   ./datatypes/[phase0, altair, bellatrix],
   ./helpers
 

beacon_chain/statusbar.nim

@@ -48,9 +48,9 @@ func loadFragmentsLayout(contentLayout: string): ContentFragments {.
 
 func loadCellsLayout(cellsLayout: string): seq[StatusBarCell] {.
     raises: [Defect, ValueError].} =
-  var cells = cellsLayout.split(';')
+  let cells = cellsLayout.split(';')
   for cell in cells:
-    var columns = cell.split(':', maxSplit = 1)
+    let columns = cell.split(':', maxSplit = 1)
     if columns.len == 2:
       result.add StatusBarCell(
         label: strip(columns[0]),
@@ -60,7 +60,7 @@ func loadCellsLayout(cellsLayout: string): seq[StatusBarCell] {.
         contentFragments: loadFragmentsLayout(columns[0]))
 
 func loadLayout(layout: string): Layout {.raises: [Defect, ValueError].} =
-  var sections = layout.split('|', maxSplit = 1)
+  let sections = layout.split('|', maxSplit = 1)
   result.cellsLeft = loadCellsLayout(sections[0])
   if sections.len == 2: result.cellsRight = loadCellsLayout(sections[1])
 

research/wss_sim.nim

@@ -40,6 +40,7 @@ proc findValidator(validators: seq[Validator], pubKey: ValidatorPubKey):
     Opt.some idx.ValidatorIndex
 
 from ../beacon_chain/spec/datatypes/capella import SignedBeaconBlock
+from ../beacon_chain/spec/datatypes/eip4844 import SignedBeaconBlock
 
 cli do(validatorsDir: string, secretsDir: string,
        startState: string, network: string):
@@ -204,7 +205,14 @@ cli do(validatorsDir: string, secretsDir: string,
             validators[proposer]).toValidatorSig())
         dump(".", signedBlock)
       of BeaconBlockFork.EIP4844:
-        raiseAssert $eip4844ImplementationMissing
+        blockRoot = hash_tree_root(message.eip4844Data)
+        let signedBlock = eip4844.SignedBeaconBlock(
+          message: message.eip4844Data,
+          root: blockRoot,
+          signature: get_block_signature(
+            fork, genesis_validators_root, slot, blockRoot,
+            validators[proposer]).toValidatorSig())
+        dump(".", signedBlock)
       notice "Block proposed", message, blockRoot
 
       aggregates.setLen(0)

tests/consensus_spec/fixtures_utils.nim

@@ -69,7 +69,7 @@ type
   UnconsumedInput* = object of CatchableError
   TestSizeError* = object of ValueError
 
-  # https://github.com/ethereum/consensus-specs/tree/v1.2.0-rc.1/tests/formats/rewards#rewards-tests
+  # https://github.com/ethereum/consensus-specs/tree/v1.3.0-alpha.1/tests/formats/rewards#rewards-tests
   Deltas* = object
     rewards*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT]
     penalties*: List[uint64, Limit VALIDATOR_REGISTRY_LIMIT]

tests/consensus_spec/test_fixture_fork_choice.nim

@@ -65,6 +65,9 @@ type
 from ../../beacon_chain/spec/datatypes/capella import
   BeaconBlock, BeaconState, SignedBeaconBlock
 
+from ../../beacon_chain/spec/datatypes/eip4844 import
+  BeaconBlock, BeaconState, SignedBeaconBlock
+
 proc initialLoad(
     path: string, db: BeaconChainDB,
     StateType, BlockType: typedesc
@@ -78,7 +81,13 @@ proc initialLoad(
       path/"anchor_block.ssz_snappy",
       SSZ, BlockType)
 
-  when BlockType is capella.BeaconBlock:
+  when BlockType is eip4844.BeaconBlock:
+    let signedBlock = ForkedSignedBeaconBlock.init(eip4844.SignedBeaconBlock(
+      message: blck,
+      # signature: - unused as it's trusted
+      root: hash_tree_root(blck)
+    ))
+  elif BlockType is capella.BeaconBlock:
     let signedBlock = ForkedSignedBeaconBlock.init(capella.SignedBeaconBlock(
       message: blck,
       # signature: - unused as it's trusted
@@ -166,7 +175,12 @@ proc loadOps(path: string, fork: BeaconStateFork): seq[Operation] =
         result.add Operation(kind: opOnBlock,
           blck: ForkedSignedBeaconBlock.init(blck))
       of BeaconStateFork.EIP4844:
-        raiseAssert $eip4844ImplementationMissing & ": test_fixture_fork_choice.nim:loadOps"
+        let blck = parseTest(
+          path/filename & ".ssz_snappy",
+          SSZ, eip4844.SignedBeaconBlock
+        )
+        result.add Operation(kind: opOnBlock,
+          blck: ForkedSignedBeaconBlock.init(blck))
     elif step.hasKey"attester_slashing":
       let filename = step["attester_slashing"].getStr()
       let attesterSlashing = parseTest(
@@ -214,6 +228,7 @@ proc stepOnBlock(
   )
 
   # 2. Add block to DAG
+  static: doAssert high(BeaconBlockFork) == BeaconBlockFork.EIP4844
   when signedBlock is phase0.SignedBeaconBlock:
     type TrustedBlock = phase0.TrustedSignedBeaconBlock
   elif signedBlock is altair.SignedBeaconBlock:
@@ -222,6 +237,8 @@ proc stepOnBlock(
     type TrustedBlock = bellatrix.TrustedSignedBeaconBlock
   elif signedBlock is capella.SignedBeaconBlock:
     type TrustedBlock = capella.TrustedSignedBeaconBlock
+  elif signedBlock is eip4844.SignedBeaconBlock:
+    type TrustedBlock = eip4844.TrustedSignedBeaconBlock
   else:
     doAssert false, "Unknown TrustedSignedBeaconBlock fork"
 
@@ -321,7 +338,7 @@ proc doRunTest(path: string, fork: BeaconStateFork) =
   let stores =
     case fork
     of BeaconStateFork.EIP4844:
-      raiseAssert $eip4844ImplementationMissing & ": test_fixture_fork_choice.nim:doRunTest"
+      initialLoad(path, db, eip4844.BeaconState, eip4844.BeaconBlock)
     of BeaconStateFork.Capella:
       initialLoad(path, db, capella.BeaconState, capella.BeaconBlock)
     of BeaconStateFork.Bellatrix:
@@ -407,6 +424,10 @@ template fcSuite(suiteName: static[string], testPathElem: static[string]) =
       if kind != pcDir or not dirExists(testsPath):
         continue
       if path.contains("eip4844"):
+        # TODO 1.3.0-alpha.1 test prestates have forks of
+        # (previous_version: 02000000, current_version: 04000000, epoch: 0)
+        # which is probably broken, and chaindag certainly doesn't like
+        discard $eip4844ImplementationMissing & ": re-enable when test pre-state forks fixed"
         continue
       let fork = forkForPathComponent(path).valueOr:
         raiseAssert "Unknown test fork: " & testsPath

tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim

@@ -52,8 +52,6 @@ proc runTest(path: string, fork: BeaconStateFork) =
           get_subtree_index(proof.leaf_index),
           forkyState.root)
 
-from std/strutils import contains
-
 suite "EF - Light client - Single merkle proof" & preset():
   const presetPath = SszTestsDir/const_preset
   for kind, path in walkDir(presetPath, relative = true, checkDir = true):
@@ -62,9 +60,6 @@ suite "EF - Light client - Single merkle proof" & preset():
       continue
     let
       fork = forkForPathComponent(path).valueOr:
-        if path.contains("eip4844"):
-          # not yet supported, might be folded into capella first regardless
-          continue
         raiseAssert "Unknown test fork: " & testsPath
       basePath = testsPath/"BeaconState"
     for kind, path in walkDir(basePath, relative = true, checkDir = true):

tests/consensus_spec/test_fixture_light_client_sync.nim

@@ -116,14 +116,9 @@ proc runTest(path: string) =
         store.optimistic_header.slot == step.checks.optimistic_slot
         hash_tree_root(store.optimistic_header) == step.checks.optimistic_root
 
-from std/strutils import contains
-
 suite "EF - Light client - Sync" & preset():
   const presetPath = SszTestsDir/const_preset
   for kind, path in walkDir(presetPath, relative = true, checkDir = true):
-    if path.contains("eip4844"):
-      # this is a maybe temporary fork, maybe folded into capella soon
-      continue
     let basePath =
       presetPath/path/"light_client"/"sync"/"pyspec_tests"
     if kind != pcDir or not dirExists(basePath):

tests/test_light_client.nim

@@ -71,7 +71,8 @@ suite "Light client" & preset():
             const nilCallback = OnCapellaBlockAdded(nil)
             dag.addHeadBlock(verifier, blck.capellaData, nilCallback)
           of BeaconBlockFork.EIP4844:
-            raiseAssert $eip4844ImplementationMissing
+            const nilCallback = OnEIP4844BlockAdded(nil)
+            dag.addHeadBlock(verifier, blck.eip4844Data, nilCallback)
 
         check: added.isOk()
         dag.updateHead(added[], quarantine)

tests/test_light_client_processor.nim

@@ -56,7 +56,8 @@ suite "Light client processor" & preset():
           const nilCallback = OnCapellaBlockAdded(nil)
           dag.addHeadBlock(verifier, blck.capellaData, nilCallback)
         of BeaconBlockFork.EIP4844:
-          raiseAssert $eip4844ImplementationMissing
+          const nilCallback = OnEIP4844BlockAdded(nil)
+          dag.addHeadBlock(verifier, blck.eip4844Data, nilCallback)
       doAssert added.isOk()
       dag.updateHead(added[], quarantine[])
 

tests/testblockutil.nim

@@ -79,7 +79,7 @@ func signBlock(
   ForkedSignedBeaconBlock.init(forked, root, signature)
 
 from ../beacon_chain/spec/datatypes/capella import
-  BeaconState, ExecutionPayload
+  BeaconState, ExecutionPayload, SignedBLSToExecutionChangeList
 
 func build_empty_merge_execution_payload(state: bellatrix.BeaconState):
     bellatrix.ExecutionPayload =
@@ -135,9 +135,6 @@ proc build_empty_merge_execution_payload(state: capella.BeaconState):
 
   payload
 
-from ../beacon_chain/spec/datatypes/capella import
-  SignedBLSToExecutionChangeList
-
 proc addTestBlockAux[EP: bellatrix.ExecutionPayload | capella.ExecutionPayload](
     state: var ForkedHashedBeaconState,
     cache: var StateCache,