add database beaconstate tests for capella and eip4844 (#4429)

AllTests-mainnet.md

@@ -35,8 +35,14 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + sanity check Bellatrix blocks [Preset: mainnet]                                            OK
 + sanity check Bellatrix states [Preset: mainnet]                                            OK
 + sanity check Bellatrix states, reusing buffers [Preset: mainnet]                           OK
++ sanity check Capella and cross-fork getState rollback [Preset: mainnet]                    OK
 + sanity check Capella blocks [Preset: mainnet]                                              OK
++ sanity check Capella states [Preset: mainnet]                                              OK
++ sanity check Capella states, reusing buffers [Preset: mainnet]                             OK
++ sanity check EIP4844 and cross-fork getState rollback [Preset: mainnet]                    OK
 + sanity check EIP4844 blocks [Preset: mainnet]                                              OK
++ sanity check EIP4844 states [Preset: mainnet]                                              OK
++ sanity check EIP4844 states, reusing buffers [Preset: mainnet]                             OK
 + sanity check genesis roundtrip [Preset: mainnet]                                           OK
 + sanity check phase 0 blocks [Preset: mainnet]                                              OK
 + sanity check phase 0 getState rollback [Preset: mainnet]                                   OK
@@ -44,7 +50,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 + sanity check phase 0 states, reusing buffers [Preset: mainnet]                             OK
 + sanity check state diff roundtrip [Preset: mainnet]                                        OK
 ```
-OK: 18/18 Fail: 0/18 Skip: 0/18
+OK: 24/24 Fail: 0/24 Skip: 0/24
 ## Beacon state [Preset: mainnet]
 ```diff
 + Smoke test initialize_beacon_state_from_eth1 [Preset: mainnet]                             OK
@@ -609,4 +615,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 338/343 Fail: 0/343 Skip: 5/343
+OK: 344/349 Fail: 0/349 Skip: 5/349

beacon_chain/beacon_chain_db.nim

@@ -668,7 +668,7 @@ type GetResult = enum
 proc getSSZ[T](db: KvStoreRef, key: openArray[byte], output: var T): GetResult =
   var status = GetResult.notFound
 
-  var outputPtr = addr output # callback is local, ptr wont escape
+  let outputPtr = addr output # callback is local, ptr wont escape
   proc decode(data: openArray[byte]) =
     status =
       if decodeSSZ(data, outputPtr[]): GetResult.found
@@ -684,7 +684,7 @@ proc putSSZ(db: KvStoreRef, key: openArray[byte], v: auto) =
 proc getSnappySSZ[T](db: KvStoreRef, key: openArray[byte], output: var T): GetResult =
   var status = GetResult.notFound
 
-  var outputPtr = addr output # callback is local, ptr wont escape
+  let outputPtr = addr output # callback is local, ptr wont escape
   proc decode(data: openArray[byte]) =
     status =
       if decodeSnappySSZ(data, outputPtr[]): GetResult.found
@@ -700,7 +700,7 @@ proc putSnappySSZ(db: KvStoreRef, key: openArray[byte], v: auto) =
 proc getSZSSZ[T](db: KvStoreRef, key: openArray[byte], output: var T): GetResult =
   var status = GetResult.notFound
 
-  var outputPtr = addr output # callback is local, ptr wont escape
+  let outputPtr = addr output # callback is local, ptr wont escape
   proc decode(data: openArray[byte]) =
     status =
       if decodeSZSSZ(data, outputPtr[]): GetResult.found
@@ -1312,6 +1312,8 @@ proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool =
     db.backend.contains(subkey(phase0.BeaconState, key)).expectDb()
 
 proc containsState*(db: BeaconChainDB, key: Eth2Digest, legacy: bool = true): bool =
+  db.statesNoVal[BeaconStateFork.EIP4844].contains(key.data).expectDb or
+  db.statesNoVal[BeaconStateFork.Capella].contains(key.data).expectDb or
   db.statesNoVal[BeaconStateFork.Bellatrix].contains(key.data).expectDb or
   db.statesNoVal[BeaconStateFork.Altair].contains(key.data).expectDb or
   db.statesNoVal[BeaconStateFork.Phase0].contains(key.data).expectDb or

beacon_chain/rpc/rest_beacon_api.nim

@@ -847,7 +847,7 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
     of BeaconStateFork.Bellatrix:
       let res =
         block:
-          var restBlock = decodeBodyJsonOrSsz(SignedBlindedBeaconBlock, body).valueOr:
+          let restBlock = decodeBodyJsonOrSsz(SignedBlindedBeaconBlock, body).valueOr:
             return RestApiResponse.jsonError(Http400, InvalidBlockObjectError,
                                              $error)
           await node.unblindAndRouteBlockMEV(restBlock)

tests/test_beacon_chain_db.nim

@@ -8,8 +8,7 @@
 {.used.}
 
 import
-  std/[algorithm, options, sequtils],
+  unittest2,
-  unittest2, snappy,
   ../beacon_chain/[beacon_chain_db, interop],
   ../beacon_chain/spec/[beaconstate, forks, state_transition],
   ../beacon_chain/spec/datatypes/[phase0, altair, bellatrix],
@@ -18,6 +17,10 @@ import
   # test utilies
   ./testutil, ./testdbutil, ./testblockutil, ./teststateutil
 
+from std/algorithm import sort
+from std/sequtils import toSeq
+from snappy import encodeFramed, uncompressedLenFramed
+
 when isMainModule:
   import chronicles # or some random compile error happens...
 
@@ -109,12 +112,18 @@ proc getTestStates(stateFork: BeaconStateFork): auto =
 
   testStates
 
-# Each of phase 0/altair/bellatrix states gets used twice, so scope them to
+# Each set of states gets used twice, so scope them to module
-# module
 let
   testStatesPhase0    = getTestStates(BeaconStateFork.Phase0)
   testStatesAltair    = getTestStates(BeaconStateFork.Altair)
   testStatesBellatrix = getTestStates(BeaconStateFork.Bellatrix)
+  testStatesCapella   = getTestStates(BeaconStateFork.Capella)
+  testStatesEIP4844   = getTestStates(BeaconStateFork.EIP4844)
+doAssert len(testStatesPhase0) > 8
+doAssert len(testStatesAltair) > 8
+doAssert len(testStatesBellatrix) > 8
+doAssert len(testStatesCapella) > 8
+doAssert len(testStatesEIP4844) > 8
 
 suite "Beacon chain DB" & preset():
   test "empty database" & preset():
@@ -413,6 +422,42 @@ suite "Beacon chain DB" & preset():
 
     db.close()
 
+  test "sanity check Capella states" & preset():
+    let db = makeTestDB(SLOTS_PER_EPOCH)
+
+    for state in testStatesCapella:
+      let root = state[].capellaData.root
+      db.putState(root, state[].capellaData.data)
+
+      check:
+        db.containsState(root)
+        hash_tree_root(db.getCapellaStateRef(root)[]) == root
+
+      db.delState(root)
+      check:
+        not db.containsState(root)
+        db.getCapellaStateRef(root).isNil
+
+    db.close()
+
+  test "sanity check EIP4844 states" & preset():
+    let db = makeTestDB(SLOTS_PER_EPOCH)
+
+    for state in testStatesEIP4844:
+      let root = state[].eip4844Data.root
+      db.putState(root, state[].eip4844Data.data)
+
+      check:
+        db.containsState(root)
+        hash_tree_root(db.getEIP4844StateRef(root)[]) == root
+
+      db.delState(root)
+      check:
+        not db.containsState(root)
+        db.getEIP4844StateRef(root).isNil
+
+    db.close()
+
   test "sanity check phase 0 states, reusing buffers" & preset():
     let db = makeTestDB(SLOTS_PER_EPOCH)
     let stateBuffer = (phase0.BeaconStateRef)()
@@ -473,6 +518,46 @@ suite "Beacon chain DB" & preset():
 
     db.close()
 
+  test "sanity check Capella states, reusing buffers" & preset():
+    let db = makeTestDB(SLOTS_PER_EPOCH)
+    let stateBuffer = (capella.BeaconStateRef)()
+
+    for state in testStatesCapella:
+      let root = state[].capellaData.root
+      db.putState(root, state[].capellaData.data)
+
+      check:
+        db.getState(root, stateBuffer[], noRollback)
+        db.containsState(root)
+        hash_tree_root(stateBuffer[]) == root
+
+      db.delState(root)
+      check:
+        not db.containsState(root)
+        not db.getState(root, stateBuffer[], noRollback)
+
+    db.close()
+
+  test "sanity check EIP4844 states, reusing buffers" & preset():
+    let db = makeTestDB(SLOTS_PER_EPOCH)
+    let stateBuffer = (eip4844.BeaconStateRef)()
+
+    for state in testStatesEIP4844:
+      let root = state[].eip4844Data.root
+      db.putState(root, state[].eip4844Data.data)
+
+      check:
+        db.getState(root, stateBuffer[], noRollback)
+        db.containsState(root)
+        hash_tree_root(stateBuffer[]) == root
+
+      db.delState(root)
+      check:
+        not db.containsState(root)
+        not db.getState(root, stateBuffer[], noRollback)
+
+    db.close()
+
   test "sanity check phase 0 getState rollback" & preset():
     var
       db = makeTestDB(SLOTS_PER_EPOCH)
@@ -548,6 +633,58 @@ suite "Beacon chain DB" & preset():
       state[].kind == BeaconStateFork.Phase0
       state[].phase0Data.data.slot != 10.Slot
 
+  test "sanity check Capella and cross-fork getState rollback" & preset():
+    var
+      db = makeTestDB(SLOTS_PER_EPOCH)
+      validatorMonitor = newClone(ValidatorMonitor.init())
+      dag = init(ChainDAGRef, defaultRuntimeConfig, db, validatorMonitor, {})
+      state = (ref ForkedHashedBeaconState)(
+        kind: BeaconStateFork.Capella,
+        capellaData: capella.HashedBeaconState(data: capella.BeaconState(
+          slot: 10.Slot)))
+      root = Eth2Digest()
+
+    db.putCorruptState(BeaconStateFork.Capella, root)
+
+    let restoreAddr = addr dag.headState
+
+    func restore() =
+      assign(state[], restoreAddr[])
+
+    check:
+      state[].capellaData.data.slot == 10.Slot
+      not db.getState(root, state[].capellaData.data, restore)
+
+      # assign() has switched the case object fork
+      state[].kind == BeaconStateFork.Phase0
+      state[].phase0Data.data.slot != 10.Slot
+
+  test "sanity check EIP4844 and cross-fork getState rollback" & preset():
+    var
+      db = makeTestDB(SLOTS_PER_EPOCH)
+      validatorMonitor = newClone(ValidatorMonitor.init())
+      dag = init(ChainDAGRef, defaultRuntimeConfig, db, validatorMonitor, {})
+      state = (ref ForkedHashedBeaconState)(
+        kind: BeaconStateFork.EIP4844,
+        eip4844Data: eip4844.HashedBeaconState(data: eip4844.BeaconState(
+          slot: 10.Slot)))
+      root = Eth2Digest()
+
+    db.putCorruptState(BeaconStateFork.EIP4844, root)
+
+    let restoreAddr = addr dag.headState
+
+    func restore() =
+      assign(state[], restoreAddr[])
+
+    check:
+      state[].eip4844Data.data.slot == 10.Slot
+      not db.getState(root, state[].eip4844Data.data, restore)
+
+      # assign() has switched the case object fork
+      state[].kind == BeaconStateFork.Phase0
+      state[].phase0Data.data.slot != 10.Slot
+
   test "find ancestors" & preset():
     var
       db = BeaconChainDB.new("", inMemory = true)

tests/teststateutil.nim

@@ -68,11 +68,15 @@ proc getTestStates*(
     info = ForkedEpochInfo()
     cfg = defaultRuntimeConfig
 
-  if stateFork in [BeaconStateFork.Altair, BeaconStateFork.Bellatrix]:
+  static: doAssert high(BeaconStateFork) == BeaconStateFork.EIP4844
+  if stateFork >= BeaconStateFork.Altair:
     cfg.ALTAIR_FORK_EPOCH = 1.Epoch
-
+  if stateFork >= BeaconStateFork.Bellatrix:
-  if stateFork == BeaconStateFork.Bellatrix:
+    cfg.BELLATRIX_FORK_EPOCH = 2.Epoch
-    cfg.BELLATRIX_FORK_EPOCH = 1.Epoch
+  if stateFork >= BeaconStateFork.Capella:
+    cfg.CAPELLA_FORK_EPOCH = 3.Epoch
+  if stateFork >= BeaconStateFork.EIP4844:
+    cfg.EIP4844_FORK_EPOCH = 4.Epoch
 
   for i, epoch in stateEpochs:
     let slot = epoch.Epoch.start_slot