add column support to beacon chain db (#6660)

* add column support to beacon chain db

* slight fix

* fix db util

* fix import issues

* gate fulu columns if fulu fork epoch is far future epoch

* fixed all tests file

* fix comment

AllTests-mainnet.md

@@ -66,6 +66,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + sanity check Fulu states [Preset: mainnet]                                                 OK
 + sanity check Fulu states, reusing buffers [Preset: mainnet]                                OK
 + sanity check blobs [Preset: mainnet]                                                       OK
++ sanity check data columns [Preset: mainnet]                                                OK
 + sanity check genesis roundtrip [Preset: mainnet]                                           OK
 + sanity check phase 0 blocks [Preset: mainnet]                                              OK
 + sanity check phase 0 getState rollback [Preset: mainnet]                                   OK
@@ -73,7 +74,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + sanity check phase 0 states, reusing buffers [Preset: mainnet]                             OK
 + sanity check state diff roundtrip [Preset: mainnet]                                        OK
 ```
-OK: 33/33 Fail: 0/33 Skip: 0/33
+OK: 34/34 Fail: 0/34 Skip: 0/34
 ## Beacon chain file test suite
 ```diff
 + Auto check/repair test (missing data)                                                      OK

beacon_chain/beacon_chain_db.nim

@@ -20,7 +20,6 @@ import
           forks,
           presets,
           state_transition],
-  ./spec/datatypes/[phase0, altair, bellatrix],
   "."/[beacon_chain_db_light_client, filepath]
 
 from ./spec/datatypes/capella import BeaconState
@@ -116,6 +115,8 @@ type
 
     blobs: KvStoreRef # (BlockRoot -> BlobSidecar)
 
+    columns: KvStoreRef # (BlockRoot -> DataColumnSidecar)
+
     stateRoots: KvStoreRef # (Slot, BlockRoot) -> StateRoot
 
     statesNoVal: array[ConsensusFork, KvStoreRef] # StateRoot -> ForkBeaconStateNoImmutableValidators
@@ -254,6 +255,13 @@ func blobkey(root: Eth2Digest, index: BlobIndex) : array[40, byte] =
 
   ret
 
+func columnkey(root: Eth2Digest, index: ColumnIndex) : array[40, byte] =
+  var ret: array[40, byte]
+  ret[0..<8] = toBytes(index)
+  ret[8..<40] = root.data
+
+  ret
+
 template expectDb(x: auto): untyped =
   # There's no meaningful error handling implemented for a corrupt database or
   # full disk - this requires manual intervention, so we'll panic for now
@@ -581,6 +589,10 @@ proc new*(T: type BeaconChainDB,
 
   var blobs = kvStore db.openKvStore("deneb_blobs").expectDb()
 
+  var columns: KvStoreRef
+  if cfg.FULU_FORK_EPOCH != FAR_FUTURE_EPOCH:
+    columns = kvStore db.openKvStore("fulu_columns").expectDb()
+
   # Versions prior to 1.4.0 (altair) stored validators in `immutable_validators`
   # which stores validator keys in compressed format - this is
   # slow to load and has been superceded by `immutable_validators2` which uses
@@ -616,6 +628,7 @@ proc new*(T: type BeaconChainDB,
     keyValues: keyValues,
     blocks: blocks,
     blobs: blobs,
+    columns: columns,
     stateRoots: stateRoots,
     statesNoVal: statesNoVal,
     stateDiffs: stateDiffs,
@@ -783,6 +796,8 @@ proc close*(db: BeaconChainDB) =
   if db.db == nil: return
 
   # Close things roughly in reverse order
+  if not isNil(db.columns):
+    discard db.columns.close()
   if not isNil(db.blobs):
     discard db.blobs.close()
   db.lcData.close()
@@ -841,6 +856,17 @@ proc delBlobSidecar*(
     root: Eth2Digest, index: BlobIndex): bool =
   db.blobs.del(blobkey(root, index)).expectDb()
 
+proc putDataColumnSidecar*(
+    db: BeaconChainDB,
+    value: DataColumnSidecar) =
+  let block_root = hash_tree_root(value.signed_block_header.message)
+  db.columns.putSZSSZ(columnkey(block_root, value.index), value)
+
+proc delDataColumnSidecar*(
+    db: BeaconChainDB,
+    root: Eth2Digest, index: ColumnIndex): bool =
+  db.columns.del(columnkey(root, index)).expectDb()
+
 proc updateImmutableValidators*(
     db: BeaconChainDB, validators: openArray[Validator]) =
   # Must be called before storing a state that references the new validators
@@ -1105,6 +1131,17 @@ proc getBlobSidecar*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex,
                      value: var BlobSidecar): bool =
   db.blobs.getSZSSZ(blobkey(root, index), value) == GetResult.found
 
+proc getDataColumnSidecarSZ*(db: BeaconChainDB, root: Eth2Digest, 
+                             index: ColumnIndex, data: var seq[byte]): bool =
+  let dataPtr = addr data # Short-lived
+  func decode(data: openArray[byte]) =
+    assign(dataPtr[], data)
+  db.columns.get(columnkey(root, index), decode).expectDb()
+
+proc getDataColumnSidecar*(db: BeaconChainDB, root: Eth2Digest, index: ColumnIndex,
+                           value: var DataColumnSidecar): bool =
+  db.columns.getSZSSZ(columnkey(root, index), value) == GetResult.found
+
 proc getBlockSZ*(
     db: BeaconChainDB, key: Eth2Digest, data: var seq[byte],
     T: type phase0.TrustedSignedBeaconBlock): bool =

tests/test_beacon_chain_db.nim

@@ -1140,6 +1140,106 @@ suite "Beacon chain DB" & preset():
 
     db.close()
 
+  test "sanity check data columns" & preset():
+    const
+      blockHeader0 = SignedBeaconBlockHeader(
+        message: BeaconBlockHeader(slot: Slot(0)))
+      blockHeader1 = SignedBeaconBlockHeader(
+        message: BeaconBlockHeader(slot: Slot(1)))
+
+    let
+      blockRoot0 = hash_tree_root(blockHeader0.message)
+      blockRoot1 = hash_tree_root(blockHeader1.message)
+
+      # Ensure minimal-difference pairs on both block root and 
+      # data column index to verify that the columnkey uses both
+      dataColumnSidecar0 = DataColumnSidecar(signed_block_header: blockHeader0, index: 3)
+      dataColumnSidecar1 = DataColumnSidecar(signed_block_header: blockHeader0, index: 2)
+      dataColumnSidecar2 = DataColumnSidecar(signed_block_header: blockHeader1, index: 2)
+
+      db = makeTestDB(SLOTS_PER_EPOCH)
+
+    var
+      buf: seq[byte]
+      dataColumnSidecar: DataColumnSidecar
+
+    check:
+      not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      not db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    db.putDataColumnSidecar(dataColumnSidecar0)
+    
+    check:
+      db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar0
+      not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      not db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      not db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    db.putDataColumnSidecar(dataColumnSidecar1)
+
+    check:
+      db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar0
+      db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar1
+      not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      not db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    check db.delDataColumnSidecar(blockRoot0, 3)
+
+    check:
+      not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar1
+      not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      not db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    db.putDataColumnSidecar(dataColumnSidecar2)
+
+    check:
+      not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar1
+      db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar2
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    check db.delDataColumnSidecar(blockRoot0, 2)
+
+    check:
+      not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      dataColumnSidecar == dataColumnSidecar2
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      not db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    check db.delDataColumnSidecar(blockRoot1, 2)
+
+    check:
+      not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar)
+      not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar)
+      not db.getDataColumnSidecarSZ(blockRoot0, 3, buf)
+      not db.getDataColumnSidecarSZ(blockRoot0, 2, buf)
+      not db.getDataColumnSidecarSZ(blockRoot1, 2, buf)
+
+    db.close()
+
 suite "FinalizedBlocks" & preset():
   test "Basic ops" & preset():
     var

tests/testdbutil.nim

@@ -26,6 +26,7 @@ proc makeTestDB*(
     flags: UpdateFlags = {},
     cfg = defaultRuntimeConfig): BeaconChainDB =
   # Blob support requires DENEB_FORK_EPOCH != FAR_FUTURE_EPOCH
+  # Data column support requires FULU_FORK_EPOCH != FAR_FUTURE_EPOCH
   var cfg = cfg
   if cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH:
     cfg.CAPELLA_FORK_EPOCH = 90000.Epoch
@@ -36,7 +37,6 @@ proc makeTestDB*(
   if cfg.FULU_FORK_EPOCH == FAR_FUTURE_EPOCH:
     cfg.FULU_FORK_EPOCH = 120000.Epoch
 
-
   var genState = (ref ForkedHashedBeaconState)(
     kind: ConsensusFork.Phase0,
     phase0Data: initialize_hashed_beacon_state_from_eth1(