From 99bb8452276a38b8b4ea39f682b94186f0eb7432 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh <80243668+agnxsh@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:53:13 +0700 Subject: [PATCH] add column support to beacon chain db (#6660) * add column support to beacon chain db * slight fix * fix db util * fix import issues * gate fulu columns if fulu fork epoch is far future epoch * fixed all tests file * fix comment --- AllTests-mainnet.md | 3 +- beacon_chain/beacon_chain_db.nim | 39 +++++++++++- tests/test_beacon_chain_db.nim | 100 +++++++++++++++++++++++++++++++ tests/testdbutil.nim | 2 +- 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 92bdcff3b..f4b326137 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -66,6 +66,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + sanity check Fulu states [Preset: mainnet] OK + sanity check Fulu states, reusing buffers [Preset: mainnet] OK + sanity check blobs [Preset: mainnet] OK ++ sanity check data columns [Preset: mainnet] OK + sanity check genesis roundtrip [Preset: mainnet] OK + sanity check phase 0 blocks [Preset: mainnet] OK + sanity check phase 0 getState rollback [Preset: mainnet] OK @@ -73,7 +74,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + sanity check phase 0 states, reusing buffers [Preset: mainnet] OK + sanity check state diff roundtrip [Preset: mainnet] OK ``` -OK: 33/33 Fail: 0/33 Skip: 0/33 +OK: 34/34 Fail: 0/34 Skip: 0/34 ## Beacon chain file test suite ```diff + Auto check/repair test (missing data) OK diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 747777bf6..3a0efac9b 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -20,7 +20,6 @@ import forks, presets, state_transition], - ./spec/datatypes/[phase0, altair, bellatrix], "."/[beacon_chain_db_light_client, filepath] from ./spec/datatypes/capella import BeaconState @@ -116,6 +115,8 @@ type blobs: KvStoreRef # (BlockRoot -> BlobSidecar) + columns: KvStoreRef # (BlockRoot -> DataColumnSidecar) + stateRoots: KvStoreRef # (Slot, BlockRoot) -> StateRoot statesNoVal: array[ConsensusFork, KvStoreRef] # StateRoot -> ForkBeaconStateNoImmutableValidators @@ -254,6 +255,13 @@ func blobkey(root: Eth2Digest, index: BlobIndex) : array[40, byte] = ret +func columnkey(root: Eth2Digest, index: ColumnIndex) : array[40, byte] = + var ret: array[40, byte] + ret[0..<8] = toBytes(index) + ret[8..<40] = root.data + + ret + template expectDb(x: auto): untyped = # There's no meaningful error handling implemented for a corrupt database or # full disk - this requires manual intervention, so we'll panic for now @@ -581,6 +589,10 @@ proc new*(T: type BeaconChainDB, var blobs = kvStore db.openKvStore("deneb_blobs").expectDb() + var columns: KvStoreRef + if cfg.FULU_FORK_EPOCH != FAR_FUTURE_EPOCH: + columns = kvStore db.openKvStore("fulu_columns").expectDb() + # Versions prior to 1.4.0 (altair) stored validators in `immutable_validators` # which stores validator keys in compressed format - this is # slow to load and has been superceded by `immutable_validators2` which uses @@ -616,6 +628,7 @@ proc new*(T: type BeaconChainDB, keyValues: keyValues, blocks: blocks, blobs: blobs, + columns: columns, stateRoots: stateRoots, statesNoVal: statesNoVal, stateDiffs: stateDiffs, @@ -783,6 +796,8 @@ proc close*(db: BeaconChainDB) = if db.db == nil: return # Close things roughly in reverse order + if not isNil(db.columns): + discard db.columns.close() if not isNil(db.blobs): discard db.blobs.close() db.lcData.close() @@ -841,6 +856,17 @@ proc delBlobSidecar*( root: Eth2Digest, index: BlobIndex): bool = db.blobs.del(blobkey(root, index)).expectDb() +proc putDataColumnSidecar*( + db: BeaconChainDB, + value: DataColumnSidecar) = + let block_root = hash_tree_root(value.signed_block_header.message) + db.columns.putSZSSZ(columnkey(block_root, value.index), value) + +proc delDataColumnSidecar*( + db: BeaconChainDB, + root: Eth2Digest, index: ColumnIndex): bool = + db.columns.del(columnkey(root, index)).expectDb() + proc updateImmutableValidators*( db: BeaconChainDB, validators: openArray[Validator]) = # Must be called before storing a state that references the new validators @@ -1105,6 +1131,17 @@ proc getBlobSidecar*(db: BeaconChainDB, root: Eth2Digest, index: BlobIndex, value: var BlobSidecar): bool = db.blobs.getSZSSZ(blobkey(root, index), value) == GetResult.found +proc getDataColumnSidecarSZ*(db: BeaconChainDB, root: Eth2Digest, + index: ColumnIndex, data: var seq[byte]): bool = + let dataPtr = addr data # Short-lived + func decode(data: openArray[byte]) = + assign(dataPtr[], data) + db.columns.get(columnkey(root, index), decode).expectDb() + +proc getDataColumnSidecar*(db: BeaconChainDB, root: Eth2Digest, index: ColumnIndex, + value: var DataColumnSidecar): bool = + db.columns.getSZSSZ(columnkey(root, index), value) == GetResult.found + proc getBlockSZ*( db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], T: type phase0.TrustedSignedBeaconBlock): bool = diff --git a/tests/test_beacon_chain_db.nim b/tests/test_beacon_chain_db.nim index 772e05e09..b56ddfcf3 100644 --- a/tests/test_beacon_chain_db.nim +++ b/tests/test_beacon_chain_db.nim @@ -1140,6 +1140,106 @@ suite "Beacon chain DB" & preset(): db.close() + test "sanity check data columns" & preset(): + const + blockHeader0 = SignedBeaconBlockHeader( + message: BeaconBlockHeader(slot: Slot(0))) + blockHeader1 = SignedBeaconBlockHeader( + message: BeaconBlockHeader(slot: Slot(1))) + + let + blockRoot0 = hash_tree_root(blockHeader0.message) + blockRoot1 = hash_tree_root(blockHeader1.message) + + # Ensure minimal-difference pairs on both block root and + # data column index to verify that the columnkey uses both + dataColumnSidecar0 = DataColumnSidecar(signed_block_header: blockHeader0, index: 3) + dataColumnSidecar1 = DataColumnSidecar(signed_block_header: blockHeader0, index: 2) + dataColumnSidecar2 = DataColumnSidecar(signed_block_header: blockHeader1, index: 2) + + db = makeTestDB(SLOTS_PER_EPOCH) + + var + buf: seq[byte] + dataColumnSidecar: DataColumnSidecar + + check: + not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + not db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + db.putDataColumnSidecar(dataColumnSidecar0) + + check: + db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar0 + not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + not db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + not db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + db.putDataColumnSidecar(dataColumnSidecar1) + + check: + db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar0 + db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar1 + not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + not db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + check db.delDataColumnSidecar(blockRoot0, 3) + + check: + not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar1 + not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + not db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + db.putDataColumnSidecar(dataColumnSidecar2) + + check: + not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar1 + db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar2 + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + check db.delDataColumnSidecar(blockRoot0, 2) + + check: + not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + dataColumnSidecar == dataColumnSidecar2 + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + not db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + check db.delDataColumnSidecar(blockRoot1, 2) + + check: + not db.getDataColumnSidecar(blockRoot0, 3, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot0, 2, dataColumnSidecar) + not db.getDataColumnSidecar(blockRoot1, 2, dataColumnSidecar) + not db.getDataColumnSidecarSZ(blockRoot0, 3, buf) + not db.getDataColumnSidecarSZ(blockRoot0, 2, buf) + not db.getDataColumnSidecarSZ(blockRoot1, 2, buf) + + db.close() + suite "FinalizedBlocks" & preset(): test "Basic ops" & preset(): var diff --git a/tests/testdbutil.nim b/tests/testdbutil.nim index 7a7c39cf9..495c23abd 100644 --- a/tests/testdbutil.nim +++ b/tests/testdbutil.nim @@ -26,6 +26,7 @@ proc makeTestDB*( flags: UpdateFlags = {}, cfg = defaultRuntimeConfig): BeaconChainDB = # Blob support requires DENEB_FORK_EPOCH != FAR_FUTURE_EPOCH + # Data column support requires FULU_FORK_EPOCH != FAR_FUTURE_EPOCH var cfg = cfg if cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH: cfg.CAPELLA_FORK_EPOCH = 90000.Epoch @@ -36,7 +37,6 @@ proc makeTestDB*( if cfg.FULU_FORK_EPOCH == FAR_FUTURE_EPOCH: cfg.FULU_FORK_EPOCH = 120000.Epoch - var genState = (ref ForkedHashedBeaconState)( kind: ConsensusFork.Phase0, phase0Data: initialize_hashed_beacon_state_from_eth1(