From adfe655b1694b7d9a100c7bae3518a68d822bbad Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 21 Feb 2022 09:48:02 +0100 Subject: [PATCH] db: make block loading generic (#3413) Streamline lookup with Forky and BeaconBlockFork (then we can do the same for era) We use type to avoid conditionals, as fork is often already known at a "higher" level. * load blockid before loading block by root - this is needed to map root to slot and will eventually be done via block summary table for "old" blocks Co-authored-by: tersec --- beacon_chain/beacon_chain_db.nim | 228 +++++++----------- .../consensus_object_pools/blockchain_dag.nim | 86 +++---- ncli/ncli_db.nim | 23 +- tests/test_beacon_chain_db.nim | 74 +++--- 4 files changed, 182 insertions(+), 229 deletions(-) diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index b16983b5d..a869fa92f 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -104,13 +104,12 @@ type checkpoint*: proc() {.gcsafe, raises: [Defect].} keyValues: KvStoreRef # Random stuff using DbKeyKind - suitable for small values mainly! - blocks: KvStoreRef # BlockRoot -> phase0.TrustedBeaconBlock - altairBlocks: KvStoreRef # BlockRoot -> altair.TrustedBeaconBlock - mergeBlocks: KvStoreRef # BlockRoot -> bellatrix.TrustedBeaconBlock + blocks: array[BeaconBlockFork, KvStoreRef] # BlockRoot -> TrustedSignedBeaconBlock + stateRoots: KvStoreRef # (Slot, BlockRoot) -> StateRoot - statesNoVal: KvStoreRef # StateRoot -> Phase0BeaconStateNoImmutableValidators - altairStatesNoVal: KvStoreRef # StateRoot -> AltairBeaconStateNoImmutableValidators - mergeStatesNoVal: KvStoreRef # StateRoot -> MergeBeaconStateNoImmutableValidators + + statesNoVal: array[BeaconStateFork, KvStoreRef] # StateRoot -> ForkBeaconStateNoImmutableValidators + stateDiffs: KvStoreRef ##\ ## StateRoot -> BeaconStateDiff ## Instead of storing full BeaconStates, one can store only the diff from @@ -418,13 +417,18 @@ proc new*(T: type BeaconChainDB, # V1 - expected-to-be small rows get without rowid optimizations keyValues = kvStore db.openKvStore("key_values", true).expectDb() - blocks = kvStore db.openKvStore("blocks").expectDb() - altairBlocks = kvStore db.openKvStore("altair_blocks").expectDb() - mergeBlocks = kvStore db.openKvStore("merge_blocks").expectDb() + blocks = [ + kvStore db.openKvStore("blocks").expectDb(), + kvStore db.openKvStore("altair_blocks").expectDb(), + kvStore db.openKvStore("merge_blocks").expectDb()] + stateRoots = kvStore db.openKvStore("state_roots", true).expectDb() - statesNoVal = kvStore db.openKvStore("state_no_validators2").expectDb() - altairStatesNoVal = kvStore db.openKvStore("altair_state_no_validators").expectDb() - mergeStatesNoVal = kvStore db.openKvStore("merge_state_no_validators").expectDb() + + statesNoVal = [ + kvStore db.openKvStore("state_no_validators2").expectDb(), + kvStore db.openKvStore("altair_state_no_validators").expectDb(), + kvStore db.openKvStore("merge_state_no_validators").expectDb()] + stateDiffs = kvStore db.openKvStore("state_diffs").expectDb() summaries = kvStore db.openKvStore("beacon_block_summaries", true).expectDb() finalizedBlocks = FinalizedBlocks.init(db, "finalized_blocks").expectDb() @@ -465,12 +469,8 @@ proc new*(T: type BeaconChainDB, checkpoint: proc() = db.checkpoint(), keyValues: keyValues, blocks: blocks, - altair_blocks: altair_blocks, - merge_blocks: merge_blocks, stateRoots: stateRoots, statesNoVal: statesNoVal, - altairStatesNoVal: altairStatesNoVal, - mergeStatesNoVal: mergeStatesNoVal, stateDiffs: stateDiffs, summaries: summaries, finalizedBlocks: finalizedBlocks, @@ -579,17 +579,13 @@ proc close*(db: BeaconChainDBV0) = proc close*(db: BeaconChainDB) = if db.db == nil: return - # Close things in reverse order + # Close things roughly in reverse order db.finalizedBlocks.close() discard db.summaries.close() discard db.stateDiffs.close() - discard db.mergeStatesNoVal.close() - discard db.altairStatesNoVal.close() - discard db.statesNoVal.close() + for kv in db.statesNoVal: discard kv.close() discard db.stateRoots.close() - discard db.mergeBlocks.close() - discard db.altairBlocks.close() - discard db.blocks.close() + for kv in db.blocks: discard kv.close() discard db.keyValues.close() db.immutableValidatorsDb.close() @@ -610,19 +606,9 @@ proc putBeaconBlockSummary( # Summaries are too simple / small to compress, store them as plain SSZ db.summaries.putSSZ(root.data, value) -proc putBlock*(db: BeaconChainDB, value: phase0.TrustedSignedBeaconBlock) = +proc putBlock*(db: BeaconChainDB, value: ForkyTrustedSignedBeaconBlock) = db.withManyWrites: - db.blocks.putSnappySSZ(value.root.data, value) - db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary()) - -proc putBlock*(db: BeaconChainDB, value: altair.TrustedSignedBeaconBlock) = - db.withManyWrites: - db.altairBlocks.putSnappySSZ(value.root.data, value) - db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary()) - -proc putBlock*(db: BeaconChainDB, value: bellatrix.TrustedSignedBeaconBlock) = - db.withManyWrites: - db.mergeBlocks.putSnappySSZ(value.root.data, value) + db.blocks[type(value).toFork].putSnappySSZ(value.root.data, value) db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary()) proc updateImmutableValidators*( @@ -651,19 +637,9 @@ template toBeaconStateNoImmutableValidators(state: bellatrix.BeaconState): MergeBeaconStateNoImmutableValidators = isomorphicCast[MergeBeaconStateNoImmutableValidators](state) -proc putState*(db: BeaconChainDB, key: Eth2Digest, value: phase0.BeaconState) = +proc putState*(db: BeaconChainDB, key: Eth2Digest, value: ForkyBeaconState) = db.updateImmutableValidators(value.validators.asSeq()) - db.statesNoVal.putSnappySSZ( - key.data, toBeaconStateNoImmutableValidators(value)) - -proc putState*(db: BeaconChainDB, key: Eth2Digest, value: altair.BeaconState) = - db.updateImmutableValidators(value.validators.asSeq()) - db.altairStatesNoVal.putSnappySSZ( - key.data, toBeaconStateNoImmutableValidators(value)) - -proc putState*(db: BeaconChainDB, key: Eth2Digest, value: bellatrix.BeaconState) = - db.updateImmutableValidators(value.validators.asSeq()) - db.mergeStatesNoVal.putSnappySSZ( + db.statesNoVal[type(value).toFork()].putSnappySSZ( key.data, toBeaconStateNoImmutableValidators(value)) proc putState*(db: BeaconChainDB, state: ForkyHashedBeaconState) = @@ -673,13 +649,13 @@ proc putState*(db: BeaconChainDB, state: ForkyHashedBeaconState) = # For testing rollback proc putCorruptPhase0State*(db: BeaconChainDB, key: Eth2Digest) = - db.statesNoVal.putSnappySSZ(key.data, Validator()) + db.statesNoVal[BeaconStateFork.Phase0].putSnappySSZ(key.data, Validator()) proc putCorruptAltairState*(db: BeaconChainDB, key: Eth2Digest) = - db.altairStatesNoVal.putSnappySSZ(key.data, Validator()) + db.statesNoVal[BeaconStateFork.Altair].putSnappySSZ(key.data, Validator()) proc putCorruptMergeState*(db: BeaconChainDB, key: Eth2Digest) = - db.mergeStatesNoVal.putSnappySSZ(key.data, Validator()) + db.statesNoVal[BeaconStateFork.Bellatrix].putSnappySSZ(key.data, Validator()) func stateRootKey(root: Eth2Digest, slot: Slot): array[40, byte] = var ret: array[40, byte] @@ -698,16 +674,12 @@ proc putStateDiff*(db: BeaconChainDB, root: Eth2Digest, value: BeaconStateDiff) proc delBlock*(db: BeaconChainDB, key: Eth2Digest) = db.withManyWrites: - db.blocks.del(key.data).expectDb() - db.altairBlocks.del(key.data).expectDb() - db.mergeBlocks.del(key.data).expectDb() + for kv in db.blocks: kv.del(key.data).expectDb() db.summaries.del(key.data).expectDb() proc delState*(db: BeaconChainDB, key: Eth2Digest) = db.withManyWrites: - db.statesNoVal.del(key.data).expectDb() - db.altairStatesNoVal.del(key.data).expectDb() - db.mergeStatesNoVal.del(key.data).expectDb() + for kv in db.statesNoVal: kv.del(key.data).expectDb() proc delStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot) = db.stateRoots.del(stateRootKey(root, slot)).expectDb() @@ -728,7 +700,8 @@ proc putEth2FinalizedTo*(db: BeaconChainDB, eth1Checkpoint: DepositContractSnapshot) = db.keyValues.putSnappySSZ(subkey(kDepositsFinalizedByEth2), eth1Checkpoint) -proc getPhase0Block(db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSignedBeaconBlock] = +proc getPhase0Block( + db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSignedBeaconBlock] = # We only store blocks that we trust in the database result.ok(default(phase0.TrustedSignedBeaconBlock)) if db.backend.getSnappySSZ( @@ -738,31 +711,25 @@ proc getPhase0Block(db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSig # set root after deserializing (so it doesn't get zeroed) result.get().root = key -proc getPhase0Block*(db: BeaconChainDB, key: Eth2Digest): - Opt[phase0.TrustedSignedBeaconBlock] = +proc getBlock*( + db: BeaconChainDB, key: Eth2Digest, + T: type phase0.TrustedSignedBeaconBlock): Opt[T] = # We only store blocks that we trust in the database - result.ok(default(phase0.TrustedSignedBeaconBlock)) - if db.blocks.getSnappySSZ(key.data, result.get) != GetResult.found: + result.ok(default(T)) + if db.blocks[T.toFork].getSnappySSZ(key.data, result.get) != GetResult.found: + # During the initial releases phase0, we stored blocks in a different table result = db.v0.getPhase0Block(key) else: # set root after deserializing (so it doesn't get zeroed) result.get().root = key -proc getAltairBlock*(db: BeaconChainDB, key: Eth2Digest): - Opt[altair.TrustedSignedBeaconBlock] = +proc getBlock*[ + X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock]( + db: BeaconChainDB, key: Eth2Digest, + T: type X): Opt[T] = # We only store blocks that we trust in the database - result.ok(default(altair.TrustedSignedBeaconBlock)) - if db.altairBlocks.getSnappySSZ(key.data, result.get) == GetResult.found: - # set root after deserializing (so it doesn't get zeroed) - result.get().root = key - else: - result.err() - -proc getMergeBlock*(db: BeaconChainDB, key: Eth2Digest): - Opt[bellatrix.TrustedSignedBeaconBlock] = - # We only store blocks that we trust in the database - result.ok(default(bellatrix.TrustedSignedBeaconBlock)) - if db.mergeBlocks.getSnappySSZ(key.data, result.get) == GetResult.found: + result.ok(default(T)) + if db.blocks[T.toFork].getSnappySSZ(key.data, result.get) == GetResult.found: # set root after deserializing (so it doesn't get zeroed) result.get().root = key else: @@ -776,31 +743,39 @@ proc getPhase0BlockSSZ(db: BeaconChainDBV0, key: Eth2Digest, data: var seq[byte] except CatchableError: success = false db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and success -proc getPhase0BlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = +# SSZ implementations are separate so as to avoid unnecessary data copies +proc getBlockSSZ*( + db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], + T: type phase0.TrustedSignedBeaconBlock): bool = let dataPtr = unsafeAddr data # Short-lived var success = true proc decode(data: openArray[byte]) = try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) except CatchableError: success = false - db.blocks.get(key.data, decode).expectDb() and success or + db.blocks[BeaconBlockFork.Phase0].get(key.data, decode).expectDb() and success or db.v0.getPhase0BlockSSZ(key, data) -proc getAltairBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = +proc getBlockSSZ*[ + X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock]( + db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], + T: type X): bool = let dataPtr = unsafeAddr data # Short-lived var success = true proc decode(data: openArray[byte]) = try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) except CatchableError: success = false - db.altairBlocks.get(key.data, decode).expectDb() and success + db.blocks[T.toFork].get(key.data, decode).expectDb() and success -proc getMergeBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = - let dataPtr = unsafeAddr data # Short-lived - var success = true - proc decode(data: openArray[byte]) = - try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) - except CatchableError: success = false - - db.mergeBlocks.get(key.data, decode).expectDb() and success +proc getBlockSSZ*( + db: BeaconChainDB, key: Eth2Digest, data: var seq[byte], + fork: BeaconBlockFork): bool = + case fork + of BeaconBlockFork.Phase0: + getBlockSSZ(db, key, data, phase0.TrustedSignedBeaconBlock) + of BeaconBlockFork.Altair: + getBlockSSZ(db, key, data, altair.TrustedSignedBeaconBlock) + of BeaconBlockFork.Bellatrix: + getBlockSSZ(db, key, data, bellatrix.TrustedSignedBeaconBlock) proc getStateOnlyMutableValidators( immutableValidators: openArray[ImmutableValidatorData2], @@ -885,14 +860,17 @@ proc getState*( # https://github.com/nim-lang/Nim/issues/14126 # TODO RVO is inefficient for large objects: # https://github.com/nim-lang/Nim/issues/13879 + type T = type(output) + if not getStateOnlyMutableValidators( - db.immutableValidators, db.statesNoVal, key.data, output, rollback): + db.immutableValidators, db.statesNoVal[T.toFork], key.data, output, rollback): db.v0.getState(db.immutableValidators, key, output, rollback) else: true proc getState*( - db: BeaconChainDB, key: Eth2Digest, output: var altair.BeaconState, + db: BeaconChainDB, key: Eth2Digest, + output: var (altair.BeaconState | bellatrix.BeaconState), rollback: RollbackProc): bool = ## Load state into `output` - BeaconState is large so we want to avoid ## re-allocating it if possible @@ -904,24 +882,10 @@ proc getState*( # https://github.com/nim-lang/Nim/issues/14126 # TODO RVO is inefficient for large objects: # https://github.com/nim-lang/Nim/issues/13879 + type T = type(output) getStateOnlyMutableValidators( - db.immutableValidators, db.altairStatesNoVal, key.data, output, rollback) - -proc getState*( - db: BeaconChainDB, key: Eth2Digest, output: var bellatrix.BeaconState, - rollback: RollbackProc): bool = - ## Load state into `output` - BeaconState is large so we want to avoid - ## re-allocating it if possible - ## Return `true` iff the entry was found in the database and `output` was - ## overwritten. - ## Rollback will be called only if output was partially written - if it was - ## not found at all, rollback will not be called - # TODO rollback is needed to deal with bug - use `noRollback` to ignore: - # https://github.com/nim-lang/Nim/issues/14126 - # TODO RVO is inefficient for large objects: - # https://github.com/nim-lang/Nim/issues/13879 - getStateOnlyMutableValidators( - db.immutableValidators, db.mergeStatesNoVal, key.data, output, rollback) + db.immutableValidators, db.statesNoVal[T.toFork], key.data, output, + rollback) proc getStateRoot(db: BeaconChainDBV0, root: Eth2Digest, @@ -974,19 +938,26 @@ proc getEth2FinalizedTo*(db: BeaconChainDB): Opt[DepositContractSnapshot] = proc containsBlock*(db: BeaconChainDBV0, key: Eth2Digest): bool = db.backend.contains(subkey(phase0.SignedBeaconBlock, key)).expectDb() -proc containsBlockPhase0*(db: BeaconChainDB, key: Eth2Digest): bool = - db.blocks.contains(key.data).expectDb() or +proc containsBlock*( + db: BeaconChainDB, key: Eth2Digest, + T: type phase0.TrustedSignedBeaconBlock): bool = + db.blocks[T.toFork].contains(key.data).expectDb() or db.v0.containsBlock(key) -proc containsBlockAltair*(db: BeaconChainDB, key: Eth2Digest): bool = - db.altairBlocks.contains(key.data).expectDb() +proc containsBlock*[ + X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock]( + db: BeaconChainDB, key: Eth2Digest, T: type X): bool = + db.blocks[X.toFork].contains(key.data).expectDb() -proc containsBlockMerge*(db: BeaconChainDB, key: Eth2Digest): bool = - db.mergeBlocks.contains(key.data).expectDb() +proc containsBlock*(db: BeaconChainDB, key: Eth2Digest, fork: BeaconBlockFork): bool = + case fork + of BeaconBlockFork.Phase0: containsBlock(db, key, phase0.TrustedSignedBeaconBlock) + else: db.blocks[fork].contains(key.data).expectDb() proc containsBlock*(db: BeaconChainDB, key: Eth2Digest): bool = - db.containsBlockMerge(key) or db.containsBlockAltair(key) or - db.containsBlockPhase0(key) + db.containsBlock(key, bellatrix.TrustedSignedBeaconBlock) or + db.containsBlock(key, altair.TrustedSignedBeaconBlock) or + db.containsBlock(key, phase0.TrustedSignedBeaconBlock) proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool = let sk = subkey(Phase0BeaconStateNoImmutableValidators, key) @@ -995,28 +966,11 @@ proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool = db.backend.contains(subkey(phase0.BeaconState, key)).expectDb() proc containsState*(db: BeaconChainDB, key: Eth2Digest, legacy: bool = true): bool = - db.mergeStatesNoVal.contains(key.data).expectDb or - db.altairStatesNoVal.contains(key.data).expectDb or - db.statesNoVal.contains(key.data).expectDb or + db.statesNoVal[BeaconStateFork.Bellatrix].contains(key.data).expectDb or + db.statesNoVal[BeaconStateFork.Altair].contains(key.data).expectDb or + db.statesNoVal[BeaconStateFork.Phase0].contains(key.data).expectDb or (legacy and db.v0.containsState(key)) -iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest): - phase0.TrustedSignedBeaconBlock = - ## Load a chain of ancestors for blck - returns a list of blocks with the - ## oldest block last (blck will be at result[0]). - ## - ## The search will go on until the ancestor cannot be found. - - var - res: phase0.TrustedSignedBeaconBlock - root = root - while db.blocks.getSnappySSZ(root.data, res) == GetResult.found or - db.v0.backend.getSnappySSZ( - subkey(phase0.SignedBeaconBlock, root), res) == GetResult.found: - res.root = root - yield res - root = res.message.parent_root - proc getBeaconBlockSummary*(db: BeaconChainDB, root: Eth2Digest): Opt[BeaconBlockSummary] = var summary: BeaconBlockSummary @@ -1107,11 +1061,11 @@ iterator getAncestorSummaries*(db: BeaconChainDB, root: Eth2Digest): if db.v0.backend.getSnappySSZ( subkey(BeaconBlockSummary, res.root), res.summary) == GetResult.found: discard # Just yield below - elif (let blck = db.getPhase0Block(res.root); blck.isSome()): + elif (let blck = db.getBlock(res.root, phase0.TrustedSignedBeaconBlock); blck.isSome()): res.summary = blck.get().message.toBeaconBlockSummary() - elif (let blck = db.getAltairBlock(res.root); blck.isSome()): + elif (let blck = db.getBlock(res.root, altair.TrustedSignedBeaconBlock); blck.isSome()): res.summary = blck.get().message.toBeaconBlockSummary() - elif (let blck = db.getMergeBlock(res.root); blck.isSome()): + elif (let blck = db.getBlock(res.root, bellatrix.TrustedSignedBeaconBlock); blck.isSome()): res.summary = blck.get().message.toBeaconBlockSummary() else: break diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index e2963fb96..1894b1d0e 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -13,8 +13,8 @@ import metrics, snappy, chronicles, ../spec/[beaconstate, eth2_merkleization, eth2_ssz_serialization, helpers, state_transition, validator], - ../spec/datatypes/[phase0, altair], ".."/beacon_chain_db, + ../spec/datatypes/[phase0, altair, bellatrix], "."/[block_pools_types, block_quarantine] export @@ -344,10 +344,7 @@ func containsForkBlock*(dag: ChainDAGRef, root: Eth2Digest): bool = proc containsBlock( cfg: RuntimeConfig, db: BeaconChainDB, slot: Slot, root: Eth2Digest): bool = - case cfg.blockForkAtEpoch(slot.epoch) - of BeaconBlockFork.Phase0: db.containsBlockPhase0(root) - of BeaconBlockFork.Altair: db.containsBlockAltair(root) - of BeaconBlockFork.Bellatrix: db.containsBlockMerge(root) + db.containsBlock(root, cfg.blockForkAtEpoch(slot.epoch)) func isStateCheckpoint(bs: BlockSlot): bool = ## State checkpoints are the points in time for which we store full state @@ -398,40 +395,57 @@ proc getForkedBlock*(db: BeaconChainDB, root: Eth2Digest): Opt[ForkedTrustedSignedBeaconBlock] = # When we only have a digest, we don't know which fork it's from so we try # them one by one - this should be used sparingly - if (let blck = db.getMergeBlock(root); blck.isSome()): + if (let blck = db.getBlock(root, bellatrix.TrustedSignedBeaconBlock); + blck.isSome()): ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) - elif (let blck = db.getAltairBlock(root); blck.isSome()): + elif (let blck = db.getBlock(root, altair.TrustedSignedBeaconBlock); + blck.isSome()): ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) - elif (let blck = db.getPhase0Block(root); blck.isSome()): + elif (let blck = db.getBlock(root, phase0.TrustedSignedBeaconBlock); + blck.isSome()): ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) else: err() -proc getForkedBlock*( - dag: ChainDAGRef, root: Eth2Digest): Opt[ForkedTrustedSignedBeaconBlock] = - dag.db.getForkedBlock(root) +proc getBlock*( + dag: ChainDAGRef, bid: BlockId, + T: type ForkyTrustedSignedBeaconBlock): Opt[T] = + withState(dag.headState.data): + dag.db.getBlock(bid.root, T) + +proc getBlockSSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool = + # Load the SSZ-encoded data of a block into `bytes`, overwriting the existing + # content + # careful: there are two snappy encodings in use, with and without framing! + # Returns true if the block is found, false if not + let fork = dag.cfg.blockForkAtEpoch(bid.slot.epoch) + dag.db.getBlockSSZ(bid.root, bytes, fork) proc getForkedBlock*( - dag: ChainDAGRef, id: BlockId): Opt[ForkedTrustedSignedBeaconBlock] = - case dag.cfg.blockForkAtEpoch(id.slot.epoch) - of BeaconBlockFork.Phase0: - let data = dag.db.getPhase0Block(id.root) - if data.isOk(): - return ok ForkedTrustedSignedBeaconBlock.init(data.get) - of BeaconBlockFork.Altair: - let data = dag.db.getAltairBlock(id.root) - if data.isOk(): - return ok ForkedTrustedSignedBeaconBlock.init(data.get) - of BeaconBlockFork.Bellatrix: - let data = dag.db.getMergeBlock(id.root) - if data.isOk(): - return ok ForkedTrustedSignedBeaconBlock.init(data.get) + dag: ChainDAGRef, bid: BlockId): Opt[ForkedTrustedSignedBeaconBlock] = + + let fork = dag.cfg.blockForkAtEpoch(bid.slot.epoch) + result.ok(ForkedTrustedSignedBeaconBlock(kind: fork)) + withBlck(result.get()): + type T = type(blck) + blck = getBlock(dag, bid, T).valueOr: + result.err() + return proc getForkedBlock*( dag: ChainDAGRef, blck: BlockRef): ForkedTrustedSignedBeaconBlock = dag.getForkedBlock(blck.bid).expect( "BlockRef block should always load, database corrupt?") +proc getForkedBlock*( + dag: ChainDAGRef, root: Eth2Digest): Opt[ForkedTrustedSignedBeaconBlock] = + let bid = dag.getBlockId(root) + if bid.isSome(): + dag.getForkedBlock(bid.get()) + else: + # In case we didn't have a summary - should be rare, but .. + dag.db.getForkedBlock(root) + proc updateBeaconMetrics(state: StateData, cache: var StateCache) = # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics # both non-negative, so difference can't overflow or underflow int64 @@ -990,19 +1004,22 @@ proc applyBlock( case dag.cfg.blockForkAtEpoch(blck.slot.epoch) of BeaconBlockFork.Phase0: - let data = dag.db.getPhase0Block(blck.root).expect("block loaded") + let data = getBlock(dag, blck.bid, phase0.TrustedSignedBeaconBlock).expect( + "block loaded") state_transition( dag.cfg, state.data, data, cache, info, dag.updateFlags + {slotProcessed}, noRollback).expect( "Blocks from database must not fail to apply") of BeaconBlockFork.Altair: - let data = dag.db.getAltairBlock(blck.root).expect("block loaded") + let data = getBlock(dag, blck.bid, altair.TrustedSignedBeaconBlock).expect( + "block loaded") state_transition( dag.cfg, state.data, data, cache, info, dag.updateFlags + {slotProcessed}, noRollback).expect( "Blocks from database must not fail to apply") of BeaconBlockFork.Bellatrix: - let data = dag.db.getMergeBlock(blck.root).expect("block loaded") + let data = getBlock(dag, blck.bid, bellatrix.TrustedSignedBeaconBlock).expect( + "block loaded") state_transition( dag.cfg, state.data, data, cache, info, dag.updateFlags + {slotProcessed}, noRollback).expect( @@ -1737,18 +1754,5 @@ proc aggregateAll*( else: ok(finish(aggregateKey)) -proc getBlockSSZ*(dag: ChainDAGRef, id: BlockId, bytes: var seq[byte]): bool = - # Load the SSZ-encoded data of a block into `bytes`, overwriting the existing - # content - # careful: there are two snappy encodings in use, with and without framing! - # Returns true if the block is found, false if not - case dag.cfg.blockForkAtEpoch(id.slot.epoch) - of BeaconBlockFork.Phase0: - dag.db.getPhase0BlockSSZ(id.root, bytes) - of BeaconBlockFork.Altair: - dag.db.getAltairBlockSSZ(id.root, bytes) - of BeaconBlockFork.Bellatrix: - dag.db.getMergeBlockSSZ(id.root, bytes) - func needsBackfill*(dag: ChainDAGRef): bool = dag.backfill.slot > dag.genesis.slot diff --git a/ncli/ncli_db.nim b/ncli/ncli_db.nim index 0889bef74..b56d80497 100644 --- a/ncli/ncli_db.nim +++ b/ncli/ncli_db.nim @@ -232,11 +232,14 @@ proc cmdBench(conf: DbConf, cfg: RuntimeConfig) = withTimer(timers[tLoadBlock]): case cfg.blockForkAtEpoch(blck.slot.epoch) of BeaconBlockFork.Phase0: - blocks[0].add dag.db.getPhase0Block(blck.root).get() + blocks[0].add dag.db.getBlock( + blck.root, phase0.TrustedSignedBeaconBlock).get() of BeaconBlockFork.Altair: - blocks[1].add dag.db.getAltairBlock(blck.root).get() + blocks[1].add dag.db.getBlock( + blck.root, altair.TrustedSignedBeaconBlock).get() of BeaconBlockFork.Bellatrix: - blocks[2].add dag.db.getMergeBlock(blck.root).get() + blocks[2].add dag.db.getBlock( + blck.root, bellatrix.TrustedSignedBeaconBlock).get() let stateData = newClone(dag.headState) @@ -359,11 +362,13 @@ proc cmdDumpBlock(conf: DbConf) = if shouldShutDown: quit QuitSuccess try: let root = Eth2Digest.fromHex(blockRoot) - if (let blck = db.getPhase0Block(root); blck.isSome): + if (let blck = db.getBlock( + root, phase0.TrustedSignedBeaconBlock); blck.isSome): dump("./", blck.get()) - elif (let blck = db.getAltairBlock(root); blck.isSome): + elif (let blck = db.getBlock( + root, altair.TrustedSignedBeaconBlock); blck.isSome): dump("./", blck.get()) - elif (let blck = db.getMergeBlock(root); blck.isSome): + elif (let blck = db.getBlock(root, bellatrix.TrustedSignedBeaconBlock); blck.isSome): dump("./", blck.get()) else: echo "Couldn't load ", blockRoot @@ -511,7 +516,7 @@ proc cmdImportEra(conf: DbConf, cfg: RuntimeConfig) = let header = readRecord(f, data).valueOr: break - if header.typ == SnappyBeaconBlock: + if header.typ == SnappyBeaconBlock: withTimer(timers[tBlock]): let uncompressed = framingFormatUncompress(data) let blck = try: readSszForkedSignedBeaconBlock(cfg, uncompressed) @@ -646,7 +651,9 @@ proc cmdValidatorPerf(conf: DbConf, cfg: RuntimeConfig) = if shouldShutDown: quit QuitSuccess for bi in 0 ..< blockRefs.len: - blck = db.getPhase0Block(blockRefs[blockRefs.len - bi - 1].root).get() + blck = db.getBlock( + blockRefs[blockRefs.len - bi - 1].root, + phase0.TrustedSignedBeaconBlock).get() while getStateField(state[].data, slot) < blck.message.slot: let nextSlot = getStateField(state[].data, slot) + 1 diff --git a/tests/test_beacon_chain_db.nim b/tests/test_beacon_chain_db.nim index d075ff29a..d83d6783a 100644 --- a/tests/test_beacon_chain_db.nim +++ b/tests/test_beacon_chain_db.nim @@ -89,7 +89,7 @@ suite "Beacon chain DB" & preset(): db = BeaconChainDB.new("", inMemory = true) check: db.getPhase0StateRef(Eth2Digest()).isNil - db.getPhase0Block(Eth2Digest()).isNone + db.getBlock(Eth2Digest(), phase0.TrustedSignedBeaconBlock).isNone test "sanity check phase 0 blocks" & preset(): let db = BeaconChainDB.new("", inMemory = true) @@ -103,21 +103,21 @@ suite "Beacon chain DB" & preset(): var tmp: seq[byte] check: db.containsBlock(root) - db.containsBlockPhase0(root) - not db.containsBlockAltair(root) - not db.containsBlockMerge(root) - db.getPhase0Block(root).get() == signedBlock - db.getPhase0BlockSSZ(root, tmp) + db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + not db.containsBlock(root, altair.TrustedSignedBeaconBlock) + not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, phase0.TrustedSignedBeaconBlock).get() == signedBlock + db.getBlockSSZ(root, tmp, phase0.TrustedSignedBeaconBlock) tmp == SSZ.encode(signedBlock) db.delBlock(root) check: not db.containsBlock(root) - not db.containsBlockPhase0(root) - not db.containsBlockAltair(root) - not db.containsBlockMerge(root) - db.getPhase0Block(root).isErr() - not db.getPhase0BlockSSZ(root, tmp) + not db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + not db.containsBlock(root, altair.TrustedSignedBeaconBlock) + not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, phase0.TrustedSignedBeaconBlock).isErr() + not db.getBlockSSZ(root, tmp, phase0.TrustedSignedBeaconBlock) db.putStateRoot(root, signedBlock.message.slot, root) var root2 = root @@ -142,21 +142,21 @@ suite "Beacon chain DB" & preset(): var tmp: seq[byte] check: db.containsBlock(root) - not db.containsBlockPhase0(root) - db.containsBlockAltair(root) - not db.containsBlockMerge(root) - db.getAltairBlock(root).get() == signedBlock - db.getAltairBlockSSZ(root, tmp) + not db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + db.containsBlock(root, altair.TrustedSignedBeaconBlock) + not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, altair.TrustedSignedBeaconBlock).get() == signedBlock + db.getBlockSSZ(root, tmp, altair.TrustedSignedBeaconBlock) tmp == SSZ.encode(signedBlock) db.delBlock(root) check: not db.containsBlock(root) - not db.containsBlockPhase0(root) - not db.containsBlockAltair(root) - not db.containsBlockMerge(root) - db.getAltairBlock(root).isErr() - not db.getAltairBlockSSZ(root, tmp) + not db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + not db.containsBlock(root, altair.TrustedSignedBeaconBlock) + not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, altair.TrustedSignedBeaconBlock).isErr() + not db.getBlockSSZ(root, tmp, altair.TrustedSignedBeaconBlock) db.putStateRoot(root, signedBlock.message.slot, root) var root2 = root @@ -181,21 +181,21 @@ suite "Beacon chain DB" & preset(): var tmp: seq[byte] check: db.containsBlock(root) - not db.containsBlockPhase0(root) - not db.containsBlockAltair(root) - db.containsBlockMerge(root) - db.getMergeBlock(root).get() == signedBlock - db.getMergeBlockSSZ(root, tmp) + not db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + not db.containsBlock(root, altair.TrustedSignedBeaconBlock) + db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, bellatrix.TrustedSignedBeaconBlock).get() == signedBlock + db.getBlockSSZ(root, tmp, bellatrix.TrustedSignedBeaconBlock) tmp == SSZ.encode(signedBlock) db.delBlock(root) check: not db.containsBlock(root) - not db.containsBlockPhase0(root) - not db.containsBlockAltair(root) - not db.containsBlockMerge(root) - db.getMergeBlock(root).isErr() - not db.getMergeBlockSSZ(root, tmp) + not db.containsBlock(root, phase0.TrustedSignedBeaconBlock) + not db.containsBlock(root, altair.TrustedSignedBeaconBlock) + not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock) + db.getBlock(root, bellatrix.TrustedSignedBeaconBlock).isErr() + not db.getBlockSSZ(root, tmp, bellatrix.TrustedSignedBeaconBlock) db.putStateRoot(root, signedBlock.message.slot, root) var root2 = root @@ -409,35 +409,23 @@ suite "Beacon chain DB" & preset(): a2 = withDigest( (phase0.TrustedBeaconBlock)(slot: GENESIS_SLOT + 2, parent_root: a1.root)) - doAssert toSeq(db.getAncestors(a0.root)) == [] - doAssert toSeq(db.getAncestors(a2.root)) == [] - doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 0 doAssert db.getBeaconBlockSummary(a2.root).isNone() db.putBlock(a2) - doAssert toSeq(db.getAncestors(a0.root)) == [] - doAssert toSeq(db.getAncestors(a2.root)) == [a2] - doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 1 doAssert db.getBeaconBlockSummary(a2.root).get().slot == a2.message.slot db.putBlock(a1) - doAssert toSeq(db.getAncestors(a0.root)) == [] - doAssert toSeq(db.getAncestors(a2.root)) == [a2, a1] - doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 2 db.putBlock(a0) - doAssert toSeq(db.getAncestors(a0.root)) == [a0] - doAssert toSeq(db.getAncestors(a2.root)) == [a2, a1, a0] - doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 1 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 3