status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

db: make block loading generic (#3413) 

Streamline lookup with Forky and BeaconBlockFork (then we can do the
same for era)

We use type to avoid conditionals, as fork is often already known at a
"higher" level.

* load blockid before loading block by root - this is needed to map root
to slot and will eventually be done via block summary table for "old"
blocks

Co-authored-by: tersec <tersec@users.noreply.github.com>
This commit is contained in:
Jacek Sieka 2022-02-21 09:48:02 +01:00 committed by GitHub
parent 84588b34da
commit adfe655b16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 182 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
beacon_chain/beacon_chain_db.nim
View File

@ -104,13 +104,12 @@ type
checkpoint*: proc() {.gcsafe, raises: [Defect].} checkpoint*: proc() {.gcsafe, raises: [Defect].}
keyValues: KvStoreRef # Random stuff using DbKeyKind - suitable for small values mainly! keyValues: KvStoreRef # Random stuff using DbKeyKind - suitable for small values mainly!
blocks: KvStoreRef # BlockRoot -> phase0.TrustedBeaconBlock blocks: array[BeaconBlockFork, KvStoreRef] # BlockRoot -> TrustedSignedBeaconBlock
altairBlocks: KvStoreRef # BlockRoot -> altair.TrustedBeaconBlock
mergeBlocks: KvStoreRef # BlockRoot -> bellatrix.TrustedBeaconBlock
stateRoots: KvStoreRef # (Slot, BlockRoot) -> StateRoot stateRoots: KvStoreRef # (Slot, BlockRoot) -> StateRoot
statesNoVal: KvStoreRef # StateRoot -> Phase0BeaconStateNoImmutableValidators
altairStatesNoVal: KvStoreRef # StateRoot -> AltairBeaconStateNoImmutableValidators statesNoVal: array[BeaconStateFork, KvStoreRef] # StateRoot -> ForkBeaconStateNoImmutableValidators
mergeStatesNoVal: KvStoreRef # StateRoot -> MergeBeaconStateNoImmutableValidators
stateDiffs: KvStoreRef ##\ stateDiffs: KvStoreRef ##\
## StateRoot -> BeaconStateDiff ## StateRoot -> BeaconStateDiff
## Instead of storing full BeaconStates, one can store only the diff from ## Instead of storing full BeaconStates, one can store only the diff from
@ -418,13 +417,18 @@ proc new*(T: type BeaconChainDB,
# V1 - expected-to-be small rows get without rowid optimizations # V1 - expected-to-be small rows get without rowid optimizations
keyValues = kvStore db.openKvStore("key_values", true).expectDb() keyValues = kvStore db.openKvStore("key_values", true).expectDb()
blocks = kvStore db.openKvStore("blocks").expectDb() blocks = [
altairBlocks = kvStore db.openKvStore("altair_blocks").expectDb() kvStore db.openKvStore("blocks").expectDb(),
mergeBlocks = kvStore db.openKvStore("merge_blocks").expectDb() kvStore db.openKvStore("altair_blocks").expectDb(),
kvStore db.openKvStore("merge_blocks").expectDb()]
stateRoots = kvStore db.openKvStore("state_roots", true).expectDb() stateRoots = kvStore db.openKvStore("state_roots", true).expectDb()
statesNoVal = kvStore db.openKvStore("state_no_validators2").expectDb()
altairStatesNoVal = kvStore db.openKvStore("altair_state_no_validators").expectDb() statesNoVal = [
mergeStatesNoVal = kvStore db.openKvStore("merge_state_no_validators").expectDb() kvStore db.openKvStore("state_no_validators2").expectDb(),
kvStore db.openKvStore("altair_state_no_validators").expectDb(),
kvStore db.openKvStore("merge_state_no_validators").expectDb()]
stateDiffs = kvStore db.openKvStore("state_diffs").expectDb() stateDiffs = kvStore db.openKvStore("state_diffs").expectDb()
summaries = kvStore db.openKvStore("beacon_block_summaries", true).expectDb() summaries = kvStore db.openKvStore("beacon_block_summaries", true).expectDb()
finalizedBlocks = FinalizedBlocks.init(db, "finalized_blocks").expectDb() finalizedBlocks = FinalizedBlocks.init(db, "finalized_blocks").expectDb()
@ -465,12 +469,8 @@ proc new*(T: type BeaconChainDB,
checkpoint: proc() = db.checkpoint(), checkpoint: proc() = db.checkpoint(),
keyValues: keyValues, keyValues: keyValues,
blocks: blocks, blocks: blocks,
altair_blocks: altair_blocks,
merge_blocks: merge_blocks,
stateRoots: stateRoots, stateRoots: stateRoots,
statesNoVal: statesNoVal, statesNoVal: statesNoVal,
altairStatesNoVal: altairStatesNoVal,
mergeStatesNoVal: mergeStatesNoVal,
stateDiffs: stateDiffs, stateDiffs: stateDiffs,
summaries: summaries, summaries: summaries,
finalizedBlocks: finalizedBlocks, finalizedBlocks: finalizedBlocks,
@ -579,17 +579,13 @@ proc close*(db: BeaconChainDBV0) =
proc close*(db: BeaconChainDB) = proc close*(db: BeaconChainDB) =
if db.db == nil: return if db.db == nil: return
# Close things in reverse order # Close things roughly in reverse order
db.finalizedBlocks.close() db.finalizedBlocks.close()
discard db.summaries.close() discard db.summaries.close()
discard db.stateDiffs.close() discard db.stateDiffs.close()
discard db.mergeStatesNoVal.close() for kv in db.statesNoVal: discard kv.close()
discard db.altairStatesNoVal.close()
discard db.statesNoVal.close()
discard db.stateRoots.close() discard db.stateRoots.close()
discard db.mergeBlocks.close() for kv in db.blocks: discard kv.close()
discard db.altairBlocks.close()
discard db.blocks.close()
discard db.keyValues.close() discard db.keyValues.close()
db.immutableValidatorsDb.close() db.immutableValidatorsDb.close()
@ -610,19 +606,9 @@ proc putBeaconBlockSummary(
# Summaries are too simple / small to compress, store them as plain SSZ # Summaries are too simple / small to compress, store them as plain SSZ
db.summaries.putSSZ(root.data, value) db.summaries.putSSZ(root.data, value)
proc putBlock*(db: BeaconChainDB, value: phase0.TrustedSignedBeaconBlock) = proc putBlock*(db: BeaconChainDB, value: ForkyTrustedSignedBeaconBlock) =
db.withManyWrites: db.withManyWrites:
db.blocks.putSnappySSZ(value.root.data, value) db.blocks[type(value).toFork].putSnappySSZ(value.root.data, value)
db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary())
proc putBlock*(db: BeaconChainDB, value: altair.TrustedSignedBeaconBlock) =
db.withManyWrites:
db.altairBlocks.putSnappySSZ(value.root.data, value)
db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary())
proc putBlock*(db: BeaconChainDB, value: bellatrix.TrustedSignedBeaconBlock) =
db.withManyWrites:
db.mergeBlocks.putSnappySSZ(value.root.data, value)
db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary()) db.putBeaconBlockSummary(value.root, value.message.toBeaconBlockSummary())
proc updateImmutableValidators*( proc updateImmutableValidators*(
@ -651,19 +637,9 @@ template toBeaconStateNoImmutableValidators(state: bellatrix.BeaconState):
MergeBeaconStateNoImmutableValidators = MergeBeaconStateNoImmutableValidators =
isomorphicCast[MergeBeaconStateNoImmutableValidators](state) isomorphicCast[MergeBeaconStateNoImmutableValidators](state)
proc putState*(db: BeaconChainDB, key: Eth2Digest, value: phase0.BeaconState) = proc putState*(db: BeaconChainDB, key: Eth2Digest, value: ForkyBeaconState) =
db.updateImmutableValidators(value.validators.asSeq()) db.updateImmutableValidators(value.validators.asSeq())
db.statesNoVal.putSnappySSZ( db.statesNoVal[type(value).toFork()].putSnappySSZ(
key.data, toBeaconStateNoImmutableValidators(value))
proc putState*(db: BeaconChainDB, key: Eth2Digest, value: altair.BeaconState) =
db.updateImmutableValidators(value.validators.asSeq())
db.altairStatesNoVal.putSnappySSZ(
key.data, toBeaconStateNoImmutableValidators(value))
proc putState*(db: BeaconChainDB, key: Eth2Digest, value: bellatrix.BeaconState) =
db.updateImmutableValidators(value.validators.asSeq())
db.mergeStatesNoVal.putSnappySSZ(
key.data, toBeaconStateNoImmutableValidators(value)) key.data, toBeaconStateNoImmutableValidators(value))
proc putState*(db: BeaconChainDB, state: ForkyHashedBeaconState) = proc putState*(db: BeaconChainDB, state: ForkyHashedBeaconState) =
@ -673,13 +649,13 @@ proc putState*(db: BeaconChainDB, state: ForkyHashedBeaconState) =
# For testing rollback # For testing rollback
proc putCorruptPhase0State*(db: BeaconChainDB, key: Eth2Digest) = proc putCorruptPhase0State*(db: BeaconChainDB, key: Eth2Digest) =
db.statesNoVal.putSnappySSZ(key.data, Validator()) db.statesNoVal[BeaconStateFork.Phase0].putSnappySSZ(key.data, Validator())
proc putCorruptAltairState*(db: BeaconChainDB, key: Eth2Digest) = proc putCorruptAltairState*(db: BeaconChainDB, key: Eth2Digest) =
db.altairStatesNoVal.putSnappySSZ(key.data, Validator()) db.statesNoVal[BeaconStateFork.Altair].putSnappySSZ(key.data, Validator())
proc putCorruptMergeState*(db: BeaconChainDB, key: Eth2Digest) = proc putCorruptMergeState*(db: BeaconChainDB, key: Eth2Digest) =
db.mergeStatesNoVal.putSnappySSZ(key.data, Validator()) db.statesNoVal[BeaconStateFork.Bellatrix].putSnappySSZ(key.data, Validator())
func stateRootKey(root: Eth2Digest, slot: Slot): array[40, byte] = func stateRootKey(root: Eth2Digest, slot: Slot): array[40, byte] =
var ret: array[40, byte] var ret: array[40, byte]
@ -698,16 +674,12 @@ proc putStateDiff*(db: BeaconChainDB, root: Eth2Digest, value: BeaconStateDiff)
proc delBlock*(db: BeaconChainDB, key: Eth2Digest) = proc delBlock*(db: BeaconChainDB, key: Eth2Digest) =
db.withManyWrites: db.withManyWrites:
db.blocks.del(key.data).expectDb() for kv in db.blocks: kv.del(key.data).expectDb()
db.altairBlocks.del(key.data).expectDb()
db.mergeBlocks.del(key.data).expectDb()
db.summaries.del(key.data).expectDb() db.summaries.del(key.data).expectDb()
proc delState*(db: BeaconChainDB, key: Eth2Digest) = proc delState*(db: BeaconChainDB, key: Eth2Digest) =
db.withManyWrites: db.withManyWrites:
db.statesNoVal.del(key.data).expectDb() for kv in db.statesNoVal: kv.del(key.data).expectDb()
db.altairStatesNoVal.del(key.data).expectDb()
db.mergeStatesNoVal.del(key.data).expectDb()
proc delStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot) = proc delStateRoot*(db: BeaconChainDB, root: Eth2Digest, slot: Slot) =
db.stateRoots.del(stateRootKey(root, slot)).expectDb() db.stateRoots.del(stateRootKey(root, slot)).expectDb()
@ -728,7 +700,8 @@ proc putEth2FinalizedTo*(db: BeaconChainDB,
eth1Checkpoint: DepositContractSnapshot) = eth1Checkpoint: DepositContractSnapshot) =
db.keyValues.putSnappySSZ(subkey(kDepositsFinalizedByEth2), eth1Checkpoint) db.keyValues.putSnappySSZ(subkey(kDepositsFinalizedByEth2), eth1Checkpoint)
proc getPhase0Block(db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSignedBeaconBlock] = proc getPhase0Block(
db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSignedBeaconBlock] =
# We only store blocks that we trust in the database # We only store blocks that we trust in the database
result.ok(default(phase0.TrustedSignedBeaconBlock)) result.ok(default(phase0.TrustedSignedBeaconBlock))
if db.backend.getSnappySSZ( if db.backend.getSnappySSZ(
@ -738,31 +711,25 @@ proc getPhase0Block(db: BeaconChainDBV0, key: Eth2Digest): Opt[phase0.TrustedSig
# set root after deserializing (so it doesn't get zeroed) # set root after deserializing (so it doesn't get zeroed)
result.get().root = key result.get().root = key
proc getPhase0Block*(db: BeaconChainDB, key: Eth2Digest): proc getBlock*(
Opt[phase0.TrustedSignedBeaconBlock] = db: BeaconChainDB, key: Eth2Digest,
T: type phase0.TrustedSignedBeaconBlock): Opt[T] =
# We only store blocks that we trust in the database # We only store blocks that we trust in the database
result.ok(default(phase0.TrustedSignedBeaconBlock)) result.ok(default(T))
if db.blocks.getSnappySSZ(key.data, result.get) != GetResult.found: if db.blocks[T.toFork].getSnappySSZ(key.data, result.get) != GetResult.found:
# During the initial releases phase0, we stored blocks in a different table
result = db.v0.getPhase0Block(key) result = db.v0.getPhase0Block(key)
else: else:
# set root after deserializing (so it doesn't get zeroed) # set root after deserializing (so it doesn't get zeroed)
result.get().root = key result.get().root = key
proc getAltairBlock*(db: BeaconChainDB, key: Eth2Digest): proc getBlock*[
Opt[altair.TrustedSignedBeaconBlock] = X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock](
db: BeaconChainDB, key: Eth2Digest,
T: type X): Opt[T] =
# We only store blocks that we trust in the database # We only store blocks that we trust in the database
result.ok(default(altair.TrustedSignedBeaconBlock)) result.ok(default(T))
if db.altairBlocks.getSnappySSZ(key.data, result.get) == GetResult.found: if db.blocks[T.toFork].getSnappySSZ(key.data, result.get) == GetResult.found:
# set root after deserializing (so it doesn't get zeroed)
result.get().root = key
else:
result.err()
proc getMergeBlock*(db: BeaconChainDB, key: Eth2Digest):
Opt[bellatrix.TrustedSignedBeaconBlock] =
# We only store blocks that we trust in the database
result.ok(default(bellatrix.TrustedSignedBeaconBlock))
if db.mergeBlocks.getSnappySSZ(key.data, result.get) == GetResult.found:
# set root after deserializing (so it doesn't get zeroed) # set root after deserializing (so it doesn't get zeroed)
result.get().root = key result.get().root = key
else: else:
@ -776,31 +743,39 @@ proc getPhase0BlockSSZ(db: BeaconChainDBV0, key: Eth2Digest, data: var seq[byte]
except CatchableError: success = false except CatchableError: success = false
db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and success db.backend.get(subkey(phase0.SignedBeaconBlock, key), decode).expectDb() and success
proc getPhase0BlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = # SSZ implementations are separate so as to avoid unnecessary data copies
proc getBlockSSZ*(
db: BeaconChainDB, key: Eth2Digest, data: var seq[byte],
T: type phase0.TrustedSignedBeaconBlock): bool =
let dataPtr = unsafeAddr data # Short-lived let dataPtr = unsafeAddr data # Short-lived
var success = true var success = true
proc decode(data: openArray[byte]) = proc decode(data: openArray[byte]) =
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize)
except CatchableError: success = false except CatchableError: success = false
db.blocks.get(key.data, decode).expectDb() and success or db.blocks[BeaconBlockFork.Phase0].get(key.data, decode).expectDb() and success or
db.v0.getPhase0BlockSSZ(key, data) db.v0.getPhase0BlockSSZ(key, data)
proc getAltairBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = proc getBlockSSZ*[
X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock](
db: BeaconChainDB, key: Eth2Digest, data: var seq[byte],
T: type X): bool =
let dataPtr = unsafeAddr data # Short-lived let dataPtr = unsafeAddr data # Short-lived
var success = true var success = true
proc decode(data: openArray[byte]) = proc decode(data: openArray[byte]) =
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize)
except CatchableError: success = false except CatchableError: success = false
db.altairBlocks.get(key.data, decode).expectDb() and success db.blocks[T.toFork].get(key.data, decode).expectDb() and success
proc getMergeBlockSSZ*(db: BeaconChainDB, key: Eth2Digest, data: var seq[byte]): bool = proc getBlockSSZ*(
let dataPtr = unsafeAddr data # Short-lived db: BeaconChainDB, key: Eth2Digest, data: var seq[byte],
var success = true fork: BeaconBlockFork): bool =
proc decode(data: openArray[byte]) = case fork
try: dataPtr[] = snappy.decode(data, maxDecompressedDbRecordSize) of BeaconBlockFork.Phase0:
except CatchableError: success = false getBlockSSZ(db, key, data, phase0.TrustedSignedBeaconBlock)
of BeaconBlockFork.Altair:
db.mergeBlocks.get(key.data, decode).expectDb() and success getBlockSSZ(db, key, data, altair.TrustedSignedBeaconBlock)
of BeaconBlockFork.Bellatrix:
getBlockSSZ(db, key, data, bellatrix.TrustedSignedBeaconBlock)
proc getStateOnlyMutableValidators( proc getStateOnlyMutableValidators(
immutableValidators: openArray[ImmutableValidatorData2], immutableValidators: openArray[ImmutableValidatorData2],
@ -885,14 +860,17 @@ proc getState*(
# https://github.com/nim-lang/Nim/issues/14126 # https://github.com/nim-lang/Nim/issues/14126
# TODO RVO is inefficient for large objects: # TODO RVO is inefficient for large objects:
# https://github.com/nim-lang/Nim/issues/13879 # https://github.com/nim-lang/Nim/issues/13879
type T = type(output)
if not getStateOnlyMutableValidators( if not getStateOnlyMutableValidators(
db.immutableValidators, db.statesNoVal, key.data, output, rollback): db.immutableValidators, db.statesNoVal[T.toFork], key.data, output, rollback):
db.v0.getState(db.immutableValidators, key, output, rollback) db.v0.getState(db.immutableValidators, key, output, rollback)
else: else:
true true
proc getState*( proc getState*(
db: BeaconChainDB, key: Eth2Digest, output: var altair.BeaconState, db: BeaconChainDB, key: Eth2Digest,
output: var (altair.BeaconState | bellatrix.BeaconState),
rollback: RollbackProc): bool = rollback: RollbackProc): bool =
## Load state into `output` - BeaconState is large so we want to avoid ## Load state into `output` - BeaconState is large so we want to avoid
## re-allocating it if possible ## re-allocating it if possible
@ -904,24 +882,10 @@ proc getState*(
# https://github.com/nim-lang/Nim/issues/14126 # https://github.com/nim-lang/Nim/issues/14126
# TODO RVO is inefficient for large objects: # TODO RVO is inefficient for large objects:
# https://github.com/nim-lang/Nim/issues/13879 # https://github.com/nim-lang/Nim/issues/13879
type T = type(output)
getStateOnlyMutableValidators( getStateOnlyMutableValidators(
db.immutableValidators, db.altairStatesNoVal, key.data, output, rollback) db.immutableValidators, db.statesNoVal[T.toFork], key.data, output,
rollback)
proc getState*(
db: BeaconChainDB, key: Eth2Digest, output: var bellatrix.BeaconState,
rollback: RollbackProc): bool =
## Load state into `output` - BeaconState is large so we want to avoid
## re-allocating it if possible
## Return `true` iff the entry was found in the database and `output` was
## overwritten.
## Rollback will be called only if output was partially written - if it was
## not found at all, rollback will not be called
# TODO rollback is needed to deal with bug - use `noRollback` to ignore:
# https://github.com/nim-lang/Nim/issues/14126
# TODO RVO is inefficient for large objects:
# https://github.com/nim-lang/Nim/issues/13879
getStateOnlyMutableValidators(
db.immutableValidators, db.mergeStatesNoVal, key.data, output, rollback)
proc getStateRoot(db: BeaconChainDBV0, proc getStateRoot(db: BeaconChainDBV0,
root: Eth2Digest, root: Eth2Digest,
@ -974,19 +938,26 @@ proc getEth2FinalizedTo*(db: BeaconChainDB): Opt[DepositContractSnapshot] =
proc containsBlock*(db: BeaconChainDBV0, key: Eth2Digest): bool = proc containsBlock*(db: BeaconChainDBV0, key: Eth2Digest): bool =
db.backend.contains(subkey(phase0.SignedBeaconBlock, key)).expectDb() db.backend.contains(subkey(phase0.SignedBeaconBlock, key)).expectDb()
proc containsBlockPhase0*(db: BeaconChainDB, key: Eth2Digest): bool = proc containsBlock*(
db.blocks.contains(key.data).expectDb() or db: BeaconChainDB, key: Eth2Digest,
T: type phase0.TrustedSignedBeaconBlock): bool =
db.blocks[T.toFork].contains(key.data).expectDb() or
db.v0.containsBlock(key) db.v0.containsBlock(key)
proc containsBlockAltair*(db: BeaconChainDB, key: Eth2Digest): bool = proc containsBlock*[
db.altairBlocks.contains(key.data).expectDb() X: altair.TrustedSignedBeaconBlock | bellatrix.TrustedSignedBeaconBlock](
db: BeaconChainDB, key: Eth2Digest, T: type X): bool =
db.blocks[X.toFork].contains(key.data).expectDb()
proc containsBlockMerge*(db: BeaconChainDB, key: Eth2Digest): bool = proc containsBlock*(db: BeaconChainDB, key: Eth2Digest, fork: BeaconBlockFork): bool =
db.mergeBlocks.contains(key.data).expectDb() case fork
of BeaconBlockFork.Phase0: containsBlock(db, key, phase0.TrustedSignedBeaconBlock)
else: db.blocks[fork].contains(key.data).expectDb()
proc containsBlock*(db: BeaconChainDB, key: Eth2Digest): bool = proc containsBlock*(db: BeaconChainDB, key: Eth2Digest): bool =
db.containsBlockMerge(key) or db.containsBlockAltair(key) or db.containsBlock(key, bellatrix.TrustedSignedBeaconBlock) or
db.containsBlockPhase0(key) db.containsBlock(key, altair.TrustedSignedBeaconBlock) or
db.containsBlock(key, phase0.TrustedSignedBeaconBlock)
proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool = proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool =
let sk = subkey(Phase0BeaconStateNoImmutableValidators, key) let sk = subkey(Phase0BeaconStateNoImmutableValidators, key)
@ -995,28 +966,11 @@ proc containsState*(db: BeaconChainDBV0, key: Eth2Digest): bool =
db.backend.contains(subkey(phase0.BeaconState, key)).expectDb() db.backend.contains(subkey(phase0.BeaconState, key)).expectDb()
proc containsState*(db: BeaconChainDB, key: Eth2Digest, legacy: bool = true): bool = proc containsState*(db: BeaconChainDB, key: Eth2Digest, legacy: bool = true): bool =
db.mergeStatesNoVal.contains(key.data).expectDb or db.statesNoVal[BeaconStateFork.Bellatrix].contains(key.data).expectDb or
db.altairStatesNoVal.contains(key.data).expectDb or db.statesNoVal[BeaconStateFork.Altair].contains(key.data).expectDb or
db.statesNoVal.contains(key.data).expectDb or db.statesNoVal[BeaconStateFork.Phase0].contains(key.data).expectDb or
(legacy and db.v0.containsState(key)) (legacy and db.v0.containsState(key))
iterator getAncestors*(db: BeaconChainDB, root: Eth2Digest):
phase0.TrustedSignedBeaconBlock =
## Load a chain of ancestors for blck - returns a list of blocks with the
## oldest block last (blck will be at result[0]).
##
## The search will go on until the ancestor cannot be found.
var
res: phase0.TrustedSignedBeaconBlock
root = root
while db.blocks.getSnappySSZ(root.data, res) == GetResult.found or
db.v0.backend.getSnappySSZ(
subkey(phase0.SignedBeaconBlock, root), res) == GetResult.found:
res.root = root
yield res
root = res.message.parent_root
proc getBeaconBlockSummary*(db: BeaconChainDB, root: Eth2Digest): proc getBeaconBlockSummary*(db: BeaconChainDB, root: Eth2Digest):
Opt[BeaconBlockSummary] = Opt[BeaconBlockSummary] =
var summary: BeaconBlockSummary var summary: BeaconBlockSummary
@ -1107,11 +1061,11 @@ iterator getAncestorSummaries*(db: BeaconChainDB, root: Eth2Digest):
if db.v0.backend.getSnappySSZ( if db.v0.backend.getSnappySSZ(
subkey(BeaconBlockSummary, res.root), res.summary) == GetResult.found: subkey(BeaconBlockSummary, res.root), res.summary) == GetResult.found:
discard # Just yield below discard # Just yield below
elif (let blck = db.getPhase0Block(res.root); blck.isSome()): elif (let blck = db.getBlock(res.root, phase0.TrustedSignedBeaconBlock); blck.isSome()):
res.summary = blck.get().message.toBeaconBlockSummary() res.summary = blck.get().message.toBeaconBlockSummary()
elif (let blck = db.getAltairBlock(res.root); blck.isSome()): elif (let blck = db.getBlock(res.root, altair.TrustedSignedBeaconBlock); blck.isSome()):
res.summary = blck.get().message.toBeaconBlockSummary() res.summary = blck.get().message.toBeaconBlockSummary()
elif (let blck = db.getMergeBlock(res.root); blck.isSome()): elif (let blck = db.getBlock(res.root, bellatrix.TrustedSignedBeaconBlock); blck.isSome()):
res.summary = blck.get().message.toBeaconBlockSummary() res.summary = blck.get().message.toBeaconBlockSummary()
else: else:
break break

86
beacon_chain/consensus_object_pools/blockchain_dag.nim
View File

@ -13,8 +13,8 @@ import
metrics, snappy, chronicles, metrics, snappy, chronicles,
../spec/[beaconstate, eth2_merkleization, eth2_ssz_serialization, helpers, ../spec/[beaconstate, eth2_merkleization, eth2_ssz_serialization, helpers,
state_transition, validator], state_transition, validator],
../spec/datatypes/[phase0, altair],
".."/beacon_chain_db, ".."/beacon_chain_db,
../spec/datatypes/[phase0, altair, bellatrix],
"."/[block_pools_types, block_quarantine] "."/[block_pools_types, block_quarantine]
export export
@ -344,10 +344,7 @@ func containsForkBlock*(dag: ChainDAGRef, root: Eth2Digest): bool =
proc containsBlock( proc containsBlock(
cfg: RuntimeConfig, db: BeaconChainDB, slot: Slot, root: Eth2Digest): bool = cfg: RuntimeConfig, db: BeaconChainDB, slot: Slot, root: Eth2Digest): bool =
case cfg.blockForkAtEpoch(slot.epoch) db.containsBlock(root, cfg.blockForkAtEpoch(slot.epoch))
of BeaconBlockFork.Phase0: db.containsBlockPhase0(root)
of BeaconBlockFork.Altair: db.containsBlockAltair(root)
of BeaconBlockFork.Bellatrix: db.containsBlockMerge(root)
func isStateCheckpoint(bs: BlockSlot): bool = func isStateCheckpoint(bs: BlockSlot): bool =
## State checkpoints are the points in time for which we store full state ## State checkpoints are the points in time for which we store full state
@ -398,40 +395,57 @@ proc getForkedBlock*(db: BeaconChainDB, root: Eth2Digest):
Opt[ForkedTrustedSignedBeaconBlock] = Opt[ForkedTrustedSignedBeaconBlock] =
# When we only have a digest, we don't know which fork it's from so we try # When we only have a digest, we don't know which fork it's from so we try
# them one by one - this should be used sparingly # them one by one - this should be used sparingly
if (let blck = db.getMergeBlock(root); blck.isSome()): if (let blck = db.getBlock(root, bellatrix.TrustedSignedBeaconBlock);
blck.isSome()):
ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) ok(ForkedTrustedSignedBeaconBlock.init(blck.get()))
elif (let blck = db.getAltairBlock(root); blck.isSome()): elif (let blck = db.getBlock(root, altair.TrustedSignedBeaconBlock);
blck.isSome()):
ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) ok(ForkedTrustedSignedBeaconBlock.init(blck.get()))
elif (let blck = db.getPhase0Block(root); blck.isSome()): elif (let blck = db.getBlock(root, phase0.TrustedSignedBeaconBlock);
blck.isSome()):
ok(ForkedTrustedSignedBeaconBlock.init(blck.get())) ok(ForkedTrustedSignedBeaconBlock.init(blck.get()))
else: else:
err() err()
proc getForkedBlock*( proc getBlock*(
dag: ChainDAGRef, root: Eth2Digest): Opt[ForkedTrustedSignedBeaconBlock] = dag: ChainDAGRef, bid: BlockId,
dag.db.getForkedBlock(root) T: type ForkyTrustedSignedBeaconBlock): Opt[T] =
withState(dag.headState.data):
dag.db.getBlock(bid.root, T)
proc getBlockSSZ*(dag: ChainDAGRef, bid: BlockId, bytes: var seq[byte]): bool =
# Load the SSZ-encoded data of a block into `bytes`, overwriting the existing
# content
# careful: there are two snappy encodings in use, with and without framing!
# Returns true if the block is found, false if not
let fork = dag.cfg.blockForkAtEpoch(bid.slot.epoch)
dag.db.getBlockSSZ(bid.root, bytes, fork)
proc getForkedBlock*( proc getForkedBlock*(
dag: ChainDAGRef, id: BlockId): Opt[ForkedTrustedSignedBeaconBlock] = dag: ChainDAGRef, bid: BlockId): Opt[ForkedTrustedSignedBeaconBlock] =
case dag.cfg.blockForkAtEpoch(id.slot.epoch)
of BeaconBlockFork.Phase0: let fork = dag.cfg.blockForkAtEpoch(bid.slot.epoch)
let data = dag.db.getPhase0Block(id.root) result.ok(ForkedTrustedSignedBeaconBlock(kind: fork))
if data.isOk(): withBlck(result.get()):
return ok ForkedTrustedSignedBeaconBlock.init(data.get) type T = type(blck)
of BeaconBlockFork.Altair: blck = getBlock(dag, bid, T).valueOr:
let data = dag.db.getAltairBlock(id.root) result.err()
if data.isOk(): return
return ok ForkedTrustedSignedBeaconBlock.init(data.get)
of BeaconBlockFork.Bellatrix:
let data = dag.db.getMergeBlock(id.root)
if data.isOk():
return ok ForkedTrustedSignedBeaconBlock.init(data.get)
proc getForkedBlock*( proc getForkedBlock*(
dag: ChainDAGRef, blck: BlockRef): ForkedTrustedSignedBeaconBlock = dag: ChainDAGRef, blck: BlockRef): ForkedTrustedSignedBeaconBlock =
dag.getForkedBlock(blck.bid).expect( dag.getForkedBlock(blck.bid).expect(
"BlockRef block should always load, database corrupt?") "BlockRef block should always load, database corrupt?")
proc getForkedBlock*(
dag: ChainDAGRef, root: Eth2Digest): Opt[ForkedTrustedSignedBeaconBlock] =
let bid = dag.getBlockId(root)
if bid.isSome():
dag.getForkedBlock(bid.get())
else:
# In case we didn't have a summary - should be rare, but ..
dag.db.getForkedBlock(root)
proc updateBeaconMetrics(state: StateData, cache: var StateCache) = proc updateBeaconMetrics(state: StateData, cache: var StateCache) =
# https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics
# both non-negative, so difference can't overflow or underflow int64 # both non-negative, so difference can't overflow or underflow int64
@ -990,19 +1004,22 @@ proc applyBlock(
case dag.cfg.blockForkAtEpoch(blck.slot.epoch) case dag.cfg.blockForkAtEpoch(blck.slot.epoch)
of BeaconBlockFork.Phase0: of BeaconBlockFork.Phase0:
let data = dag.db.getPhase0Block(blck.root).expect("block loaded") let data = getBlock(dag, blck.bid, phase0.TrustedSignedBeaconBlock).expect(
"block loaded")
state_transition( state_transition(
dag.cfg, state.data, data, cache, info, dag.cfg, state.data, data, cache, info,
dag.updateFlags + {slotProcessed}, noRollback).expect( dag.updateFlags + {slotProcessed}, noRollback).expect(
"Blocks from database must not fail to apply") "Blocks from database must not fail to apply")
of BeaconBlockFork.Altair: of BeaconBlockFork.Altair:
let data = dag.db.getAltairBlock(blck.root).expect("block loaded") let data = getBlock(dag, blck.bid, altair.TrustedSignedBeaconBlock).expect(
"block loaded")
state_transition( state_transition(
dag.cfg, state.data, data, cache, info, dag.cfg, state.data, data, cache, info,
dag.updateFlags + {slotProcessed}, noRollback).expect( dag.updateFlags + {slotProcessed}, noRollback).expect(
"Blocks from database must not fail to apply") "Blocks from database must not fail to apply")
of BeaconBlockFork.Bellatrix: of BeaconBlockFork.Bellatrix:
let data = dag.db.getMergeBlock(blck.root).expect("block loaded") let data = getBlock(dag, blck.bid, bellatrix.TrustedSignedBeaconBlock).expect(
"block loaded")
state_transition( state_transition(
dag.cfg, state.data, data, cache, info, dag.cfg, state.data, data, cache, info,
dag.updateFlags + {slotProcessed}, noRollback).expect( dag.updateFlags + {slotProcessed}, noRollback).expect(
@ -1737,18 +1754,5 @@ proc aggregateAll*(
else: else:
ok(finish(aggregateKey)) ok(finish(aggregateKey))
proc getBlockSSZ*(dag: ChainDAGRef, id: BlockId, bytes: var seq[byte]): bool =
# Load the SSZ-encoded data of a block into `bytes`, overwriting the existing
# content
# careful: there are two snappy encodings in use, with and without framing!
# Returns true if the block is found, false if not
case dag.cfg.blockForkAtEpoch(id.slot.epoch)
of BeaconBlockFork.Phase0:
dag.db.getPhase0BlockSSZ(id.root, bytes)
of BeaconBlockFork.Altair:
dag.db.getAltairBlockSSZ(id.root, bytes)
of BeaconBlockFork.Bellatrix:
dag.db.getMergeBlockSSZ(id.root, bytes)
func needsBackfill*(dag: ChainDAGRef): bool = func needsBackfill*(dag: ChainDAGRef): bool =
dag.backfill.slot > dag.genesis.slot dag.backfill.slot > dag.genesis.slot

23
ncli/ncli_db.nim
View File

@ -232,11 +232,14 @@ proc cmdBench(conf: DbConf, cfg: RuntimeConfig) =
withTimer(timers[tLoadBlock]): withTimer(timers[tLoadBlock]):
case cfg.blockForkAtEpoch(blck.slot.epoch) case cfg.blockForkAtEpoch(blck.slot.epoch)
of BeaconBlockFork.Phase0: of BeaconBlockFork.Phase0:
blocks[0].add dag.db.getPhase0Block(blck.root).get() blocks[0].add dag.db.getBlock(
blck.root, phase0.TrustedSignedBeaconBlock).get()
of BeaconBlockFork.Altair: of BeaconBlockFork.Altair:
blocks[1].add dag.db.getAltairBlock(blck.root).get() blocks[1].add dag.db.getBlock(
blck.root, altair.TrustedSignedBeaconBlock).get()
of BeaconBlockFork.Bellatrix: of BeaconBlockFork.Bellatrix:
blocks[2].add dag.db.getMergeBlock(blck.root).get() blocks[2].add dag.db.getBlock(
blck.root, bellatrix.TrustedSignedBeaconBlock).get()
let stateData = newClone(dag.headState) let stateData = newClone(dag.headState)
@ -359,11 +362,13 @@ proc cmdDumpBlock(conf: DbConf) =
if shouldShutDown: quit QuitSuccess if shouldShutDown: quit QuitSuccess
try: try:
let root = Eth2Digest.fromHex(blockRoot) let root = Eth2Digest.fromHex(blockRoot)
if (let blck = db.getPhase0Block(root); blck.isSome): if (let blck = db.getBlock(
root, phase0.TrustedSignedBeaconBlock); blck.isSome):
dump("./", blck.get()) dump("./", blck.get())
elif (let blck = db.getAltairBlock(root); blck.isSome): elif (let blck = db.getBlock(
root, altair.TrustedSignedBeaconBlock); blck.isSome):
dump("./", blck.get()) dump("./", blck.get())
elif (let blck = db.getMergeBlock(root); blck.isSome): elif (let blck = db.getBlock(root, bellatrix.TrustedSignedBeaconBlock); blck.isSome):
dump("./", blck.get()) dump("./", blck.get())
else: else:
echo "Couldn't load ", blockRoot echo "Couldn't load ", blockRoot
@ -511,7 +516,7 @@ proc cmdImportEra(conf: DbConf, cfg: RuntimeConfig) =
let header = readRecord(f, data).valueOr: let header = readRecord(f, data).valueOr:
break break
if header.typ == SnappyBeaconBlock: if header.typ == SnappyBeaconBlock:
withTimer(timers[tBlock]): withTimer(timers[tBlock]):
let uncompressed = framingFormatUncompress(data) let uncompressed = framingFormatUncompress(data)
let blck = try: readSszForkedSignedBeaconBlock(cfg, uncompressed) let blck = try: readSszForkedSignedBeaconBlock(cfg, uncompressed)
@ -646,7 +651,9 @@ proc cmdValidatorPerf(conf: DbConf, cfg: RuntimeConfig) =
if shouldShutDown: quit QuitSuccess if shouldShutDown: quit QuitSuccess
for bi in 0 ..< blockRefs.len: for bi in 0 ..< blockRefs.len:
blck = db.getPhase0Block(blockRefs[blockRefs.len - bi - 1].root).get() blck = db.getBlock(
blockRefs[blockRefs.len - bi - 1].root,
phase0.TrustedSignedBeaconBlock).get()
while getStateField(state[].data, slot) < blck.message.slot: while getStateField(state[].data, slot) < blck.message.slot:
let let
nextSlot = getStateField(state[].data, slot) + 1 nextSlot = getStateField(state[].data, slot) + 1

74
tests/test_beacon_chain_db.nim
View File

@ -89,7 +89,7 @@ suite "Beacon chain DB" & preset():
db = BeaconChainDB.new("", inMemory = true) db = BeaconChainDB.new("", inMemory = true)
check: check:
db.getPhase0StateRef(Eth2Digest()).isNil db.getPhase0StateRef(Eth2Digest()).isNil
db.getPhase0Block(Eth2Digest()).isNone db.getBlock(Eth2Digest(), phase0.TrustedSignedBeaconBlock).isNone
test "sanity check phase 0 blocks" & preset(): test "sanity check phase 0 blocks" & preset():
let db = BeaconChainDB.new("", inMemory = true) let db = BeaconChainDB.new("", inMemory = true)
@ -103,21 +103,21 @@ suite "Beacon chain DB" & preset():
var tmp: seq[byte] var tmp: seq[byte]
check: check:
db.containsBlock(root) db.containsBlock(root)
db.containsBlockPhase0(root) db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
not db.containsBlockAltair(root) not db.containsBlock(root, altair.TrustedSignedBeaconBlock)
not db.containsBlockMerge(root) not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getPhase0Block(root).get() == signedBlock db.getBlock(root, phase0.TrustedSignedBeaconBlock).get() == signedBlock
db.getPhase0BlockSSZ(root, tmp) db.getBlockSSZ(root, tmp, phase0.TrustedSignedBeaconBlock)
tmp == SSZ.encode(signedBlock) tmp == SSZ.encode(signedBlock)
db.delBlock(root) db.delBlock(root)
check: check:
not db.containsBlock(root) not db.containsBlock(root)
not db.containsBlockPhase0(root) not db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
not db.containsBlockAltair(root) not db.containsBlock(root, altair.TrustedSignedBeaconBlock)
not db.containsBlockMerge(root) not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getPhase0Block(root).isErr() db.getBlock(root, phase0.TrustedSignedBeaconBlock).isErr()
not db.getPhase0BlockSSZ(root, tmp) not db.getBlockSSZ(root, tmp, phase0.TrustedSignedBeaconBlock)
db.putStateRoot(root, signedBlock.message.slot, root) db.putStateRoot(root, signedBlock.message.slot, root)
var root2 = root var root2 = root
@ -142,21 +142,21 @@ suite "Beacon chain DB" & preset():
var tmp: seq[byte] var tmp: seq[byte]
check: check:
db.containsBlock(root) db.containsBlock(root)
not db.containsBlockPhase0(root) not db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
db.containsBlockAltair(root) db.containsBlock(root, altair.TrustedSignedBeaconBlock)
not db.containsBlockMerge(root) not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getAltairBlock(root).get() == signedBlock db.getBlock(root, altair.TrustedSignedBeaconBlock).get() == signedBlock
db.getAltairBlockSSZ(root, tmp) db.getBlockSSZ(root, tmp, altair.TrustedSignedBeaconBlock)
tmp == SSZ.encode(signedBlock) tmp == SSZ.encode(signedBlock)
db.delBlock(root) db.delBlock(root)
check: check:
not db.containsBlock(root) not db.containsBlock(root)
not db.containsBlockPhase0(root) not db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
not db.containsBlockAltair(root) not db.containsBlock(root, altair.TrustedSignedBeaconBlock)
not db.containsBlockMerge(root) not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getAltairBlock(root).isErr() db.getBlock(root, altair.TrustedSignedBeaconBlock).isErr()
not db.getAltairBlockSSZ(root, tmp) not db.getBlockSSZ(root, tmp, altair.TrustedSignedBeaconBlock)
db.putStateRoot(root, signedBlock.message.slot, root) db.putStateRoot(root, signedBlock.message.slot, root)
var root2 = root var root2 = root
@ -181,21 +181,21 @@ suite "Beacon chain DB" & preset():
var tmp: seq[byte] var tmp: seq[byte]
check: check:
db.containsBlock(root) db.containsBlock(root)
not db.containsBlockPhase0(root) not db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
not db.containsBlockAltair(root) not db.containsBlock(root, altair.TrustedSignedBeaconBlock)
db.containsBlockMerge(root) db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getMergeBlock(root).get() == signedBlock db.getBlock(root, bellatrix.TrustedSignedBeaconBlock).get() == signedBlock
db.getMergeBlockSSZ(root, tmp) db.getBlockSSZ(root, tmp, bellatrix.TrustedSignedBeaconBlock)
tmp == SSZ.encode(signedBlock) tmp == SSZ.encode(signedBlock)
db.delBlock(root) db.delBlock(root)
check: check:
not db.containsBlock(root) not db.containsBlock(root)
not db.containsBlockPhase0(root) not db.containsBlock(root, phase0.TrustedSignedBeaconBlock)
not db.containsBlockAltair(root) not db.containsBlock(root, altair.TrustedSignedBeaconBlock)
not db.containsBlockMerge(root) not db.containsBlock(root, bellatrix.TrustedSignedBeaconBlock)
db.getMergeBlock(root).isErr() db.getBlock(root, bellatrix.TrustedSignedBeaconBlock).isErr()
not db.getMergeBlockSSZ(root, tmp) not db.getBlockSSZ(root, tmp, bellatrix.TrustedSignedBeaconBlock)
db.putStateRoot(root, signedBlock.message.slot, root) db.putStateRoot(root, signedBlock.message.slot, root)
var root2 = root var root2 = root
@ -409,35 +409,23 @@ suite "Beacon chain DB" & preset():
a2 = withDigest( a2 = withDigest(
(phase0.TrustedBeaconBlock)(slot: GENESIS_SLOT + 2, parent_root: a1.root)) (phase0.TrustedBeaconBlock)(slot: GENESIS_SLOT + 2, parent_root: a1.root))
doAssert toSeq(db.getAncestors(a0.root)) == []
doAssert toSeq(db.getAncestors(a2.root)) == []
doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0
doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 0
doAssert db.getBeaconBlockSummary(a2.root).isNone() doAssert db.getBeaconBlockSummary(a2.root).isNone()
db.putBlock(a2) db.putBlock(a2)
doAssert toSeq(db.getAncestors(a0.root)) == []
doAssert toSeq(db.getAncestors(a2.root)) == [a2]
doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0
doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 1 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 1
doAssert db.getBeaconBlockSummary(a2.root).get().slot == a2.message.slot doAssert db.getBeaconBlockSummary(a2.root).get().slot == a2.message.slot
db.putBlock(a1) db.putBlock(a1)
doAssert toSeq(db.getAncestors(a0.root)) == []
doAssert toSeq(db.getAncestors(a2.root)) == [a2, a1]
doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0 doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 0
doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 2 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 2
db.putBlock(a0) db.putBlock(a0)
doAssert toSeq(db.getAncestors(a0.root)) == [a0]
doAssert toSeq(db.getAncestors(a2.root)) == [a2, a1, a0]
doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 1 doAssert toSeq(db.getAncestorSummaries(a0.root)).len == 1
doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 3 doAssert toSeq(db.getAncestorSummaries(a2.root)).len == 3