From 9f27f0d97c2dfa1a8e8d81c08e659d5a0ba03217 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 9 Dec 2021 18:06:21 +0100 Subject: [PATCH] BlockId reform (#3176) * BlockId reform Introduce `BlockId` that helps track a root/slot pair - this prepares the codebase for backfilling and handling out-of-dag blocks * move block dag code to separate module * fix finalised state root in REST event stream * fix finalised head computation on head update, when starting from checkpoint * clean up chaindag init * revert `epochAncestor` change in introduced in #3144 that would return an epoch ancestor from the canoncial history instead of the given history, causing `EpochRef` keys to point to the wrong block --- AllTests-mainnet.md | 25 +- .../consensus_object_pools/block_dag.nim | 246 ++++++++++++++++++ .../block_pools_types.nim | 53 +--- .../consensus_object_pools/blockchain_dag.nim | 228 +++++----------- beacon_chain/rpc/rest_utils.nim | 9 +- beacon_chain/rpc/rpc_beacon_api.nim | 2 +- beacon_chain/rpc/rpc_utils.nim | 21 +- tests/all_tests.nim | 1 + tests/test_block_dag.nim | 121 +++++++++ tests/test_blockchain_dag.nim | 84 +----- 10 files changed, 491 insertions(+), 299 deletions(-) create mode 100644 beacon_chain/consensus_object_pools/block_dag.nim create mode 100644 tests/test_block_dag.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 1cdaf6ac7..c082e66e2 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -90,18 +90,29 @@ OK: 5/5 Fail: 0/5 Skip: 0/5 + Reverse order block add & get [Preset: mainnet] OK ``` OK: 1/1 Fail: 0/1 Skip: 0/1 -## BlockRef and helpers [Preset: mainnet] +## BlockId and helpers ```diff -+ get_ancestor sanity [Preset: mainnet] OK -+ isAncestorOf sanity [Preset: mainnet] OK ++ atSlot sanity OK ++ parent sanity OK ``` OK: 2/2 Fail: 0/2 Skip: 0/2 -## BlockSlot and helpers [Preset: mainnet] +## BlockRef and helpers ```diff -+ atSlot sanity [Preset: mainnet] OK -+ parent sanity [Preset: mainnet] OK ++ get_ancestor sanity OK ++ isAncestorOf sanity OK ``` OK: 2/2 Fail: 0/2 Skip: 0/2 +## BlockSlot and helpers +```diff ++ atSlot sanity OK ++ parent sanity OK +``` +OK: 2/2 Fail: 0/2 Skip: 0/2 +## ChainDAG helpers +```diff ++ epochAncestor sanity [Preset: mainnet] OK +``` +OK: 1/1 Fail: 0/1 Skip: 0/1 ## Diverging hardforks ```diff + Non-tail block in common OK @@ -377,4 +388,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 1/1 Fail: 0/1 Skip: 0/1 ---TOTAL--- -OK: 209/211 Fail: 0/211 Skip: 2/211 +OK: 212/214 Fail: 0/214 Skip: 2/214 diff --git a/beacon_chain/consensus_object_pools/block_dag.nim b/beacon_chain/consensus_object_pools/block_dag.nim new file mode 100644 index 000000000..882487538 --- /dev/null +++ b/beacon_chain/consensus_object_pools/block_dag.nim @@ -0,0 +1,246 @@ +# beacon_chain +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [Defect].} + +import + chronicles, + + ../spec/datatypes/[phase0, altair, merge], + ../spec/[helpers] + +export chronicles, phase0, altair, merge, helpers + +type + BlockId* = object + ## A BlockId is the root and the slot in which that block was + ## produced - there are no guarantees that this block is part of + ## the canonical chain, or that we have validated it + root*: Eth2Digest + slot*: Slot + + BlockSlotId* = object + ## A BlockId at a slot equal to or higher than the slot of the block + bid*: BlockId + slot*: Slot + + BlockRef* = ref object + ## Node in object graph guaranteed to lead back to tail block, and to have + ## a corresponding entry in database. + ## + ## All blocks identified by a `BlockRef` are valid per the state transition + ## rules and that at some point were candidates for head selection. The + ## ChainDAG offers stronger guarantees: it only returns `BlockRef` instances + ## that are rooted in the currently finalized chain - however, these + ## guarantees are valid only until the next head update - in particular, + ## they are not valid across `await` calls. + ## + ## Block graph forms a tree - in particular, there are no cycles. + + bid*: BlockId ##\ + ## Root that can be used to retrieve block data from database + + parent*: BlockRef ##\ + ## Not nil, except for the tail + + BlockSlot* = object + ## Unique identifier for a particular fork and time in the block chain - + ## normally, there's a block for every slot, but in the case a block is not + ## produced, the chain progresses anyway, producing a new state for every + ## slot. + blck*: BlockRef + slot*: Slot ##\ + ## Slot time for this BlockSlot which may differ from blck.slot when time + ## has advanced without blocks + + +template root*(blck: BlockRef): Eth2Digest = blck.bid.root +template slot*(blck: BlockRef): Slot = blck.bid.slot + +func init*(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef = + BlockRef( + bid: BlockId(root: root, slot: slot) + ) + +func init*(T: type BlockRef, root: Eth2Digest, blck: SomeSomeBeaconBlock): + BlockRef = + BlockRef.init(root, blck.slot) + +func toBlockId*(blck: SomeSomeSignedBeaconBlock): BlockId = + BlockId(root: blck.root, slot: blck.message.slot) + +func toBlockId*(blck: ForkedSignedBeaconBlock): BlockId = + withBlck(blck): BlockId(root: blck.root, slot: blck.message.slot) + +func parent*(bs: BlockSlot): BlockSlot = + ## Return a blockslot representing the previous slot, using the parent block + ## if the current slot had a block + if bs.slot == Slot(0): + BlockSlot(blck: nil, slot: Slot(0)) + else: + BlockSlot( + blck: if bs.slot > bs.blck.slot: bs.blck else: bs.blck.parent, + slot: bs.slot - 1 + ) + +func parentOrSlot*(bs: BlockSlot): BlockSlot = + ## Return a blockslot representing the previous slot, using the parent block + ## with the current slot if the current had a block + if bs.blck.isNil(): + BlockSlot(blck: nil, slot: Slot(0)) + elif bs.slot == bs.blck.slot: + BlockSlot(blck: bs.blck.parent, slot: bs.slot) + else: + BlockSlot(blck: bs.blck, slot: bs.slot - 1) + +func getDepth*(a, b: BlockRef): tuple[ancestor: bool, depth: int] = + var b = b + var depth = 0 + const maxDepth = (100'i64 * 365 * 24 * 60 * 60 div SECONDS_PER_SLOT.int) + while true: + if a == b: + return (true, depth) + + # for now, use an assert for block chain length since a chain this long + # indicates a circular reference here.. + doAssert depth < maxDepth + depth += 1 + + if a.slot >= b.slot or b.parent.isNil: + return (false, depth) + + doAssert b.slot > b.parent.slot + b = b.parent + +func isAncestorOf*(a, b: BlockRef): bool = + let (isAncestor, _) = getDepth(a, b) + isAncestor + +func link*(parent, child: BlockRef) = + doAssert (not (parent.root == Eth2Digest() or child.root == Eth2Digest())), + "blocks missing root!" + doAssert parent.root != child.root, "self-references not allowed" + + child.parent = parent + +func get_ancestor*(blck: BlockRef, slot: Slot, + maxDepth = 100'i64 * 365 * 24 * 60 * 60 div SECONDS_PER_SLOT.int): + BlockRef = + ## https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/fork-choice.md#get_ancestor + ## Return the most recent block as of the time at `slot` that not more recent + ## than `blck` itself + if isNil(blck): return nil + + var blck = blck + + var depth = 0 + + while true: + if blck.slot <= slot: + return blck + + if isNil(blck.parent): + return nil + + doAssert depth < maxDepth + depth += 1 + + blck = blck.parent + +func atSlot*(blck: BlockRef, slot: Slot): BlockSlot = + ## Return a BlockSlot at a given slot, with the block set to the closest block + ## available. If slot comes from before the block, a suitable block ancestor + ## will be used, else blck is returned as if all slots after it were empty. + ## This helper is useful when imagining what the chain looked like at a + ## particular moment in time, or when imagining what it will look like in the + ## near future if nothing happens (such as when looking ahead for the next + ## block proposal) + BlockSlot(blck: blck.get_ancestor(slot), slot: slot) + +func atSlot*(blck: BlockRef): BlockSlot = + blck.atSlot(blck.slot) + +func atSlot*(bid: BlockId, slot: Slot): BlockSlotId = + BlockSlotId(bid: bid, slot: slot) + +func atSlot*(bid: BlockId): BlockSlotId = + bid.atSlot(bid.slot) + +func atEpochStart*(blck: BlockRef, epoch: Epoch): BlockSlot = + ## Return the BlockSlot corresponding to the first slot in the given epoch + atSlot(blck, epoch.compute_start_slot_at_epoch()) + +func atSlotEpoch*(blck: BlockRef, epoch: Epoch): BlockSlot = + ## Return the last block that was included in the chain leading + ## up to the given epoch - this amounts to the state at the time + ## when epoch processing for `epoch` has been done, but no block + ## has yet been applied + if epoch == GENESIS_EPOCH: + blck.atEpochStart(epoch) + else: + let start = epoch.compute_start_slot_at_epoch() + let tmp = blck.atSlot(start - 1) + if isNil(tmp.blck): + BlockSlot() + else: + tmp.blck.atSlot(start) + +func toBlockSlotId*(bs: BlockSlot): BlockSlotId = + if isNil(bs.blck): + BlockSlotId() + else: + bs.blck.bid.atSlot(bs.slot) + +func isProposed*(bid: BlockId, slot: Slot): bool = + ## Return true if `bid` was proposed in the given slot + bid.slot == slot + +func isProposed*(blck: BlockRef, slot: Slot): bool = + ## Return true if `blck` was proposed in the given slot + not isNil(blck) and blck.isProposed(slot) + +func isProposed*(bs: BlockSlot): bool = + ## Return true if `bs` represents the proposed block (as opposed to an empty + ## slot) + bs.blck.isProposed(bs.slot) + +func isProposed*(bsi: BlockSlotId): bool = + ## Return true if `bs` represents the proposed block (as opposed to an empty + ## slot) + bsi.bid.isProposed(bsi.slot) + +func shortLog*(v: BlockId): string = + # epoch:root when logging epoch, root:slot when logging slot! + shortLog(v.root) & ":" & $v.slot + +func shortLog*(v: BlockSlotId): string = + # epoch:root when logging epoch, root:slot when logging slot! + if v.bid.slot == v.slot: + shortLog(v.bid) + else: # There was a gap - log it + shortLog(v.bid) & "@" & $v.slot + +func shortLog*(v: BlockRef): string = + # epoch:root when logging epoch, root:slot when logging slot! + if v.isNil(): + "nil:0" + else: + shortLog(v.bid) + +func shortLog*(v: BlockSlot): string = + # epoch:root when logging epoch, root:slot when logging slot! + if isNil(v.blck): + "nil:0@" & $v.slot + elif v.blck.slot == v.slot: + shortLog(v.blck) + else: # There was a gap - log it + shortLog(v.blck) & "@" & $v.slot + +chronicles.formatIt BlockId: shortLog(it) +chronicles.formatIt BlockSlotId: shortLog(it) +chronicles.formatIt BlockSlot: shortLog(it) +chronicles.formatIt BlockRef: shortLog(it) diff --git a/beacon_chain/consensus_object_pools/block_pools_types.nim b/beacon_chain/consensus_object_pools/block_pools_types.nim index 25063326c..8f5b16619 100644 --- a/beacon_chain/consensus_object_pools/block_pools_types.nim +++ b/beacon_chain/consensus_object_pools/block_pools_types.nim @@ -9,15 +9,16 @@ import # Standard library - std/[sets, tables, hashes], + std/[options, sets, tables, hashes], # Status libraries stew/endians2, chronicles, # Internals - ../spec/[signatures_batch, forks], + ../spec/[signatures_batch, forks, helpers], ../spec/datatypes/[phase0, altair, merge], - ".."/beacon_chain_db + ".."/beacon_chain_db, + ./block_dag -export sets, tables +export options, sets, tables, hashes, helpers, beacon_chain_db, block_dag # ChainDAG and types related to forming a DAG of blocks, keeping track of their # relationships and allowing various forms of lookups @@ -184,19 +185,6 @@ type # balances, as used in fork choice effective_balances_bytes*: seq[byte] - BlockRef* = ref object - ## Node in object graph guaranteed to lead back to tail block, and to have - ## a corresponding entry in database. - ## Block graph should form a tree - in particular, there are no cycles. - - root*: Eth2Digest ##\ - ## Root that can be used to retrieve block data from database - - parent*: BlockRef ##\ - ## Not nil, except for the tail - - slot*: Slot # could calculate this by walking to root, but.. - BlockData* = object ## Body and graph in one @@ -209,16 +197,6 @@ type blck*: BlockRef ##\ ## The block associated with the state found in data - BlockSlot* = object - ## Unique identifier for a particular fork and time in the block chain - - ## normally, there's a block for every slot, but in the case a block is not - ## produced, the chain progresses anyway, producing a new state for every - ## slot. - blck*: BlockRef - slot*: Slot ##\ - ## Slot time for this BlockSlot which may differ from blck.slot when time - ## has advanced without blocks - OnPhase0BlockAdded* = proc( blckRef: BlockRef, blck: phase0.TrustedSignedBeaconBlock, @@ -259,22 +237,6 @@ template head*(dag: ChainDAGRef): BlockRef = dag.headState.blck template epoch*(e: EpochRef): Epoch = e.key.epoch -func shortLog*(v: BlockRef): string = - # epoch:root when logging epoch, root:slot when logging slot! - if v.isNil(): - "nil:0" - else: - shortLog(v.root) & ":" & $v.slot - -func shortLog*(v: BlockSlot): string = - # epoch:root when logging epoch, root:slot when logging slot! - if v.blck.isNil(): - "nil:0@" & $v.slot - elif v.blck.slot == v.slot: - shortLog(v.blck) - else: # There was a gap - log it - shortLog(v.blck) & "@" & $v.slot - func shortLog*(v: EpochKey): string = # epoch:root when logging epoch, root:slot when logging slot! $v.epoch & ":" & shortLog(v.blck) @@ -286,8 +248,6 @@ func shortLog*(v: EpochRef): string = else: shortLog(v.key) -chronicles.formatIt BlockSlot: shortLog(it) -chronicles.formatIt BlockRef: shortLog(it) chronicles.formatIt EpochKey: shortLog(it) chronicles.formatIt EpochRef: shortLog(it) @@ -299,7 +259,7 @@ func `==`*(a, b: KeyedBlockRef): bool = func asLookupKey*(T: type KeyedBlockRef, root: Eth2Digest): KeyedBlockRef = # Create a special, temporary BlockRef instance that just has the key set - KeyedBlockRef(data: BlockRef(root: root)) + KeyedBlockRef(data: BlockRef(bid: BlockId(root: root))) func init*(T: type KeyedBlockRef, blck: BlockRef): KeyedBlockRef = KeyedBlockRef(data: blck) @@ -340,3 +300,4 @@ func init*(t: typedesc[FinalizationInfoObject], blockRoot: Eth2Digest, state_root: stateRoot, epoch: epoch ) + diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index 642b9d8d3..fdbc8ca34 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -81,27 +81,6 @@ template withState*( withStateVars(stateData): body -func parent*(bs: BlockSlot): BlockSlot = - ## Return a blockslot representing the previous slot, using the parent block - ## if the current slot had a block - if bs.slot == Slot(0): - BlockSlot(blck: nil, slot: Slot(0)) - else: - BlockSlot( - blck: if bs.slot > bs.blck.slot: bs.blck else: bs.blck.parent, - slot: bs.slot - 1 - ) - -func parentOrSlot*(bs: BlockSlot): BlockSlot = - ## Return a blockslot representing the previous slot, using the parent block - ## with the current slot if the current had a block - if bs.blck.isNil(): - BlockSlot(blck: nil, slot: Slot(0)) - elif bs.slot == bs.blck.slot: - BlockSlot(blck: bs.blck.parent, slot: bs.slot) - else: - BlockSlot(blck: bs.blck, slot: bs.slot - 1) - func get_effective_balances(validators: openArray[Validator], epoch: Epoch): seq[Gwei] = ## Get the balances from a state as counted for fork choice @@ -137,8 +116,6 @@ func validatorKey*( ## non-head branch)! validatorKey(epochRef.dag, index) -func epochAncestor*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochKey - func init*( T: type EpochRef, dag: ChainDAGRef, state: StateData, cache: var StateCache): T = @@ -146,7 +123,7 @@ func init*( epoch = state.data.get_current_epoch() epochRef = EpochRef( dag: dag, # This gives access to the validator pubkeys through an EpochRef - key: epochAncestor(dag, state.blck, epoch), + key: state.blck.epochAncestor(epoch), eth1_data: getStateField(state.data, eth1_data), eth1_deposit_index: getStateField(state.data, eth1_deposit_index), current_justified_checkpoint: @@ -163,10 +140,11 @@ func init*( state.data.mergeData.data.latest_execution_payload_header != ExecutionPayloadHeader() ) + epochStart = epoch.compute_start_slot_at_epoch() for i in 0'u64..= b.slot or b.parent.isNil: - return (false, depth) - - doAssert b.slot > b.parent.slot - b = b.parent - -func isAncestorOf*(a, b: BlockRef): bool = - let (isAncestor, _) = getDepth(a, b) - isAncestor - -func get_ancestor*(blck: BlockRef, slot: Slot, - maxDepth = 100'i64 * 365 * 24 * 60 * 60 div SECONDS_PER_SLOT.int): - BlockRef = - ## https://github.com/ethereum/consensus-specs/blob/v1.0.1/specs/phase0/fork-choice.md#get_ancestor - ## Return the most recent block as of the time at `slot` that not more recent - ## than `blck` itself - doAssert not blck.isNil - - var blck = blck - - var depth = 0 - - while true: - if blck.slot <= slot: - return blck - - if blck.parent.isNil: - return nil - - doAssert depth < maxDepth - depth += 1 - - blck = blck.parent - -func atSlot*(blck: BlockRef, slot: Slot): BlockSlot = - ## Return a BlockSlot at a given slot, with the block set to the closest block - ## available. If slot comes from before the block, a suitable block ancestor - ## will be used, else blck is returned as if all slots after it were empty. - ## This helper is useful when imagining what the chain looked like at a - ## particular moment in time, or when imagining what it will look like in the - ## near future if nothing happens (such as when looking ahead for the next - ## block proposal) - BlockSlot(blck: blck.get_ancestor(slot), slot: slot) - -func atEpochStart*(blck: BlockRef, epoch: Epoch): BlockSlot = - ## Return the BlockSlot corresponding to the first slot in the given epoch - atSlot(blck, epoch.compute_start_slot_at_epoch) - func getBlockBySlot*(dag: ChainDAGRef, slot: Slot): BlockSlot = ## Retrieve the canonical block at the given slot, or the last block that ## comes before - similar to atSlot, but without the linear scan @@ -275,26 +185,29 @@ func getBlockBySlot*(dag: ChainDAGRef, slot: Slot): BlockSlot = raiseAssert "At least the genesis block should be available!" tmp = tmp - 1 -func epochAncestor*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochKey = +func epochAncestor*(blck: BlockRef, epoch: Epoch): EpochKey = ## The state transition works by storing information from blocks in a ## "working" area until the epoch transition, then batching work collected ## during the epoch. Thus, last block in the ancestor epochs is the block ## that has an impact on epoch currently considered. ## - ## This function returns a BlockSlot pointing to that epoch boundary, ie the + ## This function returns an epoch key pointing to that epoch boundary, i.e. the ## boundary where the last block has been applied to the state and epoch ## processing has been done. - let blck = - if epoch == GENESIS_EPOCH: - dag.genesis - else: - dag.getBlockBySlot(compute_start_slot_at_epoch(epoch) - 1).blck + var blck = blck + while blck.slot.epoch >= epoch and not blck.parent.isNil: + blck = blck.parent EpochKey(epoch: epoch, blck: blck) func findEpochRef*( dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef = # may return nil! - let ancestor = epochAncestor(dag, blck, epoch) + if epoch < dag.tail.slot.epoch: + # We can't compute EpochRef instances for states before the tail because + # we do not have them! + return + + let ancestor = epochAncestor(blck, epoch) doAssert ancestor.blck != nil for i in 0.. 0: load(epoch - 1) -func init(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef = - BlockRef( - root: root, - slot: slot - ) - -func init*(T: type BlockRef, root: Eth2Digest, blck: SomeSomeBeaconBlock): - BlockRef = - BlockRef.init(root, blck.slot) - func contains*(dag: ChainDAGRef, root: Eth2Digest): bool = KeyedBlockRef.asLookupKey(root) in dag.blocks proc containsBlock( - cfg: RuntimeConfig, db: BeaconChainDB, blck: BlockRef): bool = - case cfg.blockForkAtEpoch(blck.slot.epoch) - of BeaconBlockFork.Phase0: db.containsBlockPhase0(blck.root) - of BeaconBlockFork.Altair: db.containsBlockAltair(blck.root) - of BeaconBlockFork.Merge: db.containsBlockMerge(blck.root) + cfg: RuntimeConfig, db: BeaconChainDB, slot: Slot, root: Eth2Digest): bool = + case cfg.blockForkAtEpoch(slot.epoch) + of BeaconBlockFork.Phase0: db.containsBlockPhase0(root) + of BeaconBlockFork.Altair: db.containsBlockAltair(root) + of BeaconBlockFork.Merge: db.containsBlockMerge(root) func isStateCheckpoint(bs: BlockSlot): bool = ## State checkpoints are the points in time for which we store full state @@ -459,7 +362,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, curRef = curRef.parent # Don't include blocks on incorrect hardforks - if headRef == nil and cfg.containsBlock(db, newRef): + if headRef == nil and cfg.containsBlock(db, newRef.slot, newRef.root): headRef = newRef blocks.incl(KeyedBlockRef.init(curRef)) @@ -480,7 +383,7 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, headRef = tailRef var - cur = headRef.atSlot(headRef.slot) + cur = headRef.atSlot() tmpState = (ref StateData)() # Now that we have a head block, we need to find the most recent state that @@ -524,13 +427,14 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, quit 1 let dag = ChainDAGRef( - blocks: blocks, - tail: tailRef, - genesis: genesisRef, db: db, - forkDigests: newClone ForkDigests.init( - cfg, - getStateField(tmpState.data, genesis_validators_root)), + blocks: blocks, + genesis: genesisRef, + tail: tailRef, + finalizedHead: tailRef.atSlot(), + lastPrunePoint: tailRef.atSlot(), + # Tail is implicitly finalized - we'll adjust it below when computing the + # head state heads: @[headRef], headState: tmpState[], epochRefState: tmpState[], @@ -541,6 +445,10 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, updateFlags: {verifyFinalization} * updateFlags, cfg: cfg, + forkDigests: newClone ForkDigests.init( + cfg, + getStateField(tmpState.data, genesis_validators_root)), + onBlockAdded: onBlockCb, onHeadChanged: onHeadCb, onReorgHappened: onReorgCb, @@ -554,17 +462,18 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, doAssert dag.updateFlags in [{}, {verifyFinalization}] var cache: StateCache - dag.updateStateData(dag.headState, headRef.atSlot(headRef.slot), false, cache) - # We presently save states on the epoch boundary - it means that the latest - # state we loaded might be older than head block - nonetheless, it will be - # from the same epoch as the head, thus the finalized and justified slots are - # the same - these only change on epoch boundaries. - # When we start from a snapshot state, the `finalized_checkpoint` in the - # snapshot will point to an even older state, but we trust the tail state - # (the snapshot) to be finalized, hence the `max` expression below. - let finalizedEpoch = max(getStateField(dag.headState.data, finalized_checkpoint).epoch, - tailRef.slot.epoch) - dag.finalizedHead = headRef.atEpochStart(finalizedEpoch) + dag.updateStateData(dag.headState, headRef.atSlot(), false, cache) + + # The tail block is "implicitly" finalized as it was given either as a + # checkpoint block, or is the genesis, thus we use it as a lower bound when + # computing the finalized head + let + finalized_checkpoint = + getStateField(dag.headState.data, finalized_checkpoint) + finalizedSlot = max( + finalized_checkpoint.epoch.compute_start_slot_at_epoch(), tailRef.slot) + + dag.finalizedHead = headRef.atSlot(finalizedSlot) block: dag.finalizedBlocks.setLen(dag.finalizedHead.slot.int + 1) @@ -587,10 +496,10 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, dag.headSyncCommittees = state.data.get_sync_committee_cache(cache) info "Block dag initialized", - head = shortLog(headRef), + head = shortLog(dag.head), finalizedHead = shortLog(dag.finalizedHead), - tail = shortLog(tailRef), - totalBlocks = blocks.len + tail = shortLog(dag.tail), + totalBlocks = dag.blocks.len dag @@ -638,7 +547,7 @@ proc getEpochRef*(dag: ChainDAGRef, blck: BlockRef, epoch: Epoch): EpochRef = beacon_state_data_cache_misses.inc let - ancestor = epochAncestor(dag, blck, epoch) + ancestor = epochAncestor(blck, epoch) dag.withState( dag.epochRefState, ancestor.blck.atEpochStart(ancestor.epoch)): @@ -1050,7 +959,7 @@ proc pruneBlocksDAG(dag: ChainDAGRef) = if dag.finalizedHead.blck.isAncestorOf(head): continue - var cur = head.atSlot(head.slot) + var cur = head.atSlot() while not cur.blck.isAncestorOf(dag.finalizedHead.blck): dag.delState(cur) # TODO: should we move that disk I/O to `onSlotEnd` @@ -1229,7 +1138,7 @@ proc updateHead*( # to use existing in-memory states to make this smooth var cache: StateCache updateStateData( - dag, dag.headState, newHead.atSlot(newHead.slot), false, cache) + dag, dag.headState, newHead.atSlot(), false, cache) dag.db.putHeadBlock(newHead.root) @@ -1238,8 +1147,13 @@ proc updateHead*( dag.headSyncCommittees = state.data.get_sync_committee_cache(cache) let - finalizedHead = newHead.atEpochStart( - getStateField(dag.headState.data, finalized_checkpoint).epoch) + finalized_checkpoint = + getStateField(dag.headState.data, finalized_checkpoint) + finalizedSlot = max( + finalized_checkpoint.epoch.compute_start_slot_at_epoch(), + dag.tail.slot) + + finalizedHead = newHead.atSlot(finalizedSlot) doAssert (not finalizedHead.blck.isNil), "Block graph should always lead to a finalized block" @@ -1366,13 +1280,19 @@ proc updateHead*( # Send notification about new finalization point via callback. if not(isNil(dag.onFinHappened)): - let epoch = getStateField( - dag.headState.data, finalized_checkpoint).epoch - let blckRoot = getStateField( - dag.headState.data, finalized_checkpoint).root - let data = FinalizationInfoObject.init(blckRoot, - getStateRoot(dag.headState.data), - epoch) + let stateRoot = + if dag.finalizedHead.slot == dag.head.slot: + getStateRoot(dag.headState.data) + elif dag.finalizedHead.slot + SLOTS_PER_HISTORICAL_ROOT > dag.head.slot: + getStateField(dag.headState.data, state_roots).data[ + int(dag.finalizedHead.slot mod SLOTS_PER_HISTORICAL_ROOT)] + else: + Eth2Digest() # The thing that finalized was >8192 blocks old? + + let data = FinalizationInfoObject.init( + dag.finalizedHead.blck.root, + stateRoot, + dag.finalizedHead.slot.epoch) dag.onFinHappened(data) proc isInitialized*(T: type ChainDAGRef, db: BeaconChainDB): bool = @@ -1473,12 +1393,6 @@ proc preInit*( fork = state.data.fork, validators = state.data.validators.len() -proc getGenesisBlockData*(dag: ChainDAGRef): BlockData = - dag.get(dag.genesis) - -func getGenesisBlockSlot*(dag: ChainDAGRef): BlockSlot = - BlockSlot(blck: dag.genesis, slot: GENESIS_SLOT) - proc getProposer*( dag: ChainDAGRef, head: BlockRef, slot: Slot): Option[ValidatorIndex] = let diff --git a/beacon_chain/rpc/rest_utils.nim b/beacon_chain/rpc/rest_utils.nim index 95796fb5f..7276cbbf6 100644 --- a/beacon_chain/rpc/rest_utils.nim +++ b/beacon_chain/rpc/rest_utils.nim @@ -69,9 +69,6 @@ proc getCurrentHead*(node: BeaconNode, return err("Requesting epoch for which slot would overflow") node.getCurrentHead(compute_start_slot_at_epoch(epoch)) -proc toBlockSlot*(blckRef: BlockRef): BlockSlot = - blckRef.atSlot(blckRef.slot) - proc getBlockSlot*(node: BeaconNode, stateIdent: StateIdent): Result[BlockSlot, cstring] = case stateIdent.kind @@ -79,16 +76,16 @@ proc getBlockSlot*(node: BeaconNode, ok(node.dag.getBlockBySlot(? node.getCurrentSlot(stateIdent.slot))) of StateQueryKind.Root: if stateIdent.root == getStateRoot(node.dag.headState.data): - ok(node.dag.headState.blck.toBlockSlot()) + ok(node.dag.headState.blck.atSlot()) else: # We don't have a state root -> BlockSlot mapping err("State not found") of StateQueryKind.Named: case stateIdent.value of StateIdentType.Head: - ok(node.dag.head.toBlockSlot()) + ok(node.dag.head.atSlot()) of StateIdentType.Genesis: - ok(node.dag.getGenesisBlockSlot()) + ok(node.dag.genesis.atSlot()) of StateIdentType.Finalized: ok(node.dag.finalizedHead) of StateIdentType.Justified: diff --git a/beacon_chain/rpc/rpc_beacon_api.nim b/beacon_chain/rpc/rpc_beacon_api.nim index 772412841..a5feca12a 100644 --- a/beacon_chain/rpc/rpc_beacon_api.nim +++ b/beacon_chain/rpc/rpc_beacon_api.nim @@ -161,7 +161,7 @@ proc getBlockDataFromBlockId(node: BeaconNode, blockId: string): BlockData {. of "head": node.dag.get(node.dag.head) of "genesis": - node.dag.getGenesisBlockData() + node.dag.get(node.dag.genesis) of "finalized": node.dag.get(node.dag.finalizedHead.blck) else: diff --git a/beacon_chain/rpc/rpc_utils.nim b/beacon_chain/rpc/rpc_utils.nim index 357451b0b..41ca021bf 100644 --- a/beacon_chain/rpc/rpc_utils.nim +++ b/beacon_chain/rpc/rpc_utils.nim @@ -38,9 +38,6 @@ template withStateForStateId*(stateId: string, body: untyped): untyped = node.dag.withState(rpcState[], bs): body -proc toBlockSlot*(blckRef: BlockRef): BlockSlot = - blckRef.atSlot(blckRef.slot) - proc parseRoot*(str: string): Eth2Digest {.raises: [Defect, ValueError].} = Eth2Digest(data: hexToByteArray[32](str)) @@ -74,9 +71,9 @@ proc getBlockSlotFromString*(node: BeaconNode, slot: string): BlockSlot {.raises proc stateIdToBlockSlot*(node: BeaconNode, stateId: string): BlockSlot {.raises: [Defect, CatchableError].} = case stateId: of "head": - node.dag.head.toBlockSlot() + node.dag.head.atSlot() of "genesis": - node.dag.getGenesisBlockSlot() + node.dag.genesis.atSlot() of "finalized": node.dag.finalizedHead of "justified": @@ -84,10 +81,12 @@ proc stateIdToBlockSlot*(node: BeaconNode, stateId: string): BlockSlot {.raises: getStateField(node.dag.headState.data, current_justified_checkpoint).epoch) else: if stateId.startsWith("0x"): - let blckRoot = parseRoot(stateId) - let blckRef = node.dag.getRef(blckRoot) - if blckRef.isNil: - raise newException(CatchableError, "Block not found") - blckRef.toBlockSlot() - else: + let stateRoot = parseRoot(stateId) + if stateRoot == getStateRoot(node.dag.headState.data): + node.dag.headState.blck.atSlot() + else: + # We don't have a state root -> BlockSlot mapping + raise (ref ValueError)(msg: "State not found") + + else: # Parse as slot number node.getBlockSlotFromString(stateId) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 3a494fc54..8ca0954ec 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -15,6 +15,7 @@ import # Unit test ./test_action_tracker, ./test_attestation_pool, ./test_beacon_chain_db, + ./test_block_dag, ./test_block_processor, ./test_blockchain_dag, ./test_datatypes, diff --git a/tests/test_block_dag.nim b/tests/test_block_dag.nim new file mode 100644 index 000000000..7dc2a0757 --- /dev/null +++ b/tests/test_block_dag.nim @@ -0,0 +1,121 @@ +# beacon_chain +# Copyright (c) 2018-2021 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.used.} + +import + chronicles, + unittest2, + ../beacon_chain/consensus_object_pools/block_dag + +func `$`(x: BlockRef): string = shortLog(x) + +suite "BlockRef and helpers": + test "isAncestorOf sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s1 = BlockRef(bid: BlockId(slot: Slot(1)), parent: s0) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s1) + + check: + s0.isAncestorOf(s0) + s0.isAncestorOf(s1) + s0.isAncestorOf(s2) + s1.isAncestorOf(s1) + s1.isAncestorOf(s2) + + not s2.isAncestorOf(s0) + not s2.isAncestorOf(s1) + not s1.isAncestorOf(s0) + + test "get_ancestor sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s1 = BlockRef(bid: BlockId(slot: Slot(1)), parent: s0) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s1) + s4 = BlockRef(bid: BlockId(slot: Slot(4)), parent: s2) + + check: + s0.get_ancestor(Slot(0)) == s0 + s0.get_ancestor(Slot(1)) == s0 + + s1.get_ancestor(Slot(0)) == s0 + s1.get_ancestor(Slot(1)) == s1 + + s4.get_ancestor(Slot(0)) == s0 + s4.get_ancestor(Slot(1)) == s1 + s4.get_ancestor(Slot(2)) == s2 + s4.get_ancestor(Slot(3)) == s2 + s4.get_ancestor(Slot(4)) == s4 + +suite "BlockSlot and helpers": + test "atSlot sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s1 = BlockRef(bid: BlockId(slot: Slot(1)), parent: s0) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s1) + s4 = BlockRef(bid: BlockId(slot: Slot(4)), parent: s2) + + check: + s0.atSlot(Slot(0)).blck == s0 + s0.atSlot(Slot(0)) == s1.atSlot(Slot(0)) + s1.atSlot(Slot(1)).blck == s1 + + s4.atSlot(Slot(0)).blck == s0 + + s4.atSlot() == s4.atSlot(s4.slot) + + test "parent sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s00 = BlockSlot(blck: s0, slot: Slot(0)) + s01 = BlockSlot(blck: s0, slot: Slot(1)) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s0) + s22 = BlockSlot(blck: s2, slot: Slot(2)) + s24 = BlockSlot(blck: s2, slot: Slot(4)) + + check: + s00.parent == BlockSlot(blck: nil, slot: Slot(0)) + s01.parent == s00 + s01.parentOrSlot == s00 + s22.parent == s01 + s22.parentOrSlot == BlockSlot(blck: s0, slot: Slot(2)) + s24.parent == BlockSlot(blck: s2, slot: Slot(3)) + s24.parent.parent == s22 + +suite "BlockId and helpers": + test "atSlot sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s1 = BlockRef(bid: BlockId(slot: Slot(1)), parent: s0) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s1) + s4 = BlockRef(bid: BlockId(slot: Slot(4)), parent: s2) + + check: + s0.atSlot(Slot(0)).blck == s0 + s0.atSlot(Slot(0)) == s1.atSlot(Slot(0)) + s1.atSlot(Slot(1)).blck == s1 + + s4.atSlot(Slot(0)).blck == s0 + + test "parent sanity": + let + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + s00 = BlockSlot(blck: s0, slot: Slot(0)) + s01 = BlockSlot(blck: s0, slot: Slot(1)) + s2 = BlockRef(bid: BlockId(slot: Slot(2)), parent: s0) + s22 = BlockSlot(blck: s2, slot: Slot(2)) + s24 = BlockSlot(blck: s2, slot: Slot(4)) + + check: + s00.parent == BlockSlot(blck: nil, slot: Slot(0)) + s01.parent == s00 + s01.parentOrSlot == s00 + s22.parent == s01 + s22.parentOrSlot == BlockSlot(blck: s0, slot: Slot(2)) + s24.parent == BlockSlot(blck: s2, slot: Slot(3)) + s24.parent.parent == s22 diff --git a/tests/test_blockchain_dag.nim b/tests/test_blockchain_dag.nim index bc9cd9154..c6287b9c0 100644 --- a/tests/test_blockchain_dag.nim +++ b/tests/test_blockchain_dag.nim @@ -19,8 +19,7 @@ import attestation_pool, blockchain_dag, block_quarantine, block_clearance], ./testutil, ./testdbutil, ./testblockutil -func `$`(x: BlockRef): string = - $x.root +func `$`(x: BlockRef): string = shortLog(x) const nilPhase0Callback = OnPhase0BlockAdded(nil) @@ -30,76 +29,22 @@ proc pruneAtFinalization(dag: ChainDAGRef) = if dag.needStateCachesAndForkChoicePruning(): dag.pruneStateCachesDAG() -suite "BlockRef and helpers" & preset(): - test "isAncestorOf sanity" & preset(): +suite "ChainDAG helpers": + test "epochAncestor sanity" & preset(): let - s0 = BlockRef(slot: Slot(0)) - s1 = BlockRef(slot: Slot(1), parent: s0) - s2 = BlockRef(slot: Slot(2), parent: s1) + s0 = BlockRef(bid: BlockId(slot: Slot(0))) + var cur = s0 + for i in 1..SLOTS_PER_EPOCH * 2: + cur = BlockRef(bid: BlockId(slot: Slot(i)), parent: cur) + + let ancestor = cur.epochAncestor(cur.slot.epoch) check: - s0.isAncestorOf(s0) - s0.isAncestorOf(s1) - s0.isAncestorOf(s2) - s1.isAncestorOf(s1) - s1.isAncestorOf(s2) + ancestor.epoch == cur.slot.epoch + ancestor.blck != cur # should have selected a parent - not s2.isAncestorOf(s0) - not s2.isAncestorOf(s1) - not s1.isAncestorOf(s0) - - test "get_ancestor sanity" & preset(): - let - s0 = BlockRef(slot: Slot(0)) - s1 = BlockRef(slot: Slot(1), parent: s0) - s2 = BlockRef(slot: Slot(2), parent: s1) - s4 = BlockRef(slot: Slot(4), parent: s2) - - check: - s0.get_ancestor(Slot(0)) == s0 - s0.get_ancestor(Slot(1)) == s0 - - s1.get_ancestor(Slot(0)) == s0 - s1.get_ancestor(Slot(1)) == s1 - - s4.get_ancestor(Slot(0)) == s0 - s4.get_ancestor(Slot(1)) == s1 - s4.get_ancestor(Slot(2)) == s2 - s4.get_ancestor(Slot(3)) == s2 - s4.get_ancestor(Slot(4)) == s4 - -suite "BlockSlot and helpers" & preset(): - test "atSlot sanity" & preset(): - let - s0 = BlockRef(slot: Slot(0)) - s1 = BlockRef(slot: Slot(1), parent: s0) - s2 = BlockRef(slot: Slot(2), parent: s1) - s4 = BlockRef(slot: Slot(4), parent: s2) - - check: - s0.atSlot(Slot(0)).blck == s0 - s0.atSlot(Slot(0)) == s1.atSlot(Slot(0)) - s1.atSlot(Slot(1)).blck == s1 - - s4.atSlot(Slot(0)).blck == s0 - - test "parent sanity" & preset(): - let - s0 = BlockRef(slot: Slot(0)) - s00 = BlockSlot(blck: s0, slot: Slot(0)) - s01 = BlockSlot(blck: s0, slot: Slot(1)) - s2 = BlockRef(slot: Slot(2), parent: s0) - s22 = BlockSlot(blck: s2, slot: Slot(2)) - s24 = BlockSlot(blck: s2, slot: Slot(4)) - - check: - s00.parent == BlockSlot(blck: nil, slot: Slot(0)) - s01.parent == s00 - s01.parentOrSlot == s00 - s22.parent == s01 - s22.parentOrSlot == BlockSlot(blck: s0, slot: Slot(2)) - s24.parent == BlockSlot(blck: s2, slot: Slot(3)) - s24.parent.parent == s22 + ancestor.blck.epochAncestor(cur.slot.epoch) == ancestor + ancestor.blck.epochAncestor(ancestor.blck.slot.epoch) != ancestor suite "Block pool processing" & preset(): setup: @@ -601,11 +546,8 @@ suite "Diverging hardforks": dag = init(ChainDAGRef, phase0RuntimeConfig, db, {}) verifier = BatchVerifier(rng: keys.newRng(), taskpool: Taskpool.new()) quarantine = newClone(Quarantine.init()) - nilPhase0Callback: OnPhase0BlockAdded - state = newClone(dag.headState.data) cache = StateCache() info = ForkedEpochInfo() - blck = makeTestBlock(dag.headState.data, cache) tmpState = assignClone(dag.headState.data) test "Tail block only in common":