nimbus-eth2/tests/testdbutil.nim

81 lines
3.3 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
chronicles,
../beacon_chain/[beacon_chain_db],
../beacon_chain/consensus_object_pools/blockchain_dag,
../beacon_chain/spec/datatypes/phase0,
../beacon_chain/spec/[beaconstate, forks],
eth/db/[kvstore, kvstore_sqlite3],
./testblockutil
export beacon_chain_db, testblockutil, kvstore, kvstore_sqlite3
proc makeTestDB*(
accelerate `getShufflingRef` (#4911) When an uncached `ShufflingRef` is requested, we currently replay state which can take several seconds. Acceleration is possible by: 1. Start from any state with locked-in `get_active_validator_indices`. Any blocks / slots applied to such a state can only affect that result for future epochs, so are viable for querying target epoch. `compute_activation_exit_epoch(state.slot.epoch) > target.epoch` 2. Determine highest common ancestor among `state` and `target.blck`. At the ancestor slot, same rules re `get_active_validator_indices`. `compute_activation_exit_epoch(ancestorSlot.epoch) > target.epoch` 3. We now have a `state` that shares history with `target.blck` up through a common ancestor slot. Any blocks / slots that the `state` contains, which are not part of the `target.blck` history, affect `get_active_validator_indices` at epochs _after_ `target.epoch`. 4. Select `state.randao_mixes[N]` that is closest to common ancestor. Either direction is fine (above / below ancestor). 5. From that RANDAO mix, mix in / out all RANDAO reveals from blocks in-between. This is just an XOR operation, so fully reversible. `mix = mix xor SHA256(blck.message.body.randao_reveal)` 6. Compute the attester dependent slot from `target.epoch`. `if epoch >= 2: (target.epoch - 1).start_slot - 1 else: GENESIS_SLOT` 7. Trace back from `target.blck` to the attester dependent slot. We now have the destination for which we want to obtain RANDAO. 8. Mix in all RANDAO reveals from blocks up through the `dependentBlck`. Same method, no special handling necessary for epoch transitions. 9. Combine `get_active_validator_indices` from `state` at `target.epoch` with the recovered RANDAO value at `dependentBlck` to obtain the requested shuffling, and construct the `ShufflingRef` without replay. * more tests and simplify logic * test with different number of deposits per branch * Update beacon_chain/consensus_object_pools/blockchain_dag.nim Co-authored-by: Jacek Sieka <jacek@status.im> * `commonAncestor` tests * lint --------- Co-authored-by: Jacek Sieka <jacek@status.im>
2023-05-12 17:36:59 +00:00
validators: Natural,
eth1Data = Opt.none(Eth1Data),
flags: UpdateFlags = {skipBlsValidation},
cfg = defaultRuntimeConfig): BeaconChainDB =
let
genState = (ref ForkedHashedBeaconState)(
kind: ConsensusFork.Phase0,
phase0Data: initialize_hashed_beacon_state_from_eth1(
cfg,
ZERO_HASH,
0,
accelerate `getShufflingRef` (#4911) When an uncached `ShufflingRef` is requested, we currently replay state which can take several seconds. Acceleration is possible by: 1. Start from any state with locked-in `get_active_validator_indices`. Any blocks / slots applied to such a state can only affect that result for future epochs, so are viable for querying target epoch. `compute_activation_exit_epoch(state.slot.epoch) > target.epoch` 2. Determine highest common ancestor among `state` and `target.blck`. At the ancestor slot, same rules re `get_active_validator_indices`. `compute_activation_exit_epoch(ancestorSlot.epoch) > target.epoch` 3. We now have a `state` that shares history with `target.blck` up through a common ancestor slot. Any blocks / slots that the `state` contains, which are not part of the `target.blck` history, affect `get_active_validator_indices` at epochs _after_ `target.epoch`. 4. Select `state.randao_mixes[N]` that is closest to common ancestor. Either direction is fine (above / below ancestor). 5. From that RANDAO mix, mix in / out all RANDAO reveals from blocks in-between. This is just an XOR operation, so fully reversible. `mix = mix xor SHA256(blck.message.body.randao_reveal)` 6. Compute the attester dependent slot from `target.epoch`. `if epoch >= 2: (target.epoch - 1).start_slot - 1 else: GENESIS_SLOT` 7. Trace back from `target.blck` to the attester dependent slot. We now have the destination for which we want to obtain RANDAO. 8. Mix in all RANDAO reveals from blocks up through the `dependentBlck`. Same method, no special handling necessary for epoch transitions. 9. Combine `get_active_validator_indices` from `state` at `target.epoch` with the recovered RANDAO value at `dependentBlck` to obtain the requested shuffling, and construct the `ShufflingRef` without replay. * more tests and simplify logic * test with different number of deposits per branch * Update beacon_chain/consensus_object_pools/blockchain_dag.nim Co-authored-by: Jacek Sieka <jacek@status.im> * `commonAncestor` tests * lint --------- Co-authored-by: Jacek Sieka <jacek@status.im>
2023-05-12 17:36:59 +00:00
makeInitialDeposits(validators.uint64, flags),
flags))
# Override Eth1Data on request, skipping the lengthy Eth1 voting process
if eth1Data.isOk:
withState(genState[]):
forkyState.data.eth1_data = eth1Data.get
forkyState.root = hash_tree_root(forkyState.data)
State-only checkpoint state startup (#4251) Currently, we require genesis and a checkpoint block and state to start from an arbitrary slot - this PR relaxes this requirement so that we can start with a state alone. The current trusted-node-sync algorithm works by first downloading blocks until we find an epoch aligned non-empty slot, then downloads the state via slot. However, current [proposals](https://github.com/ethereum/beacon-APIs/pull/226) for checkpointing prefer finalized state as the main reference - this allows more simple access control and caching on the server side - in particular, this should help checkpoint-syncing from sources that have a fast `finalized` state download (like infura and teku) but are slow when accessing state via slot. Earlier versions of Nimbus will not be able to read databases created without a checkpoint block and genesis. In most cases, backfilling makes the database compatible except where genesis is also missing (custom networks). * backfill checkpoint block from libp2p instead of checkpoint source, when doing trusted node sync * allow starting the client without genesis / checkpoint block * perform epoch start slot lookahead when loading tail state, so as to deal with the case where the epoch start slot does not have a block * replace `--blockId` with `--state-id` in TNS command line * when replaying, also look at the parent of the last-known-block (even if we don't have the parent block data, we can still replay from a "parent" state) - in particular, this clears the way for implementing state pruning * deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
result = BeaconChainDB.new("", cfg = cfg, inMemory = true)
State-only checkpoint state startup (#4251) Currently, we require genesis and a checkpoint block and state to start from an arbitrary slot - this PR relaxes this requirement so that we can start with a state alone. The current trusted-node-sync algorithm works by first downloading blocks until we find an epoch aligned non-empty slot, then downloads the state via slot. However, current [proposals](https://github.com/ethereum/beacon-APIs/pull/226) for checkpointing prefer finalized state as the main reference - this allows more simple access control and caching on the server side - in particular, this should help checkpoint-syncing from sources that have a fast `finalized` state download (like infura and teku) but are slow when accessing state via slot. Earlier versions of Nimbus will not be able to read databases created without a checkpoint block and genesis. In most cases, backfilling makes the database compatible except where genesis is also missing (custom networks). * backfill checkpoint block from libp2p instead of checkpoint source, when doing trusted node sync * allow starting the client without genesis / checkpoint block * perform epoch start slot lookahead when loading tail state, so as to deal with the case where the epoch start slot does not have a block * replace `--blockId` with `--state-id` in TNS command line * when replaying, also look at the parent of the last-known-block (even if we don't have the parent block data, we can still replay from a "parent" state) - in particular, this clears the way for implementing state pruning * deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
ChainDAGRef.preInit(result, genState[])
proc getEarliestInvalidBlockRoot*(
dag: ChainDAGRef, initialSearchRoot: Eth2Digest,
latestValidHash: Eth2Digest, defaultEarliestInvalidBlockRoot: Eth2Digest):
Eth2Digest =
# Earliest within a chain/fork in question, per LVH definition. Intended to
# be called with `initialRoot` as the parent of the block regarding which a
# newPayload or forkchoiceUpdated execution_status has been received as the
# tests effectively require being able to access this before the BlockRef's
# made. Therefore, to accommodate the EF consensus spec sync tests, and the
# possibilities that the LVH might be an immediate parent or a more distant
# ancestor special-case handling of an earliest invalid root as potentially
# not being from this function's search, but being provided as a default by
# the caller with access to the block.
var curBlck = dag.getBlockRef(initialSearchRoot).valueOr:
# Being asked to traverse a chain which the DAG doesn't know about -- but
# that'd imply the block's otherwise invalid for CL as well as EL.
return static(default(Eth2Digest))
# Only allow this special case outside loop; it's when the LVH is the direct
# parent of the reported invalid block
if curBlck.executionBlockHash.isSome and
curBlck.executionBlockHash.get == latestValidHash:
return defaultEarliestInvalidBlockRoot
while true:
# This was supposed to have been either caught by the pre-loop check or the
# parent check.
if curBlck.executionBlockHash.isSome and
curBlck.executionBlockHash.get == latestValidHash:
doAssert false, "getEarliestInvalidBlockRoot: unexpected LVH in loop body"
if (curBlck.parent.isNil) or
curBlck.parent.executionBlockHash.get(latestValidHash) ==
latestValidHash:
break
curBlck = curBlck.parent
curBlck.root