2021-12-06 09:49:01 +00:00
|
|
|
# beacon_chain
|
2022-06-07 17:05:06 +00:00
|
|
|
# Copyright (c) 2018-2022 Status Research & Development GmbH
|
2021-12-06 09:49:01 +00:00
|
|
|
# Licensed and distributed under either of
|
|
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
|
|
|
|
|
{.used.}
|
|
|
|
|
|
|
|
|
|
import
|
2022-06-07 17:05:06 +00:00
|
|
|
chronos,
|
2021-12-06 09:49:01 +00:00
|
|
|
std/[options, sequtils],
|
|
|
|
|
unittest2,
|
|
|
|
|
eth/keys, taskpools,
|
|
|
|
|
../beacon_chain/beacon_clock,
|
|
|
|
|
../beacon_chain/spec/[beaconstate, forks, helpers, state_transition],
|
2023-01-09 18:42:10 +00:00
|
|
|
../beacon_chain/spec/datatypes/eip4844,
|
2022-07-13 14:13:54 +00:00
|
|
|
../beacon_chain/gossip_processing/block_processor,
|
2021-12-06 09:49:01 +00:00
|
|
|
../beacon_chain/consensus_object_pools/[
|
2022-07-13 14:13:54 +00:00
|
|
|
attestation_pool, blockchain_dag, block_quarantine, block_clearance,
|
|
|
|
|
consensus_manager],
|
2022-03-25 11:40:10 +00:00
|
|
|
../beacon_chain/eth1/eth1_monitor,
|
2021-12-06 09:49:01 +00:00
|
|
|
./testutil, ./testdbutil, ./testblockutil
|
|
|
|
|
|
2022-10-14 19:48:56 +00:00
|
|
|
from chronos/unittest2/asynctests import asyncTest
|
2022-08-23 16:19:52 +00:00
|
|
|
from ../beacon_chain/spec/eth2_apis/dynamic_fee_recipients import
|
|
|
|
|
DynamicFeeRecipientsStore, init
|
2022-09-07 18:34:52 +00:00
|
|
|
from ../beacon_chain/validators/action_tracker import ActionTracker
|
2022-08-23 16:19:52 +00:00
|
|
|
from ../beacon_chain/validators/keystore_management import KeymanagerHost
|
|
|
|
|
|
2021-12-06 09:49:01 +00:00
|
|
|
proc pruneAtFinalization(dag: ChainDAGRef) =
|
|
|
|
|
if dag.needStateCachesAndForkChoicePruning():
|
|
|
|
|
dag.pruneStateCachesDAG()
|
|
|
|
|
|
|
|
|
|
suite "Block processor" & preset():
|
|
|
|
|
setup:
|
|
|
|
|
var
|
|
|
|
|
db = makeTestDB(SLOTS_PER_EPOCH)
|
2021-12-20 19:20:31 +00:00
|
|
|
validatorMonitor = newClone(ValidatorMonitor.init())
|
|
|
|
|
dag = init(ChainDAGRef, defaultRuntimeConfig, db, validatorMonitor, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
taskpool = Taskpool.new()
|
|
|
|
|
verifier = BatchVerifier(rng: keys.newRng(), taskpool: taskpool)
|
|
|
|
|
quarantine = newClone(Quarantine.init())
|
|
|
|
|
attestationPool = newClone(AttestationPool.init(dag, quarantine))
|
2022-03-25 11:40:10 +00:00
|
|
|
eth1Monitor = new Eth1Monitor
|
2022-09-07 18:34:52 +00:00
|
|
|
actionTracker: ActionTracker
|
2022-08-23 16:19:52 +00:00
|
|
|
keymanagerHost: ref KeymanagerHost
|
2022-03-25 11:40:10 +00:00
|
|
|
consensusManager = ConsensusManager.new(
|
2022-09-07 18:34:52 +00:00
|
|
|
dag, attestationPool, quarantine, eth1Monitor, actionTracker,
|
2022-09-17 05:30:07 +00:00
|
|
|
newClone(DynamicFeeRecipientsStore.init()), "",
|
2022-08-23 16:19:52 +00:00
|
|
|
default(Eth1Address))
|
2022-03-16 07:20:40 +00:00
|
|
|
state = newClone(dag.headState)
|
2021-12-06 09:49:01 +00:00
|
|
|
cache = StateCache()
|
|
|
|
|
b1 = addTestBlock(state[], cache).phase0Data
|
|
|
|
|
b2 = addTestBlock(state[], cache).phase0Data
|
2022-01-11 10:01:54 +00:00
|
|
|
getTimeFn = proc(): BeaconTime = b2.message.slot.start_beacon_time()
|
2021-12-06 09:49:01 +00:00
|
|
|
processor = BlockProcessor.new(
|
|
|
|
|
false, "", "", keys.newRng(), taskpool, consensusManager,
|
2022-09-29 06:29:49 +00:00
|
|
|
validatorMonitor, getTimeFn)
|
2021-12-06 09:49:01 +00:00
|
|
|
|
2022-10-14 19:48:56 +00:00
|
|
|
asyncTest "Reverse order block add & get" & preset():
|
|
|
|
|
let missing = await processor.storeBlock(
|
2023-01-09 18:42:10 +00:00
|
|
|
MsgSource.gossip, b2.message.slot.start_beacon_time(), b2, Opt.none(BlobsSidecar))
|
2022-11-10 17:40:27 +00:00
|
|
|
check: missing.error[0] == VerifierError.MissingParent
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
check:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
not dag.containsForkBlock(b2.root) # Unresolved, shouldn't show up
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
FetchRecord(root: b1.root) in quarantine[].checkMissing()
|
|
|
|
|
|
|
|
|
|
let
|
2022-10-14 19:48:56 +00:00
|
|
|
status = await processor.storeBlock(
|
2023-01-09 18:42:10 +00:00
|
|
|
MsgSource.gossip, b2.message.slot.start_beacon_time(), b1, Opt.none(BlobsSidecar))
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
b1Get = dag.getBlockRef(b1.root)
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
status.isOk
|
|
|
|
|
b1Get.isSome()
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
dag.containsForkBlock(b1.root)
|
|
|
|
|
not dag.containsForkBlock(b2.root) # Async pipeline must still run
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
discard processor.runQueueProcessingLoop()
|
|
|
|
|
while processor[].hasBlocks():
|
|
|
|
|
poll()
|
|
|
|
|
|
|
|
|
|
let
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
b2Get = dag.getBlockRef(b2.root)
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
b2Get.isSome()
|
|
|
|
|
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
b2Get.get().parent == b1Get.get()
|
2021-12-06 09:49:01 +00:00
|
|
|
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
dag.updateHead(b2Get.get(), quarantine[])
|
2021-12-06 09:49:01 +00:00
|
|
|
dag.pruneAtFinalization()
|
|
|
|
|
|
|
|
|
|
# The heads structure should have been updated to contain only the new
|
|
|
|
|
# b2 head
|
|
|
|
|
check:
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
dag.heads.mapIt(it) == @[b2Get.get()]
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
# check that init also reloads block graph
|
|
|
|
|
var
|
2021-12-20 19:20:31 +00:00
|
|
|
validatorMonitor2 = newClone(ValidatorMonitor.init())
|
|
|
|
|
dag2 = init(ChainDAGRef, defaultRuntimeConfig, db, validatorMonitor2, {})
|
2021-12-06 09:49:01 +00:00
|
|
|
|
|
|
|
|
check:
|
|
|
|
|
# ensure we loaded the correct head state
|
|
|
|
|
dag2.head.root == b2.root
|
2022-03-16 07:20:40 +00:00
|
|
|
getStateRoot(dag2.headState) == b2.message.state_root
|
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
|
|
|
dag2.getBlockRef(b1.root).isSome()
|
|
|
|
|
dag2.getBlockRef(b2.root).isSome()
|
2021-12-06 09:49:01 +00:00
|
|
|
dag2.heads.len == 1
|
|
|
|
|
dag2.heads[0].root == b2.root
|
