2023-01-20 14:14:37 +00:00
|
|
|
{.push raises: [].}
|
2021-03-26 06:52:01 +00:00
|
|
|
|
2018-11-23 23:58:49 +00:00
|
|
|
import
|
2024-03-10 02:19:13 +00:00
|
|
|
std/[os, times],
|
|
|
|
chronos,
|
|
|
|
stew/io2,
|
2022-01-18 13:36:52 +00:00
|
|
|
./spec/datatypes/[altair, bellatrix, phase0],
|
2024-03-10 02:19:13 +00:00
|
|
|
./spec/deposit_snapshots,
|
2023-08-23 16:39:57 +00:00
|
|
|
./validators/[keystore_management, beacon_validators],
|
2022-02-11 20:40:49 +00:00
|
|
|
"."/[
|
2024-03-10 02:19:13 +00:00
|
|
|
beacon_node,
|
|
|
|
nimbus_binary_common]
|
2021-06-03 09:43:04 +00:00
|
|
|
|
2023-03-09 00:34:17 +00:00
|
|
|
from ./spec/datatypes/deneb import SignedBeaconBlock
|
2023-01-04 12:34:15 +00:00
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
import libp2p/protocols/pubsub/gossipsub
|
History pruning (fixes #4419) (#4445)
Introduce (optional) pruning of historical data - a pruned node will
continue to answer queries for historical data up to
`MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs, or roughly 5 months, capping
typical database usage at around 60-70gb.
To enable pruning, add `--history=prune` to the command line - on the
first start, old data will be cleared (which may take a while) - after
that, data is pruned continuously.
When pruning an existing database, the database will not shrink -
instead, the freed space is recycled as the node continues to run - to
free up space, perform a trusted node sync with a fresh database.
When switching on archive mode in a pruned node, history is retained
from that point onwards.
History pruning is scheduled to be enabled by default in a future
release.
In this PR, `minimal` mode from #4419 is not implemented meaning
retention periods for states and blocks are always the same - depending
on user demand, a future PR may implement `minimal` as well.
2023-01-07 10:02:15 +00:00
|
|
|
|
2022-03-21 16:52:15 +00:00
|
|
|
proc initFullNode(
|
|
|
|
node: BeaconNode,
|
2024-01-13 10:53:53 +00:00
|
|
|
getBeaconTime: GetBeaconTimeFn) {.async.} =
|
2024-03-10 02:19:13 +00:00
|
|
|
node.router = new MessageRouter
|
2022-03-21 16:52:15 +00:00
|
|
|
|
2024-01-13 10:53:53 +00:00
|
|
|
await node.addValidators()
|
2022-03-21 16:52:15 +00:00
|
|
|
|
2024-03-18 04:31:20 +00:00
|
|
|
from "."/consensus_object_pools/blockchain_dag import preInit
|
|
|
|
from "."/consensus_object_pools/block_pools_types import ChainDAGRef
|
|
|
|
|
2021-12-22 12:37:31 +00:00
|
|
|
proc init*(T: type BeaconNode,
|
|
|
|
config: BeaconNodeConf,
|
2023-09-08 05:53:27 +00:00
|
|
|
metadata: Eth2NetworkMetadata): Future[BeaconNode]
|
2023-12-05 21:08:18 +00:00
|
|
|
{.async.} =
|
2022-12-07 10:24:51 +00:00
|
|
|
template cfg: auto = metadata.cfg
|
|
|
|
template eth1Network: auto = metadata.eth1Network
|
2022-01-21 10:59:09 +00:00
|
|
|
|
2020-01-17 13:44:01 +00:00
|
|
|
let
|
2023-01-09 18:42:10 +00:00
|
|
|
db = BeaconChainDB.new(config.databaseDir, cfg, inMemory = false)
|
2020-01-17 13:44:01 +00:00
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
let checkpointState = if config.finalizedCheckpointState.isSome:
|
2021-02-22 16:17:48 +00:00
|
|
|
let checkpointStatePath = config.finalizedCheckpointState.get.string
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
let tmp = try:
|
2021-11-10 11:39:08 +00:00
|
|
|
newClone(readSszForkedHashedBeaconState(
|
|
|
|
cfg, readAllBytes(checkpointStatePath).tryGet()))
|
|
|
|
except SszError as err:
|
2020-09-22 20:42:42 +00:00
|
|
|
quit 1
|
|
|
|
except CatchableError as err:
|
|
|
|
quit 1
|
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
if not getStateField(tmp[], slot).is_epoch:
|
2022-07-12 15:09:56 +00:00
|
|
|
quit 1
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
tmp
|
|
|
|
else:
|
|
|
|
nil
|
2020-01-17 13:44:01 +00:00
|
|
|
|
2023-02-23 02:10:07 +00:00
|
|
|
if config.finalizedDepositTreeSnapshot.isSome:
|
|
|
|
let
|
|
|
|
depositTreeSnapshotPath = config.finalizedDepositTreeSnapshot.get.string
|
|
|
|
depositTreeSnapshot = try:
|
|
|
|
SSZ.loadFile(depositTreeSnapshotPath, DepositTreeSnapshot)
|
|
|
|
except SszError as err:
|
|
|
|
quit 1
|
|
|
|
except CatchableError as err:
|
|
|
|
quit 1
|
2022-06-28 20:49:52 +00:00
|
|
|
|
2023-09-08 05:53:27 +00:00
|
|
|
var networkGenesisValidatorsRoot = metadata.bakedGenesisValidatorsRoot
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
var genesisState = checkpointState
|
|
|
|
if true:
|
|
|
|
genesisState = if checkpointState != nil and getStateField(checkpointState[], slot) == 0:
|
2023-09-08 10:09:21 +00:00
|
|
|
checkpointState
|
|
|
|
else:
|
|
|
|
let genesisBytes = block:
|
|
|
|
if metadata.genesis.kind != BakedIn and config.genesisState.isSome:
|
|
|
|
let res = io2.readAllBytes(config.genesisState.get.string)
|
|
|
|
res.valueOr:
|
|
|
|
quit 1
|
|
|
|
elif metadata.hasGenesis:
|
|
|
|
try:
|
2024-03-18 23:40:49 +00:00
|
|
|
metadata.fetchGenesisBytes()
|
2023-09-08 10:09:21 +00:00
|
|
|
except CatchableError as err:
|
|
|
|
quit 1
|
|
|
|
else:
|
|
|
|
@[]
|
|
|
|
|
|
|
|
if genesisBytes.len > 0:
|
2023-09-08 05:53:27 +00:00
|
|
|
try:
|
|
|
|
newClone readSszForkedHashedBeaconState(cfg, genesisBytes)
|
|
|
|
except CatchableError as err:
|
|
|
|
quit 1
|
|
|
|
else:
|
|
|
|
nil
|
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
if genesisState == nil and checkpointState == nil:
|
2023-01-12 00:49:43 +00:00
|
|
|
quit 1
|
2019-10-25 14:53:31 +00:00
|
|
|
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
if not genesisState.isNil and not checkpointState.isNil:
|
|
|
|
if getStateField(genesisState[], genesis_validators_root) !=
|
|
|
|
getStateField(checkpointState[], genesis_validators_root):
|
2020-04-22 23:35:55 +00:00
|
|
|
quit 1
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
|
|
|
|
try:
|
|
|
|
if not genesisState.isNil:
|
2023-09-08 05:53:27 +00:00
|
|
|
networkGenesisValidatorsRoot =
|
|
|
|
Opt.some(getStateField(genesisState[], genesis_validators_root))
|
2020-01-17 13:44:01 +00:00
|
|
|
|
2021-11-10 11:39:08 +00:00
|
|
|
if not checkpointState.isNil:
|
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
|
|
|
if genesisState.isNil or
|
|
|
|
getStateField(checkpointState[], slot) != GENESIS_SLOT:
|
2024-03-10 02:19:13 +00:00
|
|
|
ChainDAGRef.preInit(checkpointState[])
|
2021-03-26 06:52:01 +00:00
|
|
|
except CatchableError as exc:
|
2020-09-22 20:42:42 +00:00
|
|
|
quit 1
|
2021-11-10 11:39:08 +00:00
|
|
|
else:
|
|
|
|
if not checkpointState.isNil:
|
|
|
|
quit 1
|
2020-07-02 15:52:48 +00:00
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
doAssert not genesisState.isNil
|
2021-12-20 19:20:31 +00:00
|
|
|
|
2020-07-07 23:02:14 +00:00
|
|
|
let
|
2024-03-10 02:19:13 +00:00
|
|
|
genesisTime = getStateField(genesisState[], genesis_time)
|
2024-01-06 14:26:56 +00:00
|
|
|
beaconClock = BeaconClock.init(genesisTime).valueOr:
|
|
|
|
quit 1
|
|
|
|
|
2022-03-31 14:43:05 +00:00
|
|
|
getBeaconTime = beaconClock.getBeaconTimeFn()
|
2021-08-20 08:58:15 +00:00
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
let elManager = default(ELManager)
|
2023-06-14 06:46:01 +00:00
|
|
|
|
2021-05-04 13:17:28 +00:00
|
|
|
let
|
2023-09-21 18:07:02 +00:00
|
|
|
keystoreCache = KeystoreCacheRef.init()
|
2024-03-10 02:19:13 +00:00
|
|
|
validatorPool = new ValidatorPool
|
2023-02-23 09:46:18 +00:00
|
|
|
|
2022-06-07 17:01:11 +00:00
|
|
|
let node = BeaconNode(
|
2021-02-22 16:17:48 +00:00
|
|
|
config: config,
|
2021-11-01 14:50:24 +00:00
|
|
|
attachedValidators: validatorPool,
|
2023-03-05 01:40:21 +00:00
|
|
|
elManager: elManager,
|
2023-09-21 18:07:02 +00:00
|
|
|
keystoreCache: keystoreCache,
|
2021-09-17 00:13:52 +00:00
|
|
|
beaconClock: beaconClock,
|
2024-03-10 02:19:13 +00:00
|
|
|
cfg: cfg,
|
|
|
|
genesisState: genesisState)
|
2020-12-16 13:03:04 +00:00
|
|
|
|
2024-03-18 07:57:25 +00:00
|
|
|
await node.initFullNode(getBeaconTime)
|
2022-06-07 17:01:11 +00:00
|
|
|
|
2022-02-21 11:55:56 +00:00
|
|
|
node
|
2019-09-07 17:48:05 +00:00
|
|
|
|
2024-03-18 04:31:20 +00:00
|
|
|
from "."/consensus_object_pools/block_dag import BlockRef, init
|
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
func getBlockRef2(root: Eth2Digest): Opt[BlockRef] =
|
|
|
|
let newRef = BlockRef.init(
|
|
|
|
root, Opt.none Eth2Digest, executionValid = false,
|
|
|
|
0.Slot)
|
|
|
|
return ok(newRef)
|
2024-01-15 15:53:34 +00:00
|
|
|
|
2023-08-25 09:29:07 +00:00
|
|
|
proc start*(node: BeaconNode) {.raises: [CatchableError].} =
|
2024-03-10 02:19:13 +00:00
|
|
|
echo "foo"
|
2023-03-05 01:40:21 +00:00
|
|
|
node.elManager.start()
|
2021-05-19 06:38:13 +00:00
|
|
|
let
|
2024-03-10 02:19:13 +00:00
|
|
|
wallTime = node.beaconClock.now()
|
2021-02-22 16:17:48 +00:00
|
|
|
|
2024-03-19 00:42:52 +00:00
|
|
|
asyncSpawn runSlotLoop(node, wallTime)
|
2021-02-22 16:17:48 +00:00
|
|
|
|
2024-03-10 02:19:13 +00:00
|
|
|
while true:
|
|
|
|
poll()
|