2024-01-06 14:26:56 +00:00
# Copyright (c) 2018-2024 Status Research & Development GmbH
2022-01-17 09:27:08 +00:00
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
2023-01-20 14:14:37 +00:00
{. push raises : [ ] . }
2022-01-17 09:27:08 +00:00
import
2022-12-07 10:24:51 +00:00
stew / [ base10 , results ] ,
chronicles , chronos , eth / async_utils ,
2023-06-12 12:22:32 +00:00
. / sync / [ light_client_sync_helpers , sync_manager ] ,
2022-11-10 10:44:47 +00:00
. / consensus_object_pools / [ block_clearance , blockchain_dag ] ,
2022-01-17 09:27:08 +00:00
. / spec / eth2_apis / rest_beacon_client ,
2023-04-17 17:18:54 +00:00
. / spec / [ beaconstate , eth2_merkleization , forks , light_client_sync ,
network , presets ,
2022-12-07 10:24:51 +00:00
state_transition , deposit_snapshots ] ,
2022-11-10 10:44:47 +00:00
" . " / [ beacon_clock , beacon_chain_db , era_db ]
2022-01-17 09:27:08 +00:00
2022-12-07 10:24:51 +00:00
from presto import RestDecodingError
const
largeRequestsTimeout = 60 . seconds # Downloading large items such as states.
smallRequestsTimeout = 30 . seconds # Downloading smaller items such as blocks and deposit snapshots.
proc fetchDepositSnapshot ( client : RestClientRef ) :
Future [ Result [ DepositTreeSnapshot , string ] ] {. async . } =
let resp = try :
awaitWithTimeout ( client . getDepositSnapshot ( ) , smallRequestsTimeout ) :
return err " Fetching /eth/v1/beacon/deposit_snapshot timed out "
except CatchableError as e :
return err ( " The trusted node likely does not support the /eth/v1/beacon/deposit_snapshot end-point: " & e . msg )
let data = resp . data . data
let snapshot = DepositTreeSnapshot (
eth1Block : data . execution_block_hash ,
depositContractState : DepositContractState (
branch : data . finalized ,
deposit_count : depositCountBytes ( data . deposit_count ) ) ,
blockHeight : data . execution_block_height )
if not snapshot . isValid ( data . deposit_root ) :
return err " The obtained deposit snapshot contains self-contradictory data "
return ok snapshot
2023-03-09 00:34:17 +00:00
from . / spec / datatypes / deneb import asSigVerified , shortLog
2022-12-14 17:30:56 +00:00
2023-04-16 06:07:07 +00:00
type
TrustedNodeSyncKind * {. pure . } = enum
TrustedBlockRoot ,
StateId
TrustedNodeSyncTarget * = object
case kind * : TrustedNodeSyncKind
of TrustedNodeSyncKind . TrustedBlockRoot :
trustedBlockRoot * : Eth2Digest
of TrustedNodeSyncKind . StateId :
stateId * : string
func shortLog * ( v : TrustedNodeSyncTarget ) : auto =
case v . kind
of TrustedNodeSyncKind . TrustedBlockRoot :
" trustedBlockRoot( " & $ v . trustedBlockRoot & " ) "
of TrustedNodeSyncKind . StateId :
v . stateId
chronicles . formatIt ( TrustedNodeSyncTarget ) : shortLog ( it )
2022-01-17 09:27:08 +00:00
proc doTrustedNodeSync * (
2023-10-31 00:56:52 +00:00
db : BeaconChainDB ,
2022-12-07 10:24:51 +00:00
cfg : RuntimeConfig ,
databaseDir : string ,
eraDir : string ,
restUrl : string ,
2023-04-16 06:07:07 +00:00
syncTarget : TrustedNodeSyncTarget ,
2022-12-07 10:24:51 +00:00
backfill : bool ,
reindex : bool ,
downloadDepositSnapshot : bool ,
2022-01-17 09:27:08 +00:00
genesisState : ref ForkedHashedBeaconState = nil ) {. async . } =
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
logScope :
restUrl
2023-04-16 06:07:07 +00:00
syncTarget
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
2022-01-17 09:27:08 +00:00
notice " Starting trusted node sync " ,
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
databaseDir , backfill , reindex
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
var
client = RestClientRef . new ( restUrl ) . valueOr :
error " Cannot connect to server " , error = error
quit 1
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
# If possible, we'll store the genesis state in the database - this is not
# strictly necessary but renders the resulting database compatible with
# versions prior to 22.11 and makes reindexing possible
let genesisState =
if ( let genesisRoot = db . getGenesisBlock ( ) ; genesisRoot . isSome ( ) ) :
let
genesisBlock = db . getForkedBlock ( genesisRoot . get ( ) ) . valueOr :
error " Cannot load genesis block from database " ,
genesisRoot = genesisRoot . get ( )
2022-03-11 12:49:47 +00:00
quit 1
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
genesisStateRoot = getForkedBlockField ( genesisBlock , state_root )
2023-03-11 00:35:52 +00:00
consensusFork = cfg . consensusForkAtEpoch ( GENESIS_EPOCH )
2022-01-17 09:27:08 +00:00
2023-03-11 00:35:52 +00:00
tmp = ( ref ForkedHashedBeaconState ) ( kind : consensusFork )
if not db . getState ( consensusFork , genesisStateRoot , tmp [ ] , noRollback ) :
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
error " Cannot load genesis state from database " ,
genesisStateRoot
quit 1
if ( genesisState ! = nil ) and
( getStateRoot ( tmp [ ] ) ! = getStateRoot ( genesisState [ ] ) ) :
error " Unexpected genesis state in database, is this the same network? " ,
databaseRoot = getStateRoot ( tmp [ ] ) ,
genesisRoot = getStateRoot ( genesisState [ ] )
quit 1
tmp
else :
let tmp = if genesisState ! = nil :
genesisState
else :
2023-06-09 20:11:14 +00:00
case syncTarget . kind
of TrustedNodeSyncKind . TrustedBlockRoot :
error " Genesis state is required when using `trustedBlockRoot` " ,
missingNetworkMetadataFile = " genesis.ssz "
# `genesis_time` and `genesis_validators_root` are required to check
# light client data signatures. They are not part of `config.yaml`.
# We could download the initial state based on `LightClientBootstrap`
# `state_root`, but that state is unlikely to be readily available and
# can be much larger than the genesis state. Furthermore, the major
# known networks bundle the `genesis.ssz` file with network metadata,
# so adding this complexity doesn't solve a practical usecase. Users
# on private networks may obtain a trusted `genesis.ssz` file and mark
# it as trusted by moving it into the network metadata folder.
#
# Note that `historical_roots` / `historical_summaries` may be used to
# prove correctness of a particular genesis state. However, there is
# currently no endpoint to obtain proofs, and they change for every
# slot, making it tricky to actually provide them.
quit 1
of TrustedNodeSyncKind . StateId :
notice " Downloading genesis state " , restUrl
try :
awaitWithTimeout (
client . getStateV2 ( StateIdent . init ( StateIdentType . Genesis ) , cfg ) ,
largeRequestsTimeout ) :
info " Attempt to download genesis state timed out "
2023-06-28 17:21:43 +00:00
# https://github.com/nim-lang/Nim/issues/22180
( ref ForkedHashedBeaconState ) ( nil )
2023-06-09 20:11:14 +00:00
except CatchableError as exc :
info " Unable to download genesis state " ,
error = exc . msg , restUrl
2022-12-07 10:24:51 +00:00
nil
2022-01-17 09:27:08 +00:00
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if isNil ( tmp ) :
notice " Server is missing genesis state, node will not be able to reindex history " ,
restUrl
tmp
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
let
dbHead = db . getHeadBlock ( )
head = if dbHead . isSome ( ) :
let
bid = db . getBlockId ( dbHead . get ( ) ) . valueOr :
error " Database missing head block summary - database too old or corrupt " ,
headRoot = dbHead . get ( )
quit 1
Opt . some bid
else :
# When we don't have a head, we'll use the given checkpoint as head
Opt . none ( BlockId )
if head . isNone :
2023-04-16 06:07:07 +00:00
var stateRoot : Opt [ Eth2Digest ]
let stateId =
case syncTarget . kind
of TrustedNodeSyncKind . TrustedBlockRoot :
2024-01-20 11:19:47 +00:00
# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.6/specs/altair/light-client/light-client.md#light-client-sync-process
2023-04-16 06:07:07 +00:00
const lcDataFork = LightClientDataFork . high
var bestViableCheckpoint : Opt [ tuple [ slot : Slot , state_root : Eth2Digest ] ]
func trackBestViableCheckpoint ( store : lcDataFork . LightClientStore ) =
if store . finalized_header . beacon . slot . is_epoch :
bestViableCheckpoint . ok ( (
slot : store . finalized_header . beacon . slot ,
state_root : store . finalized_header . beacon . state_root ) )
2023-06-09 20:11:14 +00:00
doAssert genesisState ! = nil , " Already checked for `TrustedBlockRoot` "
2023-04-16 06:07:07 +00:00
let
2024-01-06 14:26:56 +00:00
genesisTime = getStateField ( genesisState [ ] , genesis_time )
beaconClock = BeaconClock . init ( genesisTime ) . valueOr :
error " Invalid genesis time in state " , genesisTime
quit 1
2023-04-16 06:07:07 +00:00
getBeaconTime = beaconClock . getBeaconTimeFn ( )
genesis_validators_root =
getStateField ( genesisState [ ] , genesis_validators_root )
forkDigests = newClone ForkDigests . init ( cfg , genesis_validators_root )
trustedBlockRoot = syncTarget . trustedBlockRoot
var bootstrap =
try :
notice " Downloading LC bootstrap " , trustedBlockRoot
awaitWithTimeout (
client . getLightClientBootstrap (
trustedBlockRoot , cfg , forkDigests ) ,
smallRequestsTimeout
) :
error " Attempt to download LC bootstrap timed out "
quit 1
except CatchableError as exc :
error " Unable to download LC bootstrap " , error = exc . msg
quit 1
if bootstrap . kind = = LightClientDataFork . None :
error " LC bootstrap unavailable on server "
quit 1
bootstrap . migrateToDataFork ( lcDataFork )
2023-10-04 16:11:45 +00:00
var storeRes = initialize_light_client_store (
trustedBlockRoot , bootstrap . forky ( lcDataFork ) , cfg )
2023-04-16 06:07:07 +00:00
if storeRes . isErr :
error " `initialize_light_client_store` failed " , err = storeRes . error
quit 1
template store : auto = storeRes . get
store . trackBestViableCheckpoint ( )
while true :
let
finalized =
store . finalized_header . beacon . slot . sync_committee_period
optimistic =
store . optimistic_header . beacon . slot . sync_committee_period
current =
getBeaconTime ( ) . slotOrZero ( ) . sync_committee_period
isNextSyncCommitteeKnown =
store . is_next_sync_committee_known
let
periods : Slice [ SyncCommitteePeriod ] =
if finalized = = optimistic and not isNextSyncCommitteeKnown :
if finalized > = current :
finalized .. finalized
else :
finalized .. < current
elif finalized + 1 < current :
finalized + 1 .. < current
else :
break
startPeriod = periods . a
count = min ( periods . len , MAX_REQUEST_LIGHT_CLIENT_UPDATES ) . uint64
var updates =
try :
notice " Downloading LC updates " , startPeriod , count
awaitWithTimeout (
client . getLightClientUpdatesByRange (
startPeriod , count , cfg , forkDigests ) ,
smallRequestsTimeout
) :
error " Attempt to download LC updates timed out "
quit 1
except CatchableError as exc :
error " Unable to download LC updates " , error = exc . msg
quit 1
2023-06-12 12:22:32 +00:00
let e = updates . checkLightClientUpdates ( startPeriod , count )
if e . isErr :
error " Malformed LC updates response " , resError = e . error
2023-04-16 06:07:07 +00:00
quit 1
if updates . len = = 0 :
warn " Server does not appear to be fully synced "
break
for i in 0 .. < updates . len :
doAssert updates [ i ] . kind > LightClientDataFork . None
updates [ i ] . migrateToDataFork ( lcDataFork )
let res = process_light_client_update (
store , updates [ i ] . forky ( lcDataFork ) ,
getBeaconTime ( ) . slotOrZero ( ) , cfg , genesis_validators_root )
if not res . isOk :
error " `process_light_client_update` failed " , resError = res . error
quit 1
store . trackBestViableCheckpoint ( )
var finalityUpdate =
try :
notice " Downloading LC finality update "
awaitWithTimeout (
client . getLightClientFinalityUpdate ( cfg , forkDigests ) ,
smallRequestsTimeout
) :
error " Attempt to download LC finality update timed out "
quit 1
except CatchableError as exc :
error " Unable to download LC finality update " , error = exc . msg
quit 1
if bootstrap . kind = = LightClientDataFork . None :
error " LC finality update unavailable on server "
quit 1
finalityUpdate . migrateToDataFork ( lcDataFork )
let res = process_light_client_update (
store , finalityUpdate . forky ( lcDataFork ) ,
getBeaconTime ( ) . slotOrZero ( ) , cfg , genesis_validators_root )
if not res . isOk :
error " `process_light_client_update` failed " , resError = res . error
quit 1
store . trackBestViableCheckpoint ( )
if bestViableCheckpoint . isErr :
error " CP not on epoch boundary. Retry later " ,
latestCheckpointSlot = store . finalized_header . beacon . slot
quit 1
if not store . finalized_header . beacon . slot . is_epoch :
warn " CP not on epoch boundary. Using older one " ,
latestCheckpointSlot = store . finalized_header . beacon . slot ,
bestViableCheckpointSlot = bestViableCheckpoint . get . slot
stateRoot . ok bestViableCheckpoint . get . state_root
Base10 . toString ( distinctBase ( bestViableCheckpoint . get . slot ) )
of TrustedNodeSyncKind . StateId :
syncTarget . stateId
logScope : stateId
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
notice " Downloading checkpoint state "
2022-01-17 09:27:08 +00:00
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
let
state = try :
let id = block :
let tmp = StateIdent . decodeString ( stateId ) . valueOr :
error " Cannot decode checkpoint state id, must be a slot, hash, ' finalized ' or ' head ' "
quit 1
if tmp . kind = = StateQueryKind . Slot and not tmp . slot . is_epoch ( ) :
notice " Rounding given slot to epoch "
StateIdent . init ( tmp . slot . epoch ( ) . start_slot )
else :
tmp
2022-12-07 10:24:51 +00:00
awaitWithTimeout ( client . getStateV2 ( id , cfg ) , largeRequestsTimeout ) :
error " Attempt to download checkpoint state timed out "
quit 1
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
except CatchableError as exc :
error " Unable to download checkpoint state " ,
error = exc . msg
2022-03-11 12:49:47 +00:00
quit 1
2022-01-17 09:27:08 +00:00
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if state = = nil :
2023-04-16 06:07:07 +00:00
error " No state found a given checkpoint "
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
quit 1
2022-01-17 09:27:08 +00:00
2023-04-16 06:07:07 +00:00
if stateRoot . isSome :
if state [ ] . getStateRoot ( ) ! = stateRoot . get :
error " Checkpoint state has incorrect root! " ,
expectedStateRoot = stateRoot . get ,
actualStateRoot = state [ ] . getStateRoot ( )
quit 1
info " Checkpoint state validated against LC data " ,
stateRoot = stateRoot . get
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if not getStateField ( state [ ] , slot ) . is_epoch ( ) :
error " State slot must fall on an epoch boundary " ,
slot = getStateField ( state [ ] , slot ) ,
offset = getStateField ( state [ ] , slot ) -
getStateField ( state [ ] , slot ) . epoch . start_slot
quit 1
2022-01-17 09:27:08 +00:00
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if genesisState ! = nil :
2023-06-09 20:11:14 +00:00
if getStateField ( state [ ] , genesis_time ) ! =
getStateField ( genesisState [ ] , genesis_time ) :
error " Checkpoint state does not match genesis " ,
timeInCheckpoint = getStateField ( state [ ] , genesis_time ) ,
timeInGenesis = getStateField ( genesisState [ ] , genesis_time )
quit 1
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if getStateField ( state [ ] , genesis_validators_root ) ! =
getStateField ( genesisState [ ] , genesis_validators_root ) :
error " Checkpoint state does not match genesis " ,
rootInCheckpoint = getStateField ( state [ ] , genesis_validators_root ) ,
rootInGenesis = getStateField ( genesisState [ ] , genesis_validators_root )
2022-01-17 09:27:08 +00:00
quit 1
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
ChainDAGRef . preInit ( db , genesisState [ ] )
2022-01-17 09:27:08 +00:00
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
if getStateField ( genesisState [ ] , slot ) ! = getStateField ( state [ ] , slot ) :
ChainDAGRef . preInit ( db , state [ ] )
else :
ChainDAGRef . preInit ( db , state [ ] )
2022-12-07 10:24:51 +00:00
if downloadDepositSnapshot :
# Fetch deposit snapshot. This API endpoint is still optional.
let depositSnapshot = await fetchDepositSnapshot ( client )
if depositSnapshot . isOk :
if depositSnapshot . get . matches ( getStateField ( state [ ] , eth1_data ) ) :
info " Writing deposit contracts snapshot " ,
depositRoot = depositSnapshot . get . getDepositRoot ( ) ,
depositCount = depositSnapshot . get . getDepositCountU64
db . putDepositTreeSnapshot ( depositSnapshot . get )
else :
warn " The downloaded deposit snapshot does not agree with the downloaded state "
else :
warn " Deposit tree snapshot was not imported " , reason = depositSnapshot . error
2022-03-11 12:49:47 +00:00
else :
2022-09-29 20:55:18 +00:00
notice " Skipping checkpoint download, database already exists (remove db directory to get a fresh snapshot) " ,
2022-11-10 10:44:47 +00:00
databaseDir , head = shortLog ( head . get ( ) )
2022-01-17 09:27:08 +00:00
# Coming this far, we've done what ChainDAGRef.preInit would normally do -
2022-11-10 10:44:47 +00:00
# we can now load a ChainDAG to start backfilling it
State-only checkpoint state startup (#4251)
Currently, we require genesis and a checkpoint block and state to start
from an arbitrary slot - this PR relaxes this requirement so that we can
start with a state alone.
The current trusted-node-sync algorithm works by first downloading
blocks until we find an epoch aligned non-empty slot, then downloads the
state via slot.
However, current
[proposals](https://github.com/ethereum/beacon-APIs/pull/226) for
checkpointing prefer finalized state as
the main reference - this allows more simple access control and caching
on the server side - in particular, this should help checkpoint-syncing
from sources that have a fast `finalized` state download (like infura
and teku) but are slow when accessing state via slot.
Earlier versions of Nimbus will not be able to read databases created
without a checkpoint block and genesis. In most cases, backfilling makes
the database compatible except where genesis is also missing (custom
networks).
* backfill checkpoint block from libp2p instead of checkpoint source,
when doing trusted node sync
* allow starting the client without genesis / checkpoint block
* perform epoch start slot lookahead when loading tail state, so as to
deal with the case where the epoch start slot does not have a block
* replace `--blockId` with `--state-id` in TNS command line
* when replaying, also look at the parent of the last-known-block (even
if we don't have the parent block data, we can still replay from a
"parent" state) - in particular, this clears the way for implementing
state pruning
* deprecate `--finalized-checkpoint-block` option (no longer needed)
2022-11-02 10:02:38 +00:00
let
2022-11-10 10:44:47 +00:00
validatorMonitor = newClone ( ValidatorMonitor . init ( false , false ) )
dag = ChainDAGRef . init ( cfg , db , validatorMonitor , { } , eraPath = eraDir )
fix checkpoint block potentially not getting backfilled into DB (#5863)
When using checkpoint sync, only checkpoint state is available, block is
not downloaded and backfilled later.
`dag.backfill` tracks latest filled `slot`, and latest `parent_root` for
which no block has been synced yet.
In checkpoint sync, this assumption is broken, because there, the start
`dag.backfill.slot` is set based on checkpoint state slot, and the block
is also not available.
However, sync manager in backward mode also requests `dag.backfill.slot`
and `block_clearance` then backfills the checkpoint block once it is
synced. But, there is no guarantee that a peer ever sends us that block.
They could send us all parent blocks and solely omit the checkpoint
block itself. In that situation, we would accept the parent blocks and
advance `dag.backfill`, and subsequently never request the checkpoint
block again, resulting in gap inside blocks DB that is never filled.
To mitigate that, the assumption is restored that `dag.backfill.slot`
is the latest filled `slot`, and `dag.backfill.parent_root` is the next
block that needs to be synced. By setting `slot` to `tail.slot + 1` and
`parent_root` to `tail.root`, we put a fake summary into `dag.backfill`
so that `block_clearance` only proceeds once checkpoint block exists.
2024-02-09 10:20:36 +00:00
backfillSlot = max ( dag . backfill . slot , 1 . Slot ) - 1
2022-12-23 07:42:55 +00:00
horizon = max ( dag . horizon , dag . frontfill . valueOr ( BlockId ( ) ) . slot )
2022-11-10 10:44:47 +00:00
2022-12-23 07:42:55 +00:00
let canReindex = if backfillSlot < = horizon :
info " Database backfilled " , backfill = dag . backfill , horizon
2022-03-11 12:49:47 +00:00
true
2022-01-17 09:27:08 +00:00
elif backfill :
2022-11-10 10:44:47 +00:00
# +1 because we need to download the frontfill slot for the frontfill match
# detection to kick in, in addBackfillBlock
2022-12-23 07:42:55 +00:00
let missingSlots = dag . backfill . slot - horizon + 1
2022-11-10 10:44:47 +00:00
fix checkpoint block potentially not getting backfilled into DB (#5863)
When using checkpoint sync, only checkpoint state is available, block is
not downloaded and backfilled later.
`dag.backfill` tracks latest filled `slot`, and latest `parent_root` for
which no block has been synced yet.
In checkpoint sync, this assumption is broken, because there, the start
`dag.backfill.slot` is set based on checkpoint state slot, and the block
is also not available.
However, sync manager in backward mode also requests `dag.backfill.slot`
and `block_clearance` then backfills the checkpoint block once it is
synced. But, there is no guarantee that a peer ever sends us that block.
They could send us all parent blocks and solely omit the checkpoint
block itself. In that situation, we would accept the parent blocks and
advance `dag.backfill`, and subsequently never request the checkpoint
block again, resulting in gap inside blocks DB that is never filled.
To mitigate that, the assumption is restored that `dag.backfill.slot`
is the latest filled `slot`, and `dag.backfill.parent_root` is the next
block that needs to be synced. By setting `slot` to `tail.slot + 1` and
`parent_root` to `tail.root`, we put a fake summary into `dag.backfill`
so that `block_clearance` only proceeds once checkpoint block exists.
2024-02-09 10:20:36 +00:00
notice " Downloading historical blocks - you can interrupt this process at any time and it will automatically be completed when you start the beacon node " ,
2022-12-23 07:42:55 +00:00
backfillSlot , horizon , missingSlots
2022-01-17 09:27:08 +00:00
var # Same averaging as SyncManager
syncCount = 0
processed = 0 'u64
avgSyncSpeed = 0 .0
stamp = SyncMoment . now ( 0 )
2022-11-10 10:44:47 +00:00
proc downloadBlock ( slot : Slot ) :
Future [ Option [ ref ForkedSignedBeaconBlock ] ] {. async . } =
# Download block at given slot, retrying a few times,
var lastError : ref CatchableError
for i in 0 .. < 3 :
try :
2022-12-07 10:24:51 +00:00
return awaitWithTimeout ( client . getBlockV2 ( BlockIdent . init ( slot ) , cfg ) ,
smallRequestsTimeout ) :
raise newException ( CatchableError , " Request timed out " )
2022-11-10 10:44:47 +00:00
except RestResponseError as exc :
lastError = exc
notice " Server does not support block downloads / backfilling - blocks will be downloaded later " ,
msg = exc . msg
break
except CatchableError as exc :
# We'll assume this may be a connectivity error or something similar
lastError = exc
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
warn " Retrying download of block " , slot , err = exc . msg
client = RestClientRef . new ( restUrl ) . valueOr :
error " Cannot connect to server " , url = restUrl , error = error
2022-01-17 09:27:08 +00:00
quit 1
2022-11-10 10:44:47 +00:00
raise lastError
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
# Download several blocks in parallel but process them serially
proc processBlock ( blck : Option [ ref ForkedSignedBeaconBlock ] ) =
let newStamp = SyncMoment . now ( processed )
if newStamp . stamp - stamp . stamp > 12 . seconds :
syncCount + = 1
let
2022-12-23 07:42:55 +00:00
remaining = dag . backfill . slot - horizon
2022-11-10 10:44:47 +00:00
slotsPerSec = speed ( stamp , newStamp )
avgSyncSpeed = avgSyncSpeed + ( slotsPerSec - avgSyncSpeed ) / float ( syncCount )
info " Backfilling " ,
timeleft = toTimeLeftString (
if avgSyncSpeed > = 0 .001 :
Duration . fromFloatSeconds ( remaining . float / avgSyncSpeed )
else : InfiniteDuration ) ,
slotsPerSecond = avgSyncSpeed ,
remainingSlots = remaining
stamp = newStamp
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
processed + = 1
if blck . isSome ( ) :
let
data = blck . get ( )
withBlck ( data [ ] ) :
2023-04-26 11:37:27 +00:00
let res =
case syncTarget . kind
of TrustedNodeSyncKind . TrustedBlockRoot :
# Trust-minimized sync: the server is only trusted for
# data availability, responses must be verified
2023-09-21 10:49:14 +00:00
dag . addBackfillBlock ( forkyBlck )
2023-04-26 11:37:27 +00:00
of TrustedNodeSyncKind . StateId :
# The server is fully trusted to provide accurate data;
# it could have provided a malicious state
2023-09-21 10:49:14 +00:00
dag . addBackfillBlock ( forkyBlck . asSigVerified ( ) )
2023-04-26 11:37:27 +00:00
if res . isErr ( ) :
2022-11-10 10:44:47 +00:00
case res . error ( )
2022-11-10 17:40:27 +00:00
of VerifierError . Invalid ,
VerifierError . MissingParent ,
VerifierError . UnviableFork :
2022-11-10 10:44:47 +00:00
error " Got invalid block from trusted node - is it on the right network? " ,
2023-09-21 10:49:14 +00:00
blck = shortLog ( forkyBlck ) , err = res . error ( )
2022-11-10 10:44:47 +00:00
quit 1
2022-11-10 17:40:27 +00:00
of VerifierError . Duplicate :
2022-11-10 10:44:47 +00:00
discard
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
# Download blocks backwards from the backfill slot, ie the first slot for
# which we don't have a block, when walking backwards from the head
try :
var
gets : array [ 16 , Future [ Option [ ref ForkedSignedBeaconBlock ] ] ]
2022-01-17 09:27:08 +00:00
2022-11-10 10:44:47 +00:00
for i in 0 . uint64 .. missingSlots + gets . lenu64 :
if i > = gets . lenu64 ( ) :
2022-01-17 09:27:08 +00:00
let
2022-11-10 10:44:47 +00:00
fut = gets [ int ( i mod gets . lenu64 ) ]
processBlock ( await fut )
2022-09-29 20:55:18 +00:00
2022-11-10 10:44:47 +00:00
if i < = backfillSlot :
let slot = backfillSlot - i
2022-09-29 20:55:18 +00:00
gets [ int ( i mod gets . lenu64 ) ] = downloadBlock ( slot )
if i mod 1024 = = 0 :
db . checkpoint ( ) # Transfer stuff from wal periodically
true
except CatchableError as exc : # Block download failed
notice " Backfilling incomplete - blocks will be downloaded when starting the node " , msg = exc . msg
false
2022-01-17 09:27:08 +00:00
else :
2022-12-23 07:42:55 +00:00
let missingSlots = dag . backfill . slot - horizon
2022-01-17 09:27:08 +00:00
notice " Database initialized, historical blocks will be backfilled when starting the node " ,
2022-12-23 07:42:55 +00:00
missingSlots , backfill = dag . backfill , horizon
2022-01-17 09:27:08 +00:00
2022-03-11 12:49:47 +00:00
false
if reindex and canReindex :
notice " Reindexing historical state lookup tables (you can interrupt this process at any time) "
# Build a DAG
dag . rebuildIndex ( )
2022-08-01 00:32:08 +00:00
notice " Done, your beacon node is ready to serve you! Don ' t forget to check that you ' re on the canonical chain by comparing the checkpoint root with other online sources. See https://nimbus.guide/trusted-node-sync.html for more information. " ,
2022-11-10 10:44:47 +00:00
checkpoint = dag . head
2022-01-17 09:27:08 +00:00
when isMainModule :
2022-08-18 16:20:50 +00:00
import
2024-02-07 19:26:29 +00:00
std / os ,
2022-08-18 16:20:50 +00:00
networking / network_metadata
limit by-root requests to non-finalized blocks (#3293)
* limit by-root requests to non-finalized blocks
Presently, we keep a mapping from block root to `BlockRef` in memory -
this has simplified reasoning about the dag, but is not sustainable with
the chain growing.
We can distinguish between two cases where by-root access is useful:
* unfinalized blocks - this is where the beacon chain is operating
generally, by validating incoming data as interesting for future fork
choice decisions - bounded by the length of the unfinalized period
* finalized blocks - historical access in the REST API etc - no bounds,
really
In this PR, we limit the by-root block index to the first use case:
finalized chain data can more efficiently be addressed by slot number.
Future work includes:
* limiting the `BlockRef` horizon in general - each instance is 40
bytes+overhead which adds up - this needs further refactoring to deal
with the tail vs state problem
* persisting the finalized slot-to-hash index - this one also keeps
growing unbounded (albeit slowly)
Anyway, this PR easily shaves ~128mb of memory usage at the time of
writing.
* No longer honor `BeaconBlocksByRoot` requests outside of the
non-finalized period - previously, Nimbus would generously return any
block through this libp2p request - per the spec, finalized blocks
should be fetched via `BeaconBlocksByRange` instead.
* return `Opt[BlockRef]` instead of `nil` when blocks can't be found -
this becomes a lot more common now and thus deserves more attention
* `dag.blocks` -> `dag.forkBlocks` - this index only carries unfinalized
blocks from now - `finalizedBlocks` covers the other `BlockRef`
instances
* in backfill, verify that the last backfilled block leads back to
genesis, or panic
* add backfill timings to log
* fix missing check that `BlockRef` block can be fetched with
`getForkedBlock` reliably
* shortcut doppelganger check when feature is not enabled
* in REST/JSON-RPC, fetch blocks without involving `BlockRef`
* fix dag.blocks ref
2022-01-21 11:33:16 +00:00
2023-04-16 06:07:07 +00:00
let
2024-02-07 19:26:29 +00:00
cfg = getRuntimeConfig ( some os . paramStr ( 1 ) )
databaseDir = os . paramStr ( 2 )
2023-04-16 06:07:07 +00:00
syncTarget = TrustedNodeSyncTarget (
kind : TrustedNodeSyncKind . StateId ,
stateId : os . paramStr ( 5 ) )
backfill = os . paramCount ( ) > 5 and os . paramStr ( 6 ) = = " true "
2023-10-31 00:56:52 +00:00
db = BeaconChainDB . new ( databaseDir , cfg , inMemory = false )
waitFor db . doTrustedNodeSync (
2024-02-07 19:26:29 +00:00
cfg , databaseDir , os . paramStr ( 3 ) ,
2023-04-16 06:07:07 +00:00
os . paramStr ( 4 ) , syncTarget , backfill , false , true )
2023-10-31 00:56:52 +00:00
db . close ( )
