nimbus-eth2/beacon_chain/beacon_node.nim
Eugene Kabanov 18409a69e1
Light forward sync mechanism (#6515)
* Initial commit.

* Add hybrid syncing.

* Compilation fixes.

* Cast custom event for our purposes.

* Instantiate AsyncEventQueue properly.

* Fix mistype.

* Further research on optimistic updates.

* Fixing circular deps.

* Add backfilling.

* Add block download feature.

* Add block store.

* Update backfill information before storing block.

* Use custom block verifier for backfilling sync.

* Skip signature verification in backfilling.

* Add one more generic reload to storeBackfillBlock().

* Add block verification debugging statements.

* Add more debugging

* Do not use database for backfilling, part 1.

* Fix for stash.

* Stash fixes part 2.

* Prepare for testing.

* Fix assertion.

* Fix post-restart syncing process.

* Update backfill loading log statement.
Use proper backfill slot callback for sync manager.

* Add handling of Duplicates.

* Fix store duration and block backfilled log statements.

* Add proper syncing state log statement.

* Add snappy compression to beaconchain_file.
Format syncing speed properly.

* Add blobs verification.

* Add `slot` number to file structure for easy navigation over stream of compressed objects.

* Change database filename.

* Fix structure size.

* Add more consistency properties.

* Fix checkRepair() issues.

* Preparation to state rebuild process.

* Add plain & compressed size.

* Debugging snappy encode process.

* Add one more debugging line.

* Dump blocks.

* One more filedump.

* Fix chunk corruption code.

* Fix detection issue.

* Some fixes in state rebuilding process.

* Add more clearance steps.

* Move updateHead() back to block_processor.

* Fix compilation issues.

* Make code more async friendly.

* Fix async issues.
Add more information when proposer verification failed.

* Fix 8192 slots issue.

* Fix Future double completion issue.

* Pass updateFlags to some of the core procedures.

* Fix tests.

* Improve initial sync handling mechanism.

* Fix checkStateTransition() performance improvements.

* Add some performance tuning and meters.

* Light client performance tuning.

* Remove debugging statement.

* Use single file descriptor for blockchain file.

* Attempt to fix LC.

* Fix timeleft calculation when untrusted sync backfilling started right after LC block received.

* Workaround for `chronicles` + `results` `error` issue.
Remove some compilation warnings.
Fix `CatchableError` leaks on Windows.

* Address review comments.

* Address review comments part 2.

* Address review comments part 1.

* Rebase and fix the issues.

* Address review comments part 3.

* Add tests and fix some issues in auto-repair mechanism.

* Add tests to all_tests.

* Rename binary test file to pass restrictions.

* Add `bin` extension to excluded list.
Recover binary test data.

* Rename fixture file to .bin again.

* Update AllTests.

* Address review comments part 4.

* Address review comments part 5 and fix tests.

* Address review comments part 6.

* Eliminate foldl and combine from blobs processing.
Add some tests to ensure that checkResponse() also checks for correct order.

* Fix forgotten place.

* Post rebase fixes.

* Add unique slots tests.

* Optimize updateHead() code.

* Add forgotten changes.

* Address review comments on state as argument.
2024-10-30 05:38:53 +00:00

170 lines
6.2 KiB
Nim

# beacon_chain
# Copyright (c) 2018-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
# Everything needed to run a full Beacon Node
import
std/osproc,
# Nimble packages
chronos, presto, bearssl/rand,
# Local modules
"."/[beacon_clock, beacon_chain_db, conf, light_client],
./gossip_processing/[eth2_processor, block_processor, optimistic_processor],
./networking/eth2_network,
./el/el_manager,
./consensus_object_pools/[
blockchain_dag, blob_quarantine, block_quarantine, consensus_manager,
data_column_quarantine, attestation_pool, sync_committee_msg_pool, validator_change_pool,
blockchain_list],
./spec/datatypes/[base, altair],
./spec/eth2_apis/dynamic_fee_recipients,
./spec/signatures_batch,
./sync/[sync_manager, request_manager, sync_types],
./validators/[
action_tracker, message_router, validator_monitor, validator_pool,
keystore_management],
./rpc/state_ttl_cache
export
osproc, chronos, presto, action_tracker,
beacon_clock, beacon_chain_db, conf, light_client,
attestation_pool, sync_committee_msg_pool, validator_change_pool,
eth2_network, el_manager, request_manager, sync_manager,
eth2_processor, optimistic_processor, blockchain_dag, block_quarantine,
base, message_router, validator_monitor, validator_pool,
consensus_manager, dynamic_fee_recipients, sync_types
type
EventBus* = object
headQueue*: AsyncEventQueue[HeadChangeInfoObject]
blocksQueue*: AsyncEventQueue[EventBeaconBlockObject]
attestQueue*: AsyncEventQueue[phase0.Attestation]
exitQueue*: AsyncEventQueue[SignedVoluntaryExit]
blsToExecQueue*: AsyncEventQueue[SignedBLSToExecutionChange]
propSlashQueue*: AsyncEventQueue[ProposerSlashing]
attSlashQueue*: AsyncEventQueue[phase0.AttesterSlashing]
blobSidecarQueue*: AsyncEventQueue[BlobSidecarInfoObject]
finalQueue*: AsyncEventQueue[FinalizationInfoObject]
reorgQueue*: AsyncEventQueue[ReorgInfoObject]
contribQueue*: AsyncEventQueue[SignedContributionAndProof]
finUpdateQueue*: AsyncEventQueue[
RestVersioned[ForkedLightClientFinalityUpdate]]
optUpdateQueue*: AsyncEventQueue[
RestVersioned[ForkedLightClientOptimisticUpdate]]
optFinHeaderUpdateQueue*: AsyncEventQueue[ForkedLightClientHeader]
BeaconNode* = ref object
nickname*: string
graffitiBytes*: GraffitiBytes
network*: Eth2Node
netKeys*: NetKeyPair
db*: BeaconChainDB
config*: BeaconNodeConf
attachedValidators*: ref ValidatorPool
optimisticProcessor*: OptimisticProcessor
optimisticFcuFut*: Future[(PayloadExecutionStatus, Opt[BlockHash])]
.Raising([CancelledError])
lightClient*: LightClient
dag*: ChainDAGRef
list*: ChainListRef
quarantine*: ref Quarantine
blobQuarantine*: ref BlobQuarantine
dataColumnQuarantine*: ref DataColumnQuarantine
attestationPool*: ref AttestationPool
syncCommitteeMsgPool*: ref SyncCommitteeMsgPool
lightClientPool*: ref LightClientPool
validatorChangePool*: ref ValidatorChangePool
elManager*: ELManager
restServer*: RestServerRef
keymanagerHost*: ref KeymanagerHost
keymanagerServer*: RestServerRef
keystoreCache*: KeystoreCacheRef
eventBus*: EventBus
vcProcess*: Process
requestManager*: RequestManager
syncManager*: SyncManager[Peer, PeerId]
backfiller*: SyncManager[Peer, PeerId]
untrustedManager*: SyncManager[Peer, PeerId]
syncOverseer*: SyncOverseerRef
genesisSnapshotContent*: string
processor*: ref Eth2Processor
batchVerifier*: ref BatchVerifier
blockProcessor*: ref BlockProcessor
consensusManager*: ref ConsensusManager
attachedValidatorBalanceTotal*: Gwei
gossipState*: GossipState
blocksGossipState*: GossipState
beaconClock*: BeaconClock
restKeysCache*: Table[ValidatorPubKey, ValidatorIndex]
validatorMonitor*: ref ValidatorMonitor
stateTtlCache*: StateTtlCache
router*: ref MessageRouter
dynamicFeeRecipientsStore*: ref DynamicFeeRecipientsStore
externalBuilderRegistrations*:
Table[ValidatorPubKey, SignedValidatorRegistrationV1]
dutyValidatorCount*: int
## Number of validators that we've checked for activation
processingDelay*: Opt[Duration]
lastValidAttestedBlock*: Opt[BlockSlot]
shutdownEvent*: AsyncEvent
template findIt*(s: openArray, predicate: untyped): int =
var res = -1
for i, it {.inject.} in s:
if predicate:
res = i
break
res
template rng*(node: BeaconNode): ref HmacDrbgContext =
node.network.rng
proc currentSlot*(node: BeaconNode): Slot =
node.beaconClock.now.slotOrZero
func getPayloadBuilderAddress*(config: BeaconNodeConf): Opt[string] =
if config.payloadBuilderEnable:
Opt.some config.payloadBuilderUrl
else:
Opt.none(string)
proc getPayloadBuilderAddress*(
node: BeaconNode, pubkey: ValidatorPubKey): Opt[string] =
let defaultPayloadBuilderAddress = node.config.getPayloadBuilderAddress
if node.keymanagerHost.isNil:
defaultPayloadBuilderAddress
else:
node.keymanagerHost[].getBuilderConfig(pubkey).valueOr:
defaultPayloadBuilderAddress
proc getPayloadBuilderClient*(
node: BeaconNode, validator_index: uint64): RestResult[RestClientRef] =
if not node.config.payloadBuilderEnable:
return err "Payload builder globally disabled"
let
pubkey = withState(node.dag.headState):
if validator_index >= forkyState.data.validators.lenu64:
return err "Validator index too high"
forkyState.data.validators.item(validator_index).pubkey
payloadBuilderAddress = node.getPayloadBuilderAddress(pubkey)
if payloadBuilderAddress.isNone:
return err "Payload builder disabled"
let
flags = {RestClientFlag.CommaSeparatedArray,
RestClientFlag.ResolveAlways}
socketFlags = {SocketFlags.TcpNoDelay}
RestClientRef.new(payloadBuilderAddress.get, flags = flags,
socketFlags = socketFlags)