2024-09-09 09:12:56 +00:00
|
|
|
# Nimbus
|
|
|
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
|
|
|
|
# Licensed under either of
|
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
|
|
|
# except according to those terms.
|
|
|
|
|
|
|
|
|
|
{.push raises: [].}
|
|
|
|
|
|
|
|
|
|
import
|
|
|
|
|
pkg/[chronicles, chronos, eth/p2p, results],
|
|
|
|
|
pkg/stew/[interval_set, sorted_set],
|
2024-10-01 09:19:29 +00:00
|
|
|
../core/chain,
|
2024-10-21 18:01:45 +00:00
|
|
|
./beacon/[worker, worker_desc, worker/db],
|
2024-09-09 09:12:56 +00:00
|
|
|
"."/[sync_desc, sync_sched, protocol]
|
|
|
|
|
|
|
|
|
|
logScope:
|
2024-10-02 11:31:33 +00:00
|
|
|
topics = "beacon"
|
2024-09-09 09:12:56 +00:00
|
|
|
|
|
|
|
|
type
|
2024-10-02 11:31:33 +00:00
|
|
|
BeaconSyncRef* = RunnerSyncRef[BeaconCtxData,BeaconBuddyData]
|
2024-09-09 09:12:56 +00:00
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
# Virtual methods/interface, `mixin` functions
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runSetup(ctx: BeaconCtxRef): bool =
|
2024-10-01 09:19:29 +00:00
|
|
|
worker.setup(ctx)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runRelease(ctx: BeaconCtxRef) =
|
2024-10-01 09:19:29 +00:00
|
|
|
worker.release(ctx)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runDaemon(ctx: BeaconCtxRef) {.async.} =
|
2024-10-01 09:19:29 +00:00
|
|
|
await worker.runDaemon(ctx)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runStart(buddy: BeaconBuddyRef): bool =
|
2024-10-01 09:19:29 +00:00
|
|
|
worker.start(buddy)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runStop(buddy: BeaconBuddyRef) =
|
2024-10-01 09:19:29 +00:00
|
|
|
worker.stop(buddy)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runPool(buddy: BeaconBuddyRef; last: bool; laps: int): bool =
|
2024-10-01 09:19:29 +00:00
|
|
|
worker.runPool(buddy, last, laps)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-02 11:31:33 +00:00
|
|
|
proc runPeer(buddy: BeaconBuddyRef) {.async.} =
|
2024-10-01 09:19:29 +00:00
|
|
|
await worker.runPeer(buddy)
|
2024-09-09 09:12:56 +00:00
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
# Public functions
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
proc init*(
|
2024-10-02 11:31:33 +00:00
|
|
|
T: type BeaconSyncRef;
|
2024-09-09 09:12:56 +00:00
|
|
|
ethNode: EthereumNode;
|
|
|
|
|
chain: ForkedChainRef;
|
|
|
|
|
maxPeers: int;
|
Flare sync (#2627)
* Cosmetics, small fixes, add stashed headers verifier
* Remove direct `Era1` support
why:
Era1 is indirectly supported by using the import tool before syncing.
* Clarify database persistent save function.
why:
Function relied on the last saved state block number which was wrong.
It now relies on the tx-level. If it is 0, then data are saved directly.
Otherwise the task that owns the tx will do it.
* Extracted configuration constants into separate file
* Enable single peer mode for debugging
* Fix peer losing issue in multi-mode
details:
Running concurrent download peers was previously programmed as running
a batch downloading and storing ~8k headers and then leaving the `async`
function to be restarted by a scheduler.
This was unfortunate because of occasionally occurring long waiting
times for restart.
While the time gap until restarting were typically observed a few
millisecs, there were always a few outliers which well exceed several
seconds. This seemed to let remote peers run into timeouts.
* Prefix function names `unprocXxx()` and `stagedYyy()` by `headers`
why:
There will be other `unproc` and `staged` modules.
* Remove cruft, update logging
* Fix accounting issue
details:
When staging after fetching headers from the network, there was an off
by 1 error occurring when the result was by one smaller than requested.
Also, a whole range was mis-accounted when a peer was terminating
connection immediately after responding.
* Fix slow/error header accounting when fetching
why:
Originally set for detecting slow headers in a row, the counter
was wrongly extended to general errors.
* Ban peers for a while that respond with too few headers continuously
why:
Some peers only returned one header at a time. If these peers sit on a
farm, they might collectively slow down the download process.
* Update RPC beacon header updater
why:
Old function hook has slightly changed its meaning since it was used
for snap sync. Also, the old hook is used by other functions already.
* Limit number of peers or set to single peer mode
details:
Merge several concepts, single peer mode being one of it.
* Some code clean up, fixings for removing of compiler warnings
* De-noise header fetch related sources
why:
Header download looks relatively stable, so general debugging is not
needed, anymore. This is the equivalent of removing the scaffold from
the part of the building where work has completed.
* More clean up and code prettification for headers stuff
* Implement body fetch and block import
details:
Available headers are used stage blocks by combining existing headers
with newly fetched blocks. Then these blocks are imported/executed via
`persistBlocks()`.
* Logger cosmetics and cleanup
* Remove staged block queue debugging
details:
Feature still available, just not executed anymore
* Docu, logging update
* Update/simplify `runDaemon()`
* Re-calibrate block body requests and soft config for import blocks batch
why:
* For fetching, larger fetch requests are mostly truncated anyway on
MainNet.
* For executing, smaller batch sizes reduce the memory needed for the
price of longer execution times.
* Update metrics counters
* Docu update
* Some fixes, formatting updates, etc.
* Update `borrowed` type: uint -. uint64
also:
Always convert to `uint64` rather than `uint` where appropriate
2024-09-27 15:07:42 +00:00
|
|
|
chunkSize: int;
|
2024-09-09 09:12:56 +00:00
|
|
|
): T =
|
Flare sync (#2627)
* Cosmetics, small fixes, add stashed headers verifier
* Remove direct `Era1` support
why:
Era1 is indirectly supported by using the import tool before syncing.
* Clarify database persistent save function.
why:
Function relied on the last saved state block number which was wrong.
It now relies on the tx-level. If it is 0, then data are saved directly.
Otherwise the task that owns the tx will do it.
* Extracted configuration constants into separate file
* Enable single peer mode for debugging
* Fix peer losing issue in multi-mode
details:
Running concurrent download peers was previously programmed as running
a batch downloading and storing ~8k headers and then leaving the `async`
function to be restarted by a scheduler.
This was unfortunate because of occasionally occurring long waiting
times for restart.
While the time gap until restarting were typically observed a few
millisecs, there were always a few outliers which well exceed several
seconds. This seemed to let remote peers run into timeouts.
* Prefix function names `unprocXxx()` and `stagedYyy()` by `headers`
why:
There will be other `unproc` and `staged` modules.
* Remove cruft, update logging
* Fix accounting issue
details:
When staging after fetching headers from the network, there was an off
by 1 error occurring when the result was by one smaller than requested.
Also, a whole range was mis-accounted when a peer was terminating
connection immediately after responding.
* Fix slow/error header accounting when fetching
why:
Originally set for detecting slow headers in a row, the counter
was wrongly extended to general errors.
* Ban peers for a while that respond with too few headers continuously
why:
Some peers only returned one header at a time. If these peers sit on a
farm, they might collectively slow down the download process.
* Update RPC beacon header updater
why:
Old function hook has slightly changed its meaning since it was used
for snap sync. Also, the old hook is used by other functions already.
* Limit number of peers or set to single peer mode
details:
Merge several concepts, single peer mode being one of it.
* Some code clean up, fixings for removing of compiler warnings
* De-noise header fetch related sources
why:
Header download looks relatively stable, so general debugging is not
needed, anymore. This is the equivalent of removing the scaffold from
the part of the building where work has completed.
* More clean up and code prettification for headers stuff
* Implement body fetch and block import
details:
Available headers are used stage blocks by combining existing headers
with newly fetched blocks. Then these blocks are imported/executed via
`persistBlocks()`.
* Logger cosmetics and cleanup
* Remove staged block queue debugging
details:
Feature still available, just not executed anymore
* Docu, logging update
* Update/simplify `runDaemon()`
* Re-calibrate block body requests and soft config for import blocks batch
why:
* For fetching, larger fetch requests are mostly truncated anyway on
MainNet.
* For executing, smaller batch sizes reduce the memory needed for the
price of longer execution times.
* Update metrics counters
* Docu update
* Some fixes, formatting updates, etc.
* Update `borrowed` type: uint -. uint64
also:
Always convert to `uint64` rather than `uint` where appropriate
2024-09-27 15:07:42 +00:00
|
|
|
var desc = T()
|
2024-10-01 09:19:29 +00:00
|
|
|
desc.initSync(ethNode, maxPeers)
|
Flare sync (#2627)
* Cosmetics, small fixes, add stashed headers verifier
* Remove direct `Era1` support
why:
Era1 is indirectly supported by using the import tool before syncing.
* Clarify database persistent save function.
why:
Function relied on the last saved state block number which was wrong.
It now relies on the tx-level. If it is 0, then data are saved directly.
Otherwise the task that owns the tx will do it.
* Extracted configuration constants into separate file
* Enable single peer mode for debugging
* Fix peer losing issue in multi-mode
details:
Running concurrent download peers was previously programmed as running
a batch downloading and storing ~8k headers and then leaving the `async`
function to be restarted by a scheduler.
This was unfortunate because of occasionally occurring long waiting
times for restart.
While the time gap until restarting were typically observed a few
millisecs, there were always a few outliers which well exceed several
seconds. This seemed to let remote peers run into timeouts.
* Prefix function names `unprocXxx()` and `stagedYyy()` by `headers`
why:
There will be other `unproc` and `staged` modules.
* Remove cruft, update logging
* Fix accounting issue
details:
When staging after fetching headers from the network, there was an off
by 1 error occurring when the result was by one smaller than requested.
Also, a whole range was mis-accounted when a peer was terminating
connection immediately after responding.
* Fix slow/error header accounting when fetching
why:
Originally set for detecting slow headers in a row, the counter
was wrongly extended to general errors.
* Ban peers for a while that respond with too few headers continuously
why:
Some peers only returned one header at a time. If these peers sit on a
farm, they might collectively slow down the download process.
* Update RPC beacon header updater
why:
Old function hook has slightly changed its meaning since it was used
for snap sync. Also, the old hook is used by other functions already.
* Limit number of peers or set to single peer mode
details:
Merge several concepts, single peer mode being one of it.
* Some code clean up, fixings for removing of compiler warnings
* De-noise header fetch related sources
why:
Header download looks relatively stable, so general debugging is not
needed, anymore. This is the equivalent of removing the scaffold from
the part of the building where work has completed.
* More clean up and code prettification for headers stuff
* Implement body fetch and block import
details:
Available headers are used stage blocks by combining existing headers
with newly fetched blocks. Then these blocks are imported/executed via
`persistBlocks()`.
* Logger cosmetics and cleanup
* Remove staged block queue debugging
details:
Feature still available, just not executed anymore
* Docu, logging update
* Update/simplify `runDaemon()`
* Re-calibrate block body requests and soft config for import blocks batch
why:
* For fetching, larger fetch requests are mostly truncated anyway on
MainNet.
* For executing, smaller batch sizes reduce the memory needed for the
price of longer execution times.
* Update metrics counters
* Docu update
* Some fixes, formatting updates, etc.
* Update `borrowed` type: uint -. uint64
also:
Always convert to `uint64` rather than `uint` where appropriate
2024-09-27 15:07:42 +00:00
|
|
|
desc.ctx.pool.nBodiesBatch = chunkSize
|
2024-10-17 17:59:50 +00:00
|
|
|
desc.ctx.pool.chain = chain
|
Flare sync (#2627)
* Cosmetics, small fixes, add stashed headers verifier
* Remove direct `Era1` support
why:
Era1 is indirectly supported by using the import tool before syncing.
* Clarify database persistent save function.
why:
Function relied on the last saved state block number which was wrong.
It now relies on the tx-level. If it is 0, then data are saved directly.
Otherwise the task that owns the tx will do it.
* Extracted configuration constants into separate file
* Enable single peer mode for debugging
* Fix peer losing issue in multi-mode
details:
Running concurrent download peers was previously programmed as running
a batch downloading and storing ~8k headers and then leaving the `async`
function to be restarted by a scheduler.
This was unfortunate because of occasionally occurring long waiting
times for restart.
While the time gap until restarting were typically observed a few
millisecs, there were always a few outliers which well exceed several
seconds. This seemed to let remote peers run into timeouts.
* Prefix function names `unprocXxx()` and `stagedYyy()` by `headers`
why:
There will be other `unproc` and `staged` modules.
* Remove cruft, update logging
* Fix accounting issue
details:
When staging after fetching headers from the network, there was an off
by 1 error occurring when the result was by one smaller than requested.
Also, a whole range was mis-accounted when a peer was terminating
connection immediately after responding.
* Fix slow/error header accounting when fetching
why:
Originally set for detecting slow headers in a row, the counter
was wrongly extended to general errors.
* Ban peers for a while that respond with too few headers continuously
why:
Some peers only returned one header at a time. If these peers sit on a
farm, they might collectively slow down the download process.
* Update RPC beacon header updater
why:
Old function hook has slightly changed its meaning since it was used
for snap sync. Also, the old hook is used by other functions already.
* Limit number of peers or set to single peer mode
details:
Merge several concepts, single peer mode being one of it.
* Some code clean up, fixings for removing of compiler warnings
* De-noise header fetch related sources
why:
Header download looks relatively stable, so general debugging is not
needed, anymore. This is the equivalent of removing the scaffold from
the part of the building where work has completed.
* More clean up and code prettification for headers stuff
* Implement body fetch and block import
details:
Available headers are used stage blocks by combining existing headers
with newly fetched blocks. Then these blocks are imported/executed via
`persistBlocks()`.
* Logger cosmetics and cleanup
* Remove staged block queue debugging
details:
Feature still available, just not executed anymore
* Docu, logging update
* Update/simplify `runDaemon()`
* Re-calibrate block body requests and soft config for import blocks batch
why:
* For fetching, larger fetch requests are mostly truncated anyway on
MainNet.
* For executing, smaller batch sizes reduce the memory needed for the
price of longer execution times.
* Update metrics counters
* Docu update
* Some fixes, formatting updates, etc.
* Update `borrowed` type: uint -. uint64
also:
Always convert to `uint64` rather than `uint` where appropriate
2024-09-27 15:07:42 +00:00
|
|
|
desc
|
2024-09-09 09:12:56 +00:00
|
|
|
|
2024-10-21 18:01:45 +00:00
|
|
|
proc start*(desc: BeaconSyncRef; resumeOnly = false): bool =
|
|
|
|
|
## Start beacon sync. If `resumeOnly` is set `true` the syncer will only
|
|
|
|
|
## start up if it can resume work, e.g. after being previously interrupted.
|
|
|
|
|
if resumeOnly:
|
|
|
|
|
desc.ctx.dbLoadSyncStateLayout()
|
|
|
|
|
if not desc.ctx.layout.headLocked:
|
|
|
|
|
return false
|
|
|
|
|
desc.startSync()
|
|
|
|
|
|
|
|
|
|
proc stop*(desc: BeaconSyncRef) =
|
|
|
|
|
desc.stopSync()
|
2024-09-09 09:12:56 +00:00
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
# End
|
|
|
|
|
# ------------------------------------------------------------------------------
|