2022-08-04 08:04:30 +00:00
|
|
|
# Nimbus
|
2022-05-09 14:04:48 +00:00
|
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
|
|
# Licensed under either of
|
2022-05-13 16:30:10 +00:00
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
|
|
# except according to those terms.
|
2022-05-09 14:04:48 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
{.push raises: [].}
|
|
|
|
|
2022-05-09 14:04:48 +00:00
|
|
|
import
|
2022-05-17 11:09:49 +00:00
|
|
|
chronicles,
|
|
|
|
chronos,
|
2023-04-06 19:42:07 +00:00
|
|
|
eth/p2p,
|
2022-08-04 08:04:30 +00:00
|
|
|
stew/[interval_set, keyed_queue],
|
2023-04-06 19:42:07 +00:00
|
|
|
"../.."/[common, db/select_backend],
|
|
|
|
".."/[handlers/eth, protocol, sync_desc],
|
|
|
|
./worker/[pivot, play, ticker],
|
Prep for full sync after snap make 4 (#1282)
* Re-arrange fetching storage slots in batch module
why;
Previously, fetching partial slot ranges first has a chance of
terminating the worker peer 9due to network error) while there were
many inheritable storage slots on the queue.
Now, inheritance is checked first, then full slot ranges and finally
partial ranges.
* Update logging
* Bundled node information for healing into single object `NodeSpecs`
why:
Previously, partial paths and node keys were kept in separate variables.
This approach was error prone due to copying/reassembling function
argument objects.
As all partial paths, keys, and node data types are more or less handled
as `Blob`s over the network (using Eth/6x, or Snap/1) it makes sense to
hold these `Blob`s as named field in a single object (even if not all
fields are active for the current purpose.)
* For good housekeeping, using `NodeKey` type only for account keys
why:
previously, a mixture of `NodeKey` and `Hash256` was used. Now, only
state or storage root keys use the `Hash256` type.
* Always accept latest pivot (and not a slightly older one)
why;
For testing it was tried to use a slightly older pivot state root than
available. Some anecdotal tests seemed to suggest an advantage so that
more peers are willing to serve on that older pivot. But this could not
be confirmed in subsequent tests (still anecdotal, though.)
As a side note, the distance of the latest pivot to its predecessor is
at least 128 (or whatever the constant `minPivotBlockDistance` is
assigned to.)
* Reshuffle name components for some file and function names
why:
Clarifies purpose:
"storages" becomes: "storage slots"
"store" becomes: "range fetch"
* Stash away currently unused modules in sub-folder named "notused"
2022-10-27 13:49:28 +00:00
|
|
|
./worker/com/com_error,
|
2023-04-06 19:42:07 +00:00
|
|
|
./worker/db/[snapdb_desc, snapdb_pivot],
|
|
|
|
"."/[range_desc, worker_desc]
|
2022-09-16 07:24:12 +00:00
|
|
|
|
2022-05-23 16:53:19 +00:00
|
|
|
logScope:
|
2023-04-06 19:42:07 +00:00
|
|
|
topics = "snap-worker"
|
2022-10-08 17:20:50 +00:00
|
|
|
|
2022-09-30 08:22:14 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
2023-01-17 09:28:14 +00:00
|
|
|
# Private helpers
|
2022-09-30 08:22:14 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
template ignoreException(info: static[string]; code: untyped) =
|
2023-01-17 09:28:14 +00:00
|
|
|
try:
|
|
|
|
code
|
|
|
|
except CatchableError as e:
|
2023-04-06 19:42:07 +00:00
|
|
|
error "Exception at " & info & ":", name=($e.name), msg=(e.msg)
|
2022-09-30 08:22:14 +00:00
|
|
|
|
2022-10-08 17:20:50 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Private functions
|
|
|
|
# ------------------------------------------------------------------------------
|
2022-09-30 08:22:14 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
proc disableWireServices(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `setup()`: Temporarily stop useless wire protocol services.
|
|
|
|
ctx.ethWireCtx.txPoolEnabled = false
|
2022-11-25 14:56:42 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
proc enableWireServices(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `release()`
|
|
|
|
ctx.ethWireCtx.txPoolEnabled = true
|
|
|
|
|
|
|
|
# --------------
|
|
|
|
|
|
|
|
proc enableTicker(ctx: SnapCtxRef; tickerOK: bool) =
|
|
|
|
## Helper for `setup()`: Log/status ticker
|
|
|
|
if tickerOK:
|
|
|
|
ctx.pool.ticker = TickerRef.init(ctx.pool.pivotTable.tickerStats(ctx))
|
|
|
|
else:
|
|
|
|
trace "Ticker is disabled"
|
|
|
|
|
|
|
|
proc disableTicker(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `release()`
|
|
|
|
if not ctx.pool.ticker.isNil:
|
|
|
|
ctx.pool.ticker.stop()
|
|
|
|
ctx.pool.ticker = nil
|
|
|
|
|
|
|
|
# --------------
|
|
|
|
|
|
|
|
proc enableRpcMagic(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `setup()`: Enable external pivot update via RPC
|
|
|
|
ctx.chain.com.syncReqNewHead = ctx.pivotUpdateBeaconHeaderCB
|
|
|
|
|
|
|
|
proc disableRpcMagic(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `release()`
|
|
|
|
ctx.chain.com.syncReqNewHead = nil
|
|
|
|
|
|
|
|
# --------------
|
|
|
|
|
|
|
|
proc detectSnapSyncRecovery(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `setup()`: Initiate snap sync recovery (if any)
|
|
|
|
let rc = ctx.pool.snapDb.pivotRecoverDB()
|
|
|
|
if rc.isOk:
|
|
|
|
ctx.pool.recovery = SnapRecoveryRef(state: rc.value)
|
|
|
|
ctx.daemon = true
|
|
|
|
|
|
|
|
# Set up early initial pivot
|
|
|
|
ctx.pool.pivotTable.reverseUpdate(ctx.pool.recovery.state.header, ctx)
|
|
|
|
trace "Snap sync recovery started",
|
|
|
|
checkpoint=("#" & $ctx.pool.pivotTable.topNumber() & "(0)")
|
|
|
|
if not ctx.pool.ticker.isNil:
|
|
|
|
ctx.pool.ticker.startRecovery()
|
|
|
|
|
|
|
|
proc initSnapDb(ctx: SnapCtxRef) =
|
|
|
|
## Helper for `setup()`: Initialise snap sync database layer
|
|
|
|
ctx.pool.snapDb =
|
|
|
|
if ctx.pool.dbBackend.isNil: SnapDbRef.init(ctx.chain.db.db)
|
|
|
|
else: SnapDbRef.init(ctx.pool.dbBackend)
|
2022-11-25 14:56:42 +00:00
|
|
|
|
2022-06-06 13:42:08 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Public start/stop and admin functions
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2022-08-04 08:04:30 +00:00
|
|
|
proc setup*(ctx: SnapCtxRef; tickerOK: bool): bool =
|
2022-06-06 13:42:08 +00:00
|
|
|
## Global set up
|
2022-11-16 23:51:06 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
# For snap sync book keeping
|
|
|
|
ctx.pool.coveredAccounts = NodeTagRangeSet.init()
|
2022-11-25 14:56:42 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
ctx.enableRpcMagic() # Allow external pivot update via RPC
|
|
|
|
ctx.disableWireServices() # Stop unwanted public services
|
|
|
|
ctx.pool.syncMode.playInit() # Set up sync sub-mode specs.
|
|
|
|
ctx.initSnapDb() # Set database backend, subject to change
|
|
|
|
ctx.detectSnapSyncRecovery() # Check for recovery mode
|
|
|
|
ctx.enableTicker(tickerOK) # Start log/status ticker (if any)
|
2023-03-07 14:23:22 +00:00
|
|
|
|
2023-04-06 19:42:07 +00:00
|
|
|
# Experimental, also used for debugging
|
2023-03-07 14:23:22 +00:00
|
|
|
if ctx.exCtrlFile.isSome:
|
|
|
|
warn "Snap sync accepts pivot block number or hash",
|
|
|
|
syncCtrlFile=ctx.exCtrlFile.get
|
2022-11-25 14:56:42 +00:00
|
|
|
true
|
2022-08-04 08:04:30 +00:00
|
|
|
|
|
|
|
proc release*(ctx: SnapCtxRef) =
|
2022-06-06 13:42:08 +00:00
|
|
|
## Global clean up
|
2023-04-06 19:42:07 +00:00
|
|
|
ctx.disableTicker() # Stop log/status ticker (if any)
|
|
|
|
ctx.enableWireServices() # re-enable public services
|
|
|
|
ctx.disableRpcMagic() # Disable external pivot update via RPC
|
|
|
|
|
2022-06-16 08:58:50 +00:00
|
|
|
|
2022-08-04 08:04:30 +00:00
|
|
|
proc start*(buddy: SnapBuddyRef): bool =
|
|
|
|
## Initialise worker peer
|
|
|
|
let
|
|
|
|
ctx = buddy.ctx
|
|
|
|
peer = buddy.peer
|
|
|
|
if peer.supports(protocol.snap) and
|
|
|
|
peer.supports(protocol.eth) and
|
|
|
|
peer.state(protocol.eth).initialized:
|
2023-02-23 13:13:02 +00:00
|
|
|
buddy.only.errors = ComErrorStatsRef()
|
|
|
|
if not ctx.pool.ticker.isNil:
|
|
|
|
ctx.pool.ticker.startBuddy()
|
2022-06-06 13:42:08 +00:00
|
|
|
return true
|
|
|
|
|
2022-08-04 08:04:30 +00:00
|
|
|
proc stop*(buddy: SnapBuddyRef) =
|
2022-06-06 13:42:08 +00:00
|
|
|
## Clean up this peer
|
2023-01-17 09:28:14 +00:00
|
|
|
let ctx = buddy.ctx
|
2023-02-23 13:13:02 +00:00
|
|
|
if not ctx.pool.ticker.isNil:
|
|
|
|
ctx.pool.ticker.stopBuddy()
|
2022-06-06 13:42:08 +00:00
|
|
|
|
2022-05-17 11:09:49 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
2023-04-06 19:42:07 +00:00
|
|
|
# Public functions, sync handler multiplexers
|
2022-05-17 11:09:49 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2022-11-14 14:13:00 +00:00
|
|
|
proc runDaemon*(ctx: SnapCtxRef) {.async.} =
|
2023-04-06 19:42:07 +00:00
|
|
|
## Sync processsing multiplexer
|
|
|
|
ignoreException("runDaemon"):
|
|
|
|
await ctx.playSyncSpecs.daemon(ctx)
|
2022-11-14 14:13:00 +00:00
|
|
|
|
|
|
|
proc runSingle*(buddy: SnapBuddyRef) {.async.} =
|
2023-04-06 19:42:07 +00:00
|
|
|
## Sync processsing multiplexer
|
|
|
|
ignoreException("runSingle"):
|
|
|
|
await buddy.ctx.playSyncSpecs.single(buddy)
|
2022-05-09 14:04:48 +00:00
|
|
|
|
2022-11-25 14:56:42 +00:00
|
|
|
proc runPool*(buddy: SnapBuddyRef, last: bool): bool =
|
2023-04-06 19:42:07 +00:00
|
|
|
## Sync processsing multiplexer
|
|
|
|
ignoreException("runPool"):
|
|
|
|
result = buddy.ctx.playSyncSpecs.pool(buddy,last)
|
2022-05-17 11:09:49 +00:00
|
|
|
|
2022-08-04 08:04:30 +00:00
|
|
|
proc runMulti*(buddy: SnapBuddyRef) {.async.} =
|
2023-04-06 19:42:07 +00:00
|
|
|
## Sync processsing multiplexer
|
|
|
|
ignoreException("runMulti"):
|
|
|
|
await buddy.ctx.playSyncSpecs.multi(buddy)
|
2022-11-08 18:56:04 +00:00
|
|
|
|
2022-05-17 11:09:49 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# End
|
|
|
|
# ------------------------------------------------------------------------------
|