361 lines
12 KiB
Nim
361 lines
12 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
{.push raises: [].}
|
|
|
|
import
|
|
chronicles,
|
|
chronos,
|
|
eth/p2p,
|
|
stew/[interval_set, keyed_queue],
|
|
"../../.."/[handlers/eth, misc/ticker, protocol],
|
|
"../.."/[range_desc, worker_desc],
|
|
../db/[hexary_desc, snapdb_pivot],
|
|
../get/get_error,
|
|
./pass_desc,
|
|
./pass_snap/helper/[beacon_header, storage_queue],
|
|
./pass_snap/[pivot, snap_pass_desc]
|
|
|
|
logScope:
|
|
topics = "snap-play"
|
|
|
|
const
|
|
extraTraceMessages = false # or true
|
|
## Enabled additional logging noise
|
|
|
|
extraScrutinyDoubleCheckCompleteness = 1_000_000
|
|
## Double check database whether it is complete (debugging, testing). This
|
|
## action is slow and intended for debugging and testing use, only. The
|
|
## numeric value limits the action to the maximal number of account in the
|
|
## database.
|
|
##
|
|
## Set to `0` to disable.
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
template logTxt(info: static[string]): static[string] =
|
|
"Snap worker " & info
|
|
|
|
template ignoreException(info: static[string]; code: untyped) =
|
|
try:
|
|
code
|
|
except CatchableError as e:
|
|
error "Exception at " & info & ":", name=($e.name), msg=(e.msg)
|
|
|
|
# --------------
|
|
|
|
proc disableWireServices(ctx: SnapCtxRef) =
|
|
## Helper for `setup()`: Temporarily stop useless wire protocol services.
|
|
ctx.ethWireCtx.txPoolEnabled = false
|
|
|
|
proc enableWireServices(ctx: SnapCtxRef) =
|
|
## Helper for `release()`
|
|
ctx.ethWireCtx.txPoolEnabled = true
|
|
|
|
# --------------
|
|
|
|
proc enableRpcMagic(ctx: SnapCtxRef) =
|
|
## Helper for `setup()`: Enable external pivot update via RPC
|
|
ctx.chain.com.syncReqNewHead = ctx.pivotUpdateBeaconHeaderCB
|
|
|
|
proc disableRpcMagic(ctx: SnapCtxRef) =
|
|
## Helper for `release()`
|
|
ctx.chain.com.syncReqNewHead = nil
|
|
|
|
# --------------
|
|
|
|
proc detectSnapSyncRecovery(ctx: SnapCtxRef) =
|
|
## Helper for `setup()`: Initiate snap sync recovery (if any)
|
|
let rc = ctx.pool.snapDb.pivotRecoverDB()
|
|
if rc.isOk:
|
|
let snap = ctx.pool.pass
|
|
snap.recovery = RecoveryRef(state: rc.value)
|
|
ctx.daemon = true
|
|
|
|
# Set up early initial pivot
|
|
snap.pivotTable.reverseUpdate(snap.recovery.state.header, ctx)
|
|
trace logTxt "recovery started",
|
|
checkpoint=(snap.pivotTable.topNumber.toStr & "(0)")
|
|
if not ctx.pool.ticker.isNil:
|
|
ctx.pool.ticker.startRecovery()
|
|
|
|
proc recoveryStepContinue(ctx: SnapCtxRef): Future[bool] {.async.} =
|
|
let
|
|
snap = ctx.pool.pass
|
|
recov = snap.recovery
|
|
if recov.isNil:
|
|
return false
|
|
|
|
let
|
|
checkpoint = recov.state.header.blockNumber.toStr & "(" & $recov.level & ")"
|
|
topLevel = recov.level == 0
|
|
env = block:
|
|
let rc = snap.pivotTable.eq recov.state.header.stateRoot
|
|
if rc.isErr:
|
|
error logTxt "recovery pivot context gone", checkpoint, topLevel
|
|
return false
|
|
rc.value
|
|
|
|
# Cosmetics: allow other processes (e.g. ticker) to log the current recovery
|
|
# state. There is no other intended purpose of this wait state.
|
|
await sleepAsync 1100.milliseconds
|
|
|
|
#when extraTraceMessages:
|
|
# trace "Recovery continued ...", checkpoint, topLevel,
|
|
# nAccounts=recov.state.nAccounts, nDangling=recov.state.dangling.len
|
|
|
|
# Update pivot data from recovery checkpoint
|
|
env.pivotRecoverFromCheckpoint(ctx, topLevel)
|
|
|
|
# Fetch next recovery record if there is any
|
|
if recov.state.predecessor.isZero:
|
|
#when extraTraceMessages:
|
|
# trace "Recovery done", checkpoint, topLevel
|
|
return false
|
|
let rc = ctx.pool.snapDb.pivotRecoverDB(recov.state.predecessor)
|
|
if rc.isErr:
|
|
when extraTraceMessages:
|
|
trace logTxt "stale pivot, recovery stopped", checkpoint, topLevel
|
|
return false
|
|
|
|
# Set up next level pivot checkpoint
|
|
snap.recovery = RecoveryRef(
|
|
state: rc.value,
|
|
level: recov.level + 1)
|
|
|
|
# Push onto pivot table and continue recovery (i.e. do not stop it yet)
|
|
snap.pivotTable.reverseUpdate(snap.recovery.state.header, ctx)
|
|
|
|
return true # continue recovery
|
|
|
|
# --------------
|
|
|
|
proc snapSyncCompleteOk(
|
|
env: SnapPivotRef; # Current pivot environment
|
|
ctx: SnapCtxRef; # Some global context
|
|
): Future[bool]
|
|
{.async.} =
|
|
## Check whether this pivot is fully downloaded. The `async` part is for
|
|
## debugging, only and should not be used on a large database as it uses
|
|
## quite a bit of computation ressources.
|
|
if env.pivotCompleteOk():
|
|
when 0 < extraScrutinyDoubleCheckCompleteness:
|
|
# Larger sizes might be infeasible
|
|
if env.nAccounts <= extraScrutinyDoubleCheckCompleteness:
|
|
if not await env.pivotVerifyComplete(ctx):
|
|
error logTxt "inconsistent state, pivot incomplete",
|
|
pivot=env.stateHeader.blockNumber.toStr, nAccounts=env.nAccounts
|
|
return false
|
|
ctx.pool.pass.completedPivot = env
|
|
ctx.poolMode = true # Fast sync mode must be synchronized among all peers
|
|
return true
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions, snap sync admin handlers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc snapSyncSetup(ctx: SnapCtxRef) =
|
|
# Set up snap sync descriptor
|
|
ctx.pool.pass = SnapPassCtxRef()
|
|
|
|
# For snap sync book keeping
|
|
ctx.pool.pass.coveredAccounts = NodeTagRangeSet.init()
|
|
ctx.pool.ticker.init(cb = ctx.pool.pass.pivotTable.tickerStats(ctx))
|
|
|
|
ctx.enableRpcMagic() # Allow external pivot update via RPC
|
|
ctx.disableWireServices() # Stop unwanted public services
|
|
ctx.detectSnapSyncRecovery() # Check for recovery mode
|
|
|
|
proc snapSyncRelease(ctx: SnapCtxRef) =
|
|
ctx.disableRpcMagic() # Disable external pivot update via RPC
|
|
ctx.enableWireServices() # re-enable public services
|
|
ctx.pool.ticker.stop()
|
|
|
|
proc snapSyncStart(buddy: SnapBuddyRef): bool =
|
|
let
|
|
ctx = buddy.ctx
|
|
peer = buddy.peer
|
|
if peer.supports(protocol.snap) and
|
|
peer.supports(protocol.eth) and
|
|
peer.state(protocol.eth).initialized:
|
|
ctx.pool.ticker.startBuddy()
|
|
buddy.ctrl.multiOk = false # confirm default mode for soft restart
|
|
buddy.only.errors = GetErrorStatsRef()
|
|
return true
|
|
|
|
proc snapSyncStop(buddy: SnapBuddyRef) =
|
|
buddy.ctx.pool.ticker.stopBuddy()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions, snap sync action handlers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc snapSyncPool(buddy: SnapBuddyRef, last: bool, laps: int): bool =
|
|
## Enabled when `buddy.ctrl.poolMode` is `true`
|
|
##
|
|
let
|
|
ctx = buddy.ctx
|
|
snap = ctx.pool.pass
|
|
env = snap.completedPivot
|
|
|
|
# Check whether the snapshot is complete. If so, switch to full sync mode.
|
|
# This process needs to be applied to all buddy peers.
|
|
if not env.isNil:
|
|
ignoreException("snapSyncPool"):
|
|
# Stop all peers
|
|
buddy.snapSyncStop()
|
|
# After the last buddy peer was stopped switch to full sync mode
|
|
# and repeat that loop over buddy peers for re-starting them.
|
|
if last:
|
|
when extraTraceMessages:
|
|
trace logTxt "switch to full sync", peer=buddy.peer, last, laps,
|
|
pivot=env.stateHeader.blockNumber.toStr,
|
|
mode=ctx.pool.syncMode.active, state= buddy.ctrl.state
|
|
ctx.snapSyncRelease()
|
|
ctx.pool.syncMode.active = FullSyncMode
|
|
ctx.passActor.setup(ctx)
|
|
ctx.poolMode = true # repeat looping over peers
|
|
ctx.pool.fullHeader = some(env.stateHeader) # Full sync start here
|
|
|
|
return false # do stop magically when looping over peers is exhausted
|
|
|
|
# Clean up empty pivot slots (never the top one.) This needs to be run on
|
|
# a single peer only. So the loop can stop immediately (returning `true`)
|
|
# after this job is done.
|
|
var rc = snap.pivotTable.beforeLast
|
|
while rc.isOK:
|
|
let (key, env) = (rc.value.key, rc.value.data)
|
|
if env.fetchAccounts.processed.isEmpty:
|
|
snap.pivotTable.del key
|
|
rc = snap.pivotTable.prev(key)
|
|
true # Stop ok
|
|
|
|
|
|
proc snapSyncDaemon(ctx: SnapCtxRef) {.async.} =
|
|
## Enabled while `ctx.daemon` is `true`
|
|
##
|
|
if not ctx.pool.pass.recovery.isNil:
|
|
if not await ctx.recoveryStepContinue():
|
|
# Done, stop recovery
|
|
ctx.pool.pass.recovery = nil
|
|
ctx.daemon = false
|
|
|
|
# Update logging
|
|
if not ctx.pool.ticker.isNil:
|
|
ctx.pool.ticker.stopRecovery()
|
|
|
|
|
|
proc snapSyncSingle(buddy: SnapBuddyRef) {.async.} =
|
|
## Enabled while
|
|
## * `buddy.ctrl.multiOk` is `false`
|
|
## * `buddy.ctrl.poolMode` is `false`
|
|
##
|
|
# External beacon header updater
|
|
await buddy.beaconHeaderUpdateFromFile()
|
|
|
|
# Dedicate some process cycles to the recovery process (if any)
|
|
if not buddy.ctx.pool.pass.recovery.isNil:
|
|
when extraTraceMessages:
|
|
trace "Throttling single mode in favour of recovery", peer=buddy.peer
|
|
await sleepAsync 900.milliseconds
|
|
|
|
await buddy.pivotApprovePeer()
|
|
buddy.ctrl.multiOk = true
|
|
|
|
|
|
proc snapSyncMulti(buddy: SnapBuddyRef): Future[void] {.async.} =
|
|
## Enabled while
|
|
## * `buddy.ctx.multiOk` is `true`
|
|
## * `buddy.ctx.poolMode` is `false`
|
|
##
|
|
let
|
|
ctx = buddy.ctx
|
|
|
|
# Fetch latest state root environment
|
|
env = block:
|
|
let rc = ctx.pool.pass.pivotTable.lastValue
|
|
if rc.isErr:
|
|
buddy.ctrl.multiOk = false
|
|
return # nothing to do
|
|
rc.value
|
|
|
|
# Check whether this pivot is fully downloaded
|
|
if await env.snapSyncCompleteOk(ctx):
|
|
return
|
|
|
|
# If this is a new snap sync pivot, the previous one can be cleaned up and
|
|
# archived. There is no point in keeping some older space consuming state
|
|
# data any longer.
|
|
ctx.pool.pass.pivotTable.beforeTopMostlyClean()
|
|
|
|
let
|
|
peer = buddy.peer
|
|
pivot = env.stateHeader.blockNumber.toStr # for logging
|
|
fa = env.fetchAccounts
|
|
|
|
when extraTraceMessages:
|
|
trace "Multi sync runner", peer, pivot, nAccounts=env.nAccounts,
|
|
processed=fa.processed.fullPC3, nStoQ=env.storageQueueTotal(),
|
|
nSlotLists=env.nSlotLists
|
|
|
|
# This one is the syncing work horse which downloads the database
|
|
await env.execSnapSyncAction(buddy)
|
|
|
|
# Various logging entries (after accounts and storage slots download)
|
|
let
|
|
nAccounts = env.nAccounts
|
|
nSlotLists = env.nSlotLists
|
|
processed = fa.processed.fullPC3
|
|
|
|
# Archive this pivot eveironment if it has become stale
|
|
if env.archived:
|
|
when extraTraceMessages:
|
|
trace logTxt "mothballing", peer, pivot, nAccounts, nSlotLists
|
|
env.pivotMothball()
|
|
return
|
|
|
|
# Save state so sync can be resumed at next start up
|
|
let rc = env.saveCheckpoint(ctx)
|
|
if rc.isOk:
|
|
when extraTraceMessages:
|
|
trace logTxt "saved checkpoint", peer, pivot, nAccounts,
|
|
processed, nStoQ=env.storageQueueTotal(), nSlotLists,
|
|
blobSize=rc.value
|
|
return
|
|
|
|
error logTxt "failed to save checkpoint", peer, pivot, nAccounts,
|
|
processed, nStoQ=env.storageQueueTotal(), nSlotLists,
|
|
error=rc.error
|
|
|
|
# Check whether this pivot is fully downloaded
|
|
discard await env.snapSyncCompleteOk(ctx)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc passSnap*: auto =
|
|
## Return snap sync handler environment
|
|
PassActorRef(
|
|
setup: snapSyncSetup,
|
|
release: snapSyncRelease,
|
|
start: snapSyncStart,
|
|
stop: snapSyncStop,
|
|
pool: snapSyncPool,
|
|
daemon: snapSyncDaemon,
|
|
single: snapSyncSingle,
|
|
multi: snapSyncMulti)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|