mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-20 17:19:19 +00:00
4c865ec884
* Unit tests update, code cosmetics * Fix segfault with zombie handling why: In order to save memory, the data records of zombie entries are removed and only the key (aka peer node) is kept. Consequently, logging these zombies can only be done by the key. * Allow to accept V2 payload without `shanghaiTime` set while syncing why: Currently, `shanghaiTime` is missing (alt least) while snap syncing. So beacon node headers can be processed regardless. Normal (aka strict) processing will be automatically restored when leaving snap sync mode.
324 lines
9.9 KiB
Nim
324 lines
9.9 KiB
Nim
# Nimbus - Fetch account and storage states from peers efficiently
|
|
#
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
{.push raises: [].}
|
|
|
|
import
|
|
std/[strformat, strutils],
|
|
chronos,
|
|
chronicles,
|
|
eth/[common, p2p],
|
|
stint,
|
|
../../utils/prettify,
|
|
../types,
|
|
./timer_helper
|
|
|
|
logScope:
|
|
topics = "tick"
|
|
|
|
type
|
|
TickerSnapStatsUpdater* = proc: TickerSnapStats {.gcsafe, raises: [].}
|
|
## Snap sync state update function
|
|
|
|
TickerFullStatsUpdater* = proc: TickerFullStats {.gcsafe, raises: [].}
|
|
## Full sync state update function
|
|
|
|
SnapDescDetails = object
|
|
## Private state descriptor
|
|
snapCb: TickerSnapStatsUpdater
|
|
recovery: bool
|
|
lastRecov: bool
|
|
lastStats: TickerSnapStats
|
|
|
|
FullDescDetails = object
|
|
## Private state descriptor
|
|
fullCb: TickerFullStatsUpdater
|
|
lastStats: TickerFullStats
|
|
|
|
TickerSnapStats* = object
|
|
## Snap sync state (see `TickerSnapStatsUpdater`)
|
|
beaconBlock*: Option[BlockNumber]
|
|
pivotBlock*: Option[BlockNumber]
|
|
nAccounts*: (float,float) ## Mean and standard deviation
|
|
accountsFill*: (float,float,float) ## Mean, standard deviation, merged total
|
|
nAccountStats*: int ## #chunks
|
|
nSlotLists*: (float,float) ## Mean and standard deviation
|
|
nContracts*: (float,float) ## Mean and standard deviation
|
|
nStorageQueue*: Option[int]
|
|
nContractQueue*: Option[int]
|
|
nQueues*: int
|
|
|
|
TickerFullStats* = object
|
|
## Full sync state (see `TickerFullStatsUpdater`)
|
|
pivotBlock*: Option[BlockNumber]
|
|
topPersistent*: BlockNumber
|
|
nextUnprocessed*: Option[BlockNumber]
|
|
nextStaged*: Option[BlockNumber]
|
|
nStagedQueue*: int
|
|
suspended*: bool
|
|
reOrg*: bool
|
|
|
|
TickerRef* = ref object
|
|
## Ticker descriptor object
|
|
nBuddies: int
|
|
logTicker: TimerCallback
|
|
started: Moment
|
|
visited: Moment
|
|
prettyPrint: proc(t: TickerRef) {.gcsafe, raises: [].}
|
|
case fullMode: bool
|
|
of false:
|
|
snap: SnapDescDetails
|
|
of true:
|
|
full: FullDescDetails
|
|
|
|
const
|
|
extraTraceMessages = false # or true
|
|
## Enabled additional logging noise
|
|
|
|
tickerStartDelay = chronos.milliseconds(100)
|
|
tickerLogInterval = chronos.seconds(1)
|
|
tickerLogSuppressMax = chronos.seconds(100)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions: pretty printing
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc pc99(val: float): string =
|
|
if 0.99 <= val and val < 1.0: "99%"
|
|
elif 0.0 < val and val <= 0.01: "1%"
|
|
else: val.toPC(0)
|
|
|
|
proc toStr(a: Option[int]): string =
|
|
if a.isNone: "n/a"
|
|
else: $a.unsafeGet
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions: printing ticker messages
|
|
# ------------------------------------------------------------------------------
|
|
|
|
template logTxt(info: static[string]): static[string] =
|
|
"Ticker " & info
|
|
|
|
template noFmtError(info: static[string]; code: untyped) =
|
|
try:
|
|
code
|
|
except ValueError as e:
|
|
raiseAssert "Inconveivable (" & info & "): " & e.msg
|
|
|
|
proc snapTicker(t: TickerRef) {.gcsafe.} =
|
|
let
|
|
data = t.snap.snapCb()
|
|
now = Moment.now()
|
|
|
|
if data != t.snap.lastStats or
|
|
t.snap.recovery != t.snap.lastRecov or
|
|
tickerLogSuppressMax < (now - t.visited):
|
|
var
|
|
nAcc, nSto, nCon: string
|
|
pv = "n/a"
|
|
let
|
|
nStoQ = data.nStorageQueue.toStr
|
|
nConQ = data.nContractQueue.toStr
|
|
bc = data.beaconBlock.toStr
|
|
recoveryDone = t.snap.lastRecov
|
|
accCov = data.accountsFill[0].pc99 &
|
|
"(" & data.accountsFill[1].pc99 & ")" &
|
|
"/" & data.accountsFill[2].pc99 &
|
|
"~" & data.nAccountStats.uint.toSI
|
|
nInst = t.nBuddies
|
|
|
|
# With `int64`, there are more than 29*10^10 years range for seconds
|
|
up = (now - t.started).seconds.uint64.toSI
|
|
mem = getTotalMem().uint.toSI
|
|
|
|
t.snap.lastStats = data
|
|
t.visited = now
|
|
t.snap.lastRecov = t.snap.recovery
|
|
|
|
if data.pivotBlock.isSome:
|
|
pv = data.pivotBlock.toStr & "/" & $data.nQueues
|
|
|
|
noFmtError("runLogTicker"):
|
|
nAcc = (&"{(data.nAccounts[0]+0.5).int64}" &
|
|
&"({(data.nAccounts[1]+0.5).int64})")
|
|
nSto = (&"{(data.nSlotLists[0]+0.5).int64}" &
|
|
&"({(data.nSlotLists[1]+0.5).int64})")
|
|
nCon = (&"{(data.nContracts[0]+0.5).int64}" &
|
|
&"({(data.nContracts[1]+0.5).int64})")
|
|
|
|
if t.snap.recovery:
|
|
info "Snap sync ticker (recovery)",
|
|
up, nInst, bc, pv, nAcc, accCov, nSto, nStoQ, nCon, nConQ, mem
|
|
elif recoveryDone:
|
|
info "Snap sync ticker (recovery done)",
|
|
up, nInst, bc, pv, nAcc, accCov, nSto, nStoQ, nCon, nConQ, mem
|
|
else:
|
|
info "Snap sync ticker",
|
|
up, nInst, bc, pv, nAcc, accCov, nSto, nStoQ, nCon, nConQ, mem
|
|
|
|
|
|
proc fullTicker(t: TickerRef) {.gcsafe.} =
|
|
let
|
|
data = t.full.fullCb()
|
|
now = Moment.now()
|
|
|
|
if data != t.full.lastStats or
|
|
tickerLogSuppressMax < (now - t.visited):
|
|
let
|
|
persistent = data.topPersistent.toStr
|
|
staged = data.nextStaged.toStr
|
|
unprocessed = data.nextUnprocessed.toStr
|
|
queued = data.nStagedQueue
|
|
reOrg = if data.reOrg: "t" else: "f"
|
|
pv = data.pivotBlock.toStr
|
|
|
|
nInst = t.nBuddies
|
|
|
|
# With `int64`, there are more than 29*10^10 years range for seconds
|
|
up = (now - t.started).seconds.uint64.toSI
|
|
mem = getTotalMem().uint.toSI
|
|
|
|
t.full.lastStats = data
|
|
t.visited = now
|
|
|
|
if data.suspended:
|
|
info "Full sync ticker (suspended)", up, nInst, pv,
|
|
persistent, unprocessed, staged, queued, reOrg, mem
|
|
else:
|
|
info "Full sync ticker", up, nInst, pv,
|
|
persistent, unprocessed, staged, queued, reOrg, mem
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions: ticking log messages
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc setLogTicker(t: TickerRef; at: Moment) {.gcsafe.}
|
|
|
|
proc runLogTicker(t: TickerRef) {.gcsafe.} =
|
|
t.prettyPrint(t)
|
|
t.setLogTicker(Moment.fromNow(tickerLogInterval))
|
|
|
|
proc setLogTicker(t: TickerRef; at: Moment) =
|
|
if t.logTicker.isNil:
|
|
when extraTraceMessages:
|
|
debug logTxt "was stopped", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
else:
|
|
t.logTicker = safeSetTimer(at, runLogTicker, t)
|
|
|
|
proc initImpl(t: TickerRef; cb: TickerSnapStatsUpdater) =
|
|
t.fullMode = false
|
|
t.prettyPrint = snapTicker
|
|
t.snap = SnapDescDetails(snapCb: cb)
|
|
|
|
proc initImpl(t: TickerRef; cb: TickerFullStatsUpdater) =
|
|
t.fullMode = true
|
|
t.prettyPrint = fullTicker
|
|
t.full = FullDescDetails(fullCb: cb)
|
|
|
|
proc startImpl(t: TickerRef) =
|
|
t.logTicker = safeSetTimer(Moment.fromNow(tickerStartDelay),runLogTicker,t)
|
|
if t.started == Moment.default:
|
|
t.started = Moment.now()
|
|
|
|
proc stopImpl(t: TickerRef) =
|
|
## Stop ticker unconditionally
|
|
t.logTicker = nil
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public constructor and start/stop functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc init*(
|
|
T: type TickerRef;
|
|
cb: TickerSnapStatsUpdater|TickerFullStatsUpdater;
|
|
): T =
|
|
## Constructor
|
|
new result
|
|
result.initImpl(cb)
|
|
|
|
proc init*(t: TickerRef; cb: TickerSnapStatsUpdater) =
|
|
## Re-initialise ticket
|
|
if not t.isNil:
|
|
t.visited.reset
|
|
if t.fullMode:
|
|
t.prettyPrint(t) # print final state for full sync
|
|
t.initImpl(cb)
|
|
|
|
proc init*(t: TickerRef; cb: TickerFullStatsUpdater) =
|
|
## Re-initialise ticket
|
|
if not t.isNil:
|
|
t.visited.reset
|
|
if not t.fullMode:
|
|
t.prettyPrint(t) # print final state for snap sync
|
|
t.initImpl(cb)
|
|
|
|
proc start*(t: TickerRef) =
|
|
## Re/start ticker unconditionally
|
|
if not t.isNil:
|
|
when extraTraceMessages:
|
|
debug logTxt "start", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
t.startImpl()
|
|
|
|
proc stop*(t: TickerRef) =
|
|
## Stop ticker unconditionally
|
|
if not t.isNil:
|
|
t.stopImpl()
|
|
when extraTraceMessages:
|
|
debug logTxt "stop", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc startBuddy*(t: TickerRef) =
|
|
## Increment buddies counter and start ticker unless running.
|
|
if not t.isNil:
|
|
if t.nBuddies <= 0:
|
|
t.nBuddies = 1
|
|
if t.fullMode or not t.snap.recovery:
|
|
t.startImpl()
|
|
when extraTraceMessages:
|
|
debug logTxt "start buddy", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
else:
|
|
t.nBuddies.inc
|
|
|
|
proc startRecovery*(t: TickerRef) =
|
|
## Ditto for recovery mode
|
|
if not t.isNil and not t.fullMode and not t.snap.recovery:
|
|
t.snap.recovery = true
|
|
if t.nBuddies <= 0:
|
|
t.startImpl()
|
|
when extraTraceMessages:
|
|
debug logTxt "start recovery", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
|
|
proc stopBuddy*(t: TickerRef) =
|
|
## Decrement buddies counter and stop ticker if there are no more registered
|
|
## buddies.
|
|
if not t.isNil:
|
|
t.nBuddies.dec
|
|
if t.nBuddies <= 0 and not t.fullMode and not t.snap.recovery:
|
|
t.nBuddies = 0
|
|
t.stopImpl()
|
|
when extraTraceMessages:
|
|
debug logTxt "stop (buddy)", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
|
|
proc stopRecovery*(t: TickerRef) =
|
|
## Ditto for recovery mode
|
|
if not t.isNil and not t.fullMode and t.snap.recovery:
|
|
t.snap.recovery = false
|
|
t.stopImpl()
|
|
when extraTraceMessages:
|
|
debug logTxt "stop (recovery)", fullMode=t.fullMode, nBuddies=t.nBuddies
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|