import: add csv debug option (#2301)

This new option saves a CSV to disk while performing `import` such that
the performance of one import can be compared with the other.

This early version is likely to change in the future
This commit is contained in:
Jacek Sieka 2024-06-06 07:03:11 +02:00 committed by GitHub
parent 32c7fe74be
commit 0268093fcc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 50 additions and 9 deletions

View File

@ -520,6 +520,11 @@ type
defaultValue: 8192 defaultValue: 8192
name: "chunk-size" .}: uint64 name: "chunk-size" .}: uint64
csvStats* {.
hidden
desc: "Save performance statistics to CSV"
name: "debug-csv-stats".}: Option[string]
func parseCmdArg(T: type NetworkId, p: string): T func parseCmdArg(T: type NetworkId, p: string): T
{.gcsafe, raises: [ValueError].} = {.gcsafe, raises: [ValueError].} =
parseInt(p).T parseInt(p).T

View File

@ -12,7 +12,7 @@
import import
chronicles, chronicles,
chronos/timer, chronos/timer,
std/strformat, std/[strformat, strutils],
stew/io2, stew/io2,
./config, ./config,
./common/common, ./common/common,
@ -35,7 +35,8 @@ func shortLog(a: timer.Duration, parts = int.high): string {.inline.} =
res.add(n) res.add(n)
v = v mod T.nanoseconds() v = v mod T.nanoseconds()
dec parts dec parts
if v == 0 or parts <= 0: return res if v == 0 or parts <= 0:
return res
f("w", Week) f("w", Week)
f("d", Day) f("d", Day)
@ -58,11 +59,12 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
setControlCHook(controlCHandler) setControlCHook(controlCHandler)
let let
start = try: start =
com.db.getSavedStateBlockNumber().truncate(uint64) + 1 try:
except RlpError as exc: com.db.getSavedStateBlockNumber().truncate(uint64) + 1
error "Could not read block number", err = exc.msg except RlpError as exc:
quit(QuitFailure) error "Could not read block number", err = exc.msg
quit(QuitFailure)
chain = com.newChain() chain = com.newChain()
@ -71,6 +73,23 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
gas = 0.u256 gas = 0.u256
txs = 0 txs = 0
time0 = Moment.now() time0 = Moment.now()
csv =
if conf.csvStats.isSome:
try:
let f = open(conf.csvStats.get(), fmAppend)
if f.getFileSize() == 0:
f.writeLine("block_number,blocks,txs,gas,time")
f
except IOError as exc:
error "Could not open statistics output file",
file = conf.csvStats, err = exc.msg
quit(QuitFailure)
else:
File(nil)
defer:
if csv != nil:
close(csv)
template blockNumber(): uint64 = template blockNumber(): uint64 =
start + imported start + imported
@ -110,7 +129,6 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
diff1 = (time2 - time1).nanoseconds().float / 1000000000 diff1 = (time2 - time1).nanoseconds().float / 1000000000
diff0 = (time2 - time0).nanoseconds().float / 1000000000 diff0 = (time2 - time0).nanoseconds().float / 1000000000
# TODO generate csv with import statistics
info "Imported blocks", info "Imported blocks",
blockNumber, blockNumber,
blocks = imported, blocks = imported,
@ -122,7 +140,25 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
avgBps = f(imported.float / diff0), avgBps = f(imported.float / diff0),
avgTps = f(txs.float / diff0), avgTps = f(txs.float / diff0),
avgGps = f(gas.truncate(uint64).float / diff0), # TODO fix truncate avgGps = f(gas.truncate(uint64).float / diff0), # TODO fix truncate
elapsed = shortLog(time2-time0, 3) elapsed = shortLog(time2 - time0, 3)
if csv != nil:
# In the CSV, we store a line for every chunk of blocks processed so
# that the file can meaningfully be appended to when restarting the
# process - this way, each sample is independent
try:
csv.writeLine(
[
$blockNumber,
$headers.len,
$statsRes[].txs,
$statsRes[].gas,
$(time2 - time1).nanoseconds(),
].join(",")
)
csv.flushFile()
except IOError as exc:
warn "Could not write csv", err = exc.msg
headers.setLen(0) headers.setLen(0)
bodies.setLen(0) bodies.setLen(0)