2020-10-12 01:07:20 +00:00
|
|
|
import
|
|
|
|
os, stats, strformat, tables,
|
|
|
|
chronicles, confutils, stew/byteutils, eth/db/kvstore_sqlite3,
|
2021-03-05 13:12:00 +00:00
|
|
|
../beacon_chain/networking/network_metadata,
|
2020-07-30 19:18:17 +00:00
|
|
|
../beacon_chain/[beacon_chain_db, extras],
|
2021-03-04 09:13:44 +00:00
|
|
|
../beacon_chain/consensus_object_pools/blockchain_dag,
|
2020-07-07 23:02:14 +00:00
|
|
|
../beacon_chain/spec/[crypto, datatypes, digest, helpers,
|
2020-07-28 13:54:32 +00:00
|
|
|
state_transition, presets],
|
2021-03-03 06:23:05 +00:00
|
|
|
../beacon_chain/ssz, ../beacon_chain/ssz/sszdump,
|
2020-10-15 11:49:02 +00:00
|
|
|
../research/simutils
|
2020-05-28 14:19:25 +00:00
|
|
|
|
|
|
|
type Timers = enum
|
|
|
|
tInit = "Initialize DB"
|
|
|
|
tLoadBlock = "Load block from database"
|
|
|
|
tLoadState = "Load state from database"
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
tAdvanceSlot = "Advance slot, non-epoch"
|
|
|
|
tAdvanceEpoch = "Advance slot, epoch"
|
|
|
|
tApplyBlock = "Apply block, no slot processing"
|
2021-02-15 16:40:00 +00:00
|
|
|
tDbStore = "Database store"
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
type
|
|
|
|
DbCmd* = enum
|
|
|
|
bench
|
2020-06-06 11:26:19 +00:00
|
|
|
dumpState
|
2020-06-25 10:23:10 +00:00
|
|
|
dumpBlock
|
2020-09-11 13:20:34 +00:00
|
|
|
pruneDatabase
|
2020-06-16 08:49:32 +00:00
|
|
|
rewindState
|
2020-06-01 14:48:24 +00:00
|
|
|
|
2020-07-07 23:02:14 +00:00
|
|
|
# TODO:
|
|
|
|
# This should probably allow specifying a run-time preset
|
2020-06-01 14:48:24 +00:00
|
|
|
DbConf = object
|
|
|
|
databaseDir* {.
|
|
|
|
defaultValue: ""
|
|
|
|
desc: "Directory where `nbc.sqlite` is stored"
|
|
|
|
name: "db" }: InputDir
|
|
|
|
|
2020-09-01 09:01:57 +00:00
|
|
|
eth2Network* {.
|
|
|
|
desc: "The Eth2 network preset to use"
|
|
|
|
name: "network" }: Option[string]
|
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
case cmd* {.
|
|
|
|
command
|
2020-06-06 11:26:19 +00:00
|
|
|
desc: ""
|
2020-06-01 14:48:24 +00:00
|
|
|
.}: DbCmd
|
|
|
|
|
|
|
|
of bench:
|
2020-06-25 10:23:10 +00:00
|
|
|
slots* {.
|
|
|
|
defaultValue: 50000
|
|
|
|
desc: "Number of slots to run benchmark for".}: uint64
|
2020-08-27 12:52:22 +00:00
|
|
|
storeBlocks* {.
|
|
|
|
defaultValue: false
|
|
|
|
desc: "Store each read block back into a separate database".}: bool
|
2021-02-15 16:40:00 +00:00
|
|
|
storeStates* {.
|
|
|
|
defaultValue: false
|
|
|
|
desc: "Store a state each epoch into a separate database".}: bool
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
printTimes* {.
|
|
|
|
defaultValue: true
|
|
|
|
desc: "Print csv of block processing time".}: bool
|
|
|
|
resetCache* {.
|
|
|
|
defaultValue: false
|
|
|
|
desc: "Process each block with a fresh cache".}: bool
|
2020-06-06 11:26:19 +00:00
|
|
|
of dumpState:
|
|
|
|
stateRoot* {.
|
|
|
|
argument
|
|
|
|
desc: "State roots to save".}: seq[string]
|
|
|
|
|
2020-06-25 10:23:10 +00:00
|
|
|
of dumpBlock:
|
|
|
|
blockRootx* {.
|
|
|
|
argument
|
|
|
|
desc: "Block roots to save".}: seq[string]
|
|
|
|
|
2020-09-11 13:20:34 +00:00
|
|
|
of pruneDatabase:
|
|
|
|
dryRun* {.
|
|
|
|
defaultValue: false
|
|
|
|
desc: "Don't write to the database copy; only simulate actions; default false".}: bool
|
|
|
|
keepOldStates* {.
|
|
|
|
defaultValue: true
|
|
|
|
desc: "Keep pre-finalization states; default true".}: bool
|
|
|
|
verbose* {.
|
|
|
|
defaultValue: false
|
|
|
|
desc: "Enables verbose output; default false".}: bool
|
|
|
|
|
2020-06-16 08:49:32 +00:00
|
|
|
of rewindState:
|
|
|
|
blockRoot* {.
|
|
|
|
argument
|
|
|
|
desc: "Block root".}: string
|
|
|
|
|
|
|
|
slot* {.
|
|
|
|
argument
|
|
|
|
desc: "Slot".}: uint64
|
|
|
|
|
2020-09-01 09:01:57 +00:00
|
|
|
proc cmdBench(conf: DbConf, runtimePreset: RuntimePreset) =
|
2020-05-28 14:19:25 +00:00
|
|
|
var timers: array[Timers, RunningStat]
|
|
|
|
|
|
|
|
echo "Opening database..."
|
|
|
|
let
|
2020-10-15 11:49:02 +00:00
|
|
|
db = BeaconChainDB.init(runtimePreset, conf.databaseDir.string)
|
|
|
|
dbBenchmark = BeaconChainDB.init(runtimePreset, "benchmark")
|
2021-02-15 16:40:00 +00:00
|
|
|
defer:
|
|
|
|
db.close()
|
|
|
|
dbBenchmark.close()
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-07-31 14:49:06 +00:00
|
|
|
if not ChainDAGRef.isInitialized(db):
|
2020-05-28 14:19:25 +00:00
|
|
|
echo "Database not initialized"
|
|
|
|
quit 1
|
|
|
|
|
|
|
|
echo "Initializing block pool..."
|
|
|
|
let pool = withTimerRet(timers[tInit]):
|
2020-09-01 09:01:57 +00:00
|
|
|
ChainDAGRef.init(runtimePreset, db, {})
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-07-28 13:54:32 +00:00
|
|
|
echo &"Loaded {pool.blocks.len} blocks, head slot {pool.head.slot}"
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
var
|
|
|
|
blockRefs: seq[BlockRef]
|
2020-06-25 10:23:10 +00:00
|
|
|
blocks: seq[TrustedSignedBeaconBlock]
|
2020-07-28 13:54:32 +00:00
|
|
|
cur = pool.head
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
while cur != nil:
|
|
|
|
blockRefs.add cur
|
|
|
|
cur = cur.parent
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
for b in 1..<blockRefs.len: # Skip genesis block
|
2020-06-25 10:23:10 +00:00
|
|
|
if blockRefs[blockRefs.len - b - 1].slot > conf.slots:
|
|
|
|
break
|
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
withTimer(timers[tLoadBlock]):
|
|
|
|
blocks.add db.getBlock(blockRefs[blockRefs.len - b - 1].root).get()
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
let state = (ref HashedBeaconState)(
|
|
|
|
root: db.getBlock(blockRefs[^1].root).get().message.state_root
|
|
|
|
)
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
withTimer(timers[tLoadState]):
|
|
|
|
discard db.getState(state[].root, state[].data, noRollback)
|
2020-05-28 14:19:25 +00:00
|
|
|
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
var cache = StateCache()
|
|
|
|
|
|
|
|
for b in blocks.mitems():
|
|
|
|
while state[].data.slot < b.message.slot:
|
|
|
|
let isEpoch = state[].data.slot.epoch() != (state[].data.slot + 1).epoch
|
|
|
|
withTimer(timers[if isEpoch: tAdvanceEpoch else: tAdvanceSlot]):
|
|
|
|
let ok = process_slots(state[], state[].data.slot + 1, cache, {})
|
|
|
|
doAssert ok, "Slot processing can't fail with correct inputs"
|
|
|
|
|
|
|
|
var start = Moment.now()
|
|
|
|
withTimer(timers[tApplyBlock]):
|
|
|
|
if conf.resetCache:
|
|
|
|
cache = StateCache()
|
|
|
|
if not state_transition(
|
|
|
|
runtimePreset, state[], b, cache, {slotProcessed}, noRollback):
|
2020-07-16 13:16:51 +00:00
|
|
|
dump("./", b)
|
2020-06-25 10:23:10 +00:00
|
|
|
echo "State transition failed (!)"
|
|
|
|
quit 1
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
if conf.printTimes:
|
|
|
|
echo b.message.slot, ",", toHex(b.root.data), ",", nanoseconds(Moment.now() - start)
|
2020-08-27 12:52:22 +00:00
|
|
|
if conf.storeBlocks:
|
|
|
|
withTimer(timers[tDbStore]):
|
|
|
|
dbBenchmark.putBlock(b)
|
2020-06-01 14:48:24 +00:00
|
|
|
|
2021-02-15 16:40:00 +00:00
|
|
|
if conf.storeStates:
|
|
|
|
withTimer(timers[tDbStore]):
|
|
|
|
if state[].data.slot mod SLOTS_PER_EPOCH == 0:
|
|
|
|
dbBenchmark.putState(state[].root, state[].data)
|
|
|
|
dbBenchmark.checkpoint()
|
|
|
|
|
2020-06-25 10:23:10 +00:00
|
|
|
printTimers(false, timers)
|
2020-06-01 14:48:24 +00:00
|
|
|
|
2020-10-15 11:49:02 +00:00
|
|
|
proc cmdDumpState(conf: DbConf, preset: RuntimePreset) =
|
|
|
|
let db = BeaconChainDB.init(preset, conf.databaseDir.string)
|
2020-09-12 05:35:58 +00:00
|
|
|
defer: db.close()
|
2020-06-06 11:26:19 +00:00
|
|
|
|
|
|
|
for stateRoot in conf.stateRoot:
|
|
|
|
try:
|
|
|
|
let root = Eth2Digest(data: hexToByteArray[32](stateRoot))
|
|
|
|
var state = (ref HashedBeaconState)(root: root)
|
|
|
|
if not db.getState(root, state.data, noRollback):
|
|
|
|
echo "Couldn't load ", root
|
|
|
|
else:
|
|
|
|
dump("./", state[])
|
|
|
|
except CatchableError as e:
|
|
|
|
echo "Couldn't load ", stateRoot, ": ", e.msg
|
|
|
|
|
2020-10-15 11:49:02 +00:00
|
|
|
proc cmdDumpBlock(conf: DbConf, preset: RuntimePreset) =
|
|
|
|
let db = BeaconChainDB.init(preset, conf.databaseDir.string)
|
2020-09-12 05:35:58 +00:00
|
|
|
defer: db.close()
|
2020-06-25 10:23:10 +00:00
|
|
|
|
|
|
|
for blockRoot in conf.blockRootx:
|
|
|
|
try:
|
|
|
|
let root = Eth2Digest(data: hexToByteArray[32](blockRoot))
|
|
|
|
if (let blck = db.getBlock(root); blck.isSome):
|
2020-07-16 13:16:51 +00:00
|
|
|
dump("./", blck.get())
|
2020-06-25 10:23:10 +00:00
|
|
|
else:
|
|
|
|
echo "Couldn't load ", root
|
|
|
|
except CatchableError as e:
|
|
|
|
echo "Couldn't load ", blockRoot, ": ", e.msg
|
|
|
|
|
2020-09-11 13:20:34 +00:00
|
|
|
proc copyPrunedDatabase(
|
|
|
|
db: BeaconChainDB, copyDb: BeaconChainDB,
|
|
|
|
dryRun, verbose, keepOldStates: bool) =
|
|
|
|
## Create a pruned copy of the beacon chain database
|
|
|
|
|
|
|
|
let
|
|
|
|
headBlock = db.getHeadBlock()
|
|
|
|
tailBlock = db.getTailBlock()
|
|
|
|
|
|
|
|
doAssert headBlock.isOk and tailBlock.isOk
|
|
|
|
doAssert db.getBlock(headBlock.get).isOk
|
|
|
|
doAssert db.getBlock(tailBlock.get).isOk
|
|
|
|
|
|
|
|
var
|
|
|
|
beaconState: ref BeaconState
|
|
|
|
finalizedEpoch: Epoch # default value of 0 is conservative/safe
|
|
|
|
prevBlockSlot = db.getBlock(db.getHeadBlock().get).get.message.slot
|
|
|
|
|
|
|
|
beaconState = new BeaconState
|
|
|
|
let headEpoch = db.getBlock(headBlock.get).get.message.slot.epoch
|
|
|
|
|
|
|
|
# Tail states are specially addressed; no stateroot intermediary
|
|
|
|
if not db.getState(
|
|
|
|
db.getBlock(tailBlock.get).get.message.state_root, beaconState[],
|
|
|
|
noRollback):
|
|
|
|
doAssert false, "could not load tail state"
|
|
|
|
if not dry_run:
|
|
|
|
copyDb.putState(beaconState[])
|
|
|
|
|
|
|
|
for signedBlock in getAncestors(db, headBlock.get):
|
|
|
|
if not dry_run:
|
|
|
|
copyDb.putBlock(signedBlock)
|
|
|
|
if verbose:
|
|
|
|
echo "copied block at slot ", signedBlock.message.slot
|
|
|
|
|
|
|
|
for slot in countdown(prevBlockSlot, signedBlock.message.slot + 1):
|
|
|
|
if slot mod SLOTS_PER_EPOCH != 0 or
|
|
|
|
((not keepOldStates) and slot.epoch < finalizedEpoch):
|
|
|
|
continue
|
|
|
|
|
|
|
|
# Could also only copy these states, head and finalized, plus tail state
|
|
|
|
let stateRequired = slot.epoch in [finalizedEpoch, headEpoch]
|
|
|
|
|
|
|
|
let sr = db.getStateRoot(signedBlock.root, slot)
|
|
|
|
if sr.isErr:
|
|
|
|
if stateRequired:
|
2020-09-29 15:15:49 +00:00
|
|
|
echo "skipping state root required for slot ",
|
|
|
|
slot, " with root ", signedBlock.root
|
2020-09-11 13:20:34 +00:00
|
|
|
continue
|
|
|
|
|
|
|
|
if not db.getState(sr.get, beaconState[], noRollback):
|
|
|
|
# Don't copy dangling stateroot pointers
|
|
|
|
if stateRequired:
|
|
|
|
doAssert false, "state root and state required"
|
|
|
|
continue
|
|
|
|
|
|
|
|
finalizedEpoch = max(
|
|
|
|
finalizedEpoch, beaconState.finalized_checkpoint.epoch)
|
|
|
|
|
|
|
|
if not dry_run:
|
|
|
|
copyDb.putStateRoot(signedBlock.root, slot, sr.get)
|
|
|
|
copyDb.putState(beaconState[])
|
|
|
|
if verbose:
|
|
|
|
echo "copied state at slot ", slot, " from block at ", shortLog(signedBlock.message.slot)
|
|
|
|
|
|
|
|
prevBlockSlot = signedBlock.message.slot
|
|
|
|
|
|
|
|
if not dry_run:
|
|
|
|
copyDb.putHeadBlock(headBlock.get)
|
|
|
|
copyDb.putTailBlock(tailBlock.get)
|
|
|
|
|
2020-10-15 11:49:02 +00:00
|
|
|
proc cmdPrune(conf: DbConf, preset: RuntimePreset) =
|
2020-09-11 13:20:34 +00:00
|
|
|
let
|
2020-10-15 11:49:02 +00:00
|
|
|
db = BeaconChainDB.init(preset, conf.databaseDir.string)
|
2020-10-12 01:07:20 +00:00
|
|
|
# TODO: add the destination as CLI paramter
|
2020-10-15 11:49:02 +00:00
|
|
|
copyDb = BeaconChainDB.init(preset, "pruned_db")
|
2020-09-11 13:20:34 +00:00
|
|
|
|
2020-09-12 05:35:58 +00:00
|
|
|
defer:
|
|
|
|
db.close()
|
|
|
|
copyDb.close()
|
|
|
|
|
2020-09-11 13:20:34 +00:00
|
|
|
db.copyPrunedDatabase(copyDb, conf.dryRun, conf.verbose, conf.keepOldStates)
|
|
|
|
|
2020-10-15 11:49:02 +00:00
|
|
|
proc cmdRewindState(conf: DbConf, preset: RuntimePreset) =
|
2020-06-16 08:49:32 +00:00
|
|
|
echo "Opening database..."
|
2020-10-15 11:49:02 +00:00
|
|
|
let db = BeaconChainDB.init(preset, conf.databaseDir.string)
|
2020-09-12 05:35:58 +00:00
|
|
|
defer: db.close()
|
2020-06-16 08:49:32 +00:00
|
|
|
|
2020-07-31 14:49:06 +00:00
|
|
|
if not ChainDAGRef.isInitialized(db):
|
2020-06-16 08:49:32 +00:00
|
|
|
echo "Database not initialized"
|
|
|
|
quit 1
|
|
|
|
|
|
|
|
echo "Initializing block pool..."
|
2020-10-15 11:49:02 +00:00
|
|
|
let dag = init(ChainDAGRef, preset, db)
|
2020-06-16 08:49:32 +00:00
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
let blckRef = dag.getRef(fromHex(Eth2Digest, conf.blockRoot))
|
2020-06-16 08:49:32 +00:00
|
|
|
if blckRef == nil:
|
|
|
|
echo "Block not found in database"
|
|
|
|
return
|
|
|
|
|
2020-07-30 19:18:17 +00:00
|
|
|
dag.withState(dag.tmpState, blckRef.atSlot(Slot(conf.slot))):
|
2020-06-16 08:49:32 +00:00
|
|
|
echo "Writing state..."
|
|
|
|
dump("./", hashedState, blck)
|
|
|
|
|
2020-06-01 14:48:24 +00:00
|
|
|
when isMainModule:
|
2020-09-01 09:01:57 +00:00
|
|
|
var
|
2020-06-06 11:26:19 +00:00
|
|
|
conf = DbConf.load()
|
2020-09-01 09:01:57 +00:00
|
|
|
runtimePreset = getRuntimePresetForNetwork(conf.eth2Network)
|
2020-05-28 14:19:25 +00:00
|
|
|
|
2020-06-06 11:26:19 +00:00
|
|
|
case conf.cmd
|
2020-06-01 14:48:24 +00:00
|
|
|
of bench:
|
2020-09-01 09:01:57 +00:00
|
|
|
cmdBench(conf, runtimePreset)
|
2020-06-06 11:26:19 +00:00
|
|
|
of dumpState:
|
2020-10-15 11:49:02 +00:00
|
|
|
cmdDumpState(conf, runtimePreset)
|
2020-06-25 10:23:10 +00:00
|
|
|
of dumpBlock:
|
2020-10-15 11:49:02 +00:00
|
|
|
cmdDumpBlock(conf, runtimePreset)
|
2020-09-11 13:20:34 +00:00
|
|
|
of pruneDatabase:
|
2020-10-15 11:49:02 +00:00
|
|
|
cmdPrune(conf, runtimePreset)
|
2020-06-16 08:49:32 +00:00
|
|
|
of rewindState:
|
2020-09-01 09:01:57 +00:00
|
|
|
cmdRewindState(conf, runtimePreset)
|