add ncli_db subcommand to prune database of unnecessary blocks and states (#1593)
* add ncli_db subcommand to prune database of unnecessary blocks, states, and state roots * tweak comments * reduce default aggressiveness in pruning old states * move copyPrunedDatabase() to ncli_db, as it's not generally useful as part of beacon_chain_db and doesn't use any internal interfaces
This commit is contained in:
parent
c7c9b9d5f1
commit
48893f1c2e
|
@ -177,9 +177,7 @@ type
|
|||
data*: HashedBeaconState
|
||||
|
||||
blck*: BlockRef ##\
|
||||
## The block associated with the state found in data - normally
|
||||
## `blck.state_root == data.root` but the state might have been advanced
|
||||
## further with empty slots invalidating this condition.
|
||||
## The block associated with the state found in data
|
||||
|
||||
BlockSlot* = object
|
||||
## Unique identifier for a particular fork and time in the block chain -
|
||||
|
|
|
@ -22,6 +22,7 @@ type
|
|||
bench
|
||||
dumpState
|
||||
dumpBlock
|
||||
pruneDatabase
|
||||
rewindState
|
||||
|
||||
# TODO:
|
||||
|
@ -59,6 +60,17 @@ type
|
|||
argument
|
||||
desc: "Block roots to save".}: seq[string]
|
||||
|
||||
of pruneDatabase:
|
||||
dryRun* {.
|
||||
defaultValue: false
|
||||
desc: "Don't write to the database copy; only simulate actions; default false".}: bool
|
||||
keepOldStates* {.
|
||||
defaultValue: true
|
||||
desc: "Keep pre-finalization states; default true".}: bool
|
||||
verbose* {.
|
||||
defaultValue: false
|
||||
desc: "Enables verbose output; default false".}: bool
|
||||
|
||||
of rewindState:
|
||||
blockRoot* {.
|
||||
argument
|
||||
|
@ -157,6 +169,85 @@ proc cmdDumpBlock(conf: DbConf) =
|
|||
except CatchableError as e:
|
||||
echo "Couldn't load ", blockRoot, ": ", e.msg
|
||||
|
||||
proc copyPrunedDatabase(
|
||||
db: BeaconChainDB, copyDb: BeaconChainDB,
|
||||
dryRun, verbose, keepOldStates: bool) =
|
||||
## Create a pruned copy of the beacon chain database
|
||||
|
||||
let
|
||||
headBlock = db.getHeadBlock()
|
||||
tailBlock = db.getTailBlock()
|
||||
|
||||
doAssert headBlock.isOk and tailBlock.isOk
|
||||
doAssert db.getBlock(headBlock.get).isOk
|
||||
doAssert db.getBlock(tailBlock.get).isOk
|
||||
|
||||
var
|
||||
beaconState: ref BeaconState
|
||||
finalizedEpoch: Epoch # default value of 0 is conservative/safe
|
||||
prevBlockSlot = db.getBlock(db.getHeadBlock().get).get.message.slot
|
||||
|
||||
beaconState = new BeaconState
|
||||
let headEpoch = db.getBlock(headBlock.get).get.message.slot.epoch
|
||||
|
||||
# Tail states are specially addressed; no stateroot intermediary
|
||||
if not db.getState(
|
||||
db.getBlock(tailBlock.get).get.message.state_root, beaconState[],
|
||||
noRollback):
|
||||
doAssert false, "could not load tail state"
|
||||
if not dry_run:
|
||||
copyDb.putState(beaconState[])
|
||||
|
||||
for signedBlock in getAncestors(db, headBlock.get):
|
||||
if not dry_run:
|
||||
copyDb.putBlock(signedBlock)
|
||||
if verbose:
|
||||
echo "copied block at slot ", signedBlock.message.slot
|
||||
|
||||
for slot in countdown(prevBlockSlot, signedBlock.message.slot + 1):
|
||||
if slot mod SLOTS_PER_EPOCH != 0 or
|
||||
((not keepOldStates) and slot.epoch < finalizedEpoch):
|
||||
continue
|
||||
|
||||
# Could also only copy these states, head and finalized, plus tail state
|
||||
let stateRequired = slot.epoch in [finalizedEpoch, headEpoch]
|
||||
|
||||
let sr = db.getStateRoot(signedBlock.root, slot)
|
||||
if sr.isErr:
|
||||
if stateRequired:
|
||||
doAssert false, "state root and state required"
|
||||
continue
|
||||
|
||||
if not db.getState(sr.get, beaconState[], noRollback):
|
||||
# Don't copy dangling stateroot pointers
|
||||
if stateRequired:
|
||||
doAssert false, "state root and state required"
|
||||
continue
|
||||
|
||||
finalizedEpoch = max(
|
||||
finalizedEpoch, beaconState.finalized_checkpoint.epoch)
|
||||
|
||||
if not dry_run:
|
||||
copyDb.putStateRoot(signedBlock.root, slot, sr.get)
|
||||
copyDb.putState(beaconState[])
|
||||
if verbose:
|
||||
echo "copied state at slot ", slot, " from block at ", shortLog(signedBlock.message.slot)
|
||||
|
||||
prevBlockSlot = signedBlock.message.slot
|
||||
|
||||
if not dry_run:
|
||||
copyDb.putHeadBlock(headBlock.get)
|
||||
copyDb.putTailBlock(tailBlock.get)
|
||||
|
||||
proc cmdPrune(conf: DbConf) =
|
||||
let
|
||||
db = BeaconChainDB.init(
|
||||
kvStore SqStoreRef.init(conf.databaseDir.string, "nbc").tryGet())
|
||||
copyDb = BeaconChainDB.init(
|
||||
kvStore SqStoreRef.init(conf.databaseDir.string, "nbc_pruned").tryGet())
|
||||
|
||||
db.copyPrunedDatabase(copyDb, conf.dryRun, conf.verbose, conf.keepOldStates)
|
||||
|
||||
proc cmdRewindState(conf: DbConf, runtimePreset: RuntimePreset) =
|
||||
echo "Opening database..."
|
||||
let
|
||||
|
@ -191,5 +282,7 @@ when isMainModule:
|
|||
cmdDumpState(conf)
|
||||
of dumpBlock:
|
||||
cmdDumpBlock(conf)
|
||||
of pruneDatabase:
|
||||
cmdPrune(conf)
|
||||
of rewindState:
|
||||
cmdRewindState(conf, runtimePreset)
|
||||
|
|
Loading…
Reference in New Issue