add ncli_db subcommand to prune database of unnecessary blocks and states (#1593)

* add ncli_db subcommand to prune database of unnecessary blocks, states, and state roots

* tweak comments

* reduce default aggressiveness in pruning old states

* move copyPrunedDatabase() to ncli_db, as it's not generally useful as part of beacon_chain_db and doesn't use any internal interfaces

beacon_chain/block_pools/block_pools_types.nim

@@ -177,9 +177,7 @@ type
     data*: HashedBeaconState
 
     blck*: BlockRef ##\
-    ## The block associated with the state found in data - normally
-    ## `blck.state_root == data.root` but the state might have been advanced
-    ## further with empty slots invalidating this condition.
+    ## The block associated with the state found in data
 
   BlockSlot* = object
     ## Unique identifier for a particular fork and time in the block chain -

ncli/ncli_db.nim

@@ -22,6 +22,7 @@ type
     bench
     dumpState
     dumpBlock
+    pruneDatabase
     rewindState
 
   # TODO:
@@ -59,6 +60,17 @@ type
         argument
         desc: "Block roots to save".}: seq[string]
 
+    of pruneDatabase:
+      dryRun* {.
+        defaultValue: false
+        desc: "Don't write to the database copy; only simulate actions; default false".}: bool
+      keepOldStates* {.
+        defaultValue: true
+        desc: "Keep pre-finalization states; default true".}: bool
+      verbose* {.
+        defaultValue: false
+        desc: "Enables verbose output; default false".}: bool
+
     of rewindState:
       blockRoot* {.
         argument
@@ -157,6 +169,85 @@ proc cmdDumpBlock(conf: DbConf) =
     except CatchableError as e:
       echo "Couldn't load ", blockRoot, ": ", e.msg
 
+proc copyPrunedDatabase(
+    db: BeaconChainDB, copyDb: BeaconChainDB,
+    dryRun, verbose, keepOldStates: bool) =
+  ## Create a pruned copy of the beacon chain database
+
+  let
+    headBlock = db.getHeadBlock()
+    tailBlock = db.getTailBlock()
+
+  doAssert headBlock.isOk and tailBlock.isOk
+  doAssert db.getBlock(headBlock.get).isOk
+  doAssert db.getBlock(tailBlock.get).isOk
+
+  var
+    beaconState: ref BeaconState
+    finalizedEpoch: Epoch  # default value of 0 is conservative/safe
+    prevBlockSlot = db.getBlock(db.getHeadBlock().get).get.message.slot
+
+  beaconState = new BeaconState
+  let headEpoch = db.getBlock(headBlock.get).get.message.slot.epoch
+
+  # Tail states are specially addressed; no stateroot intermediary
+  if not db.getState(
+      db.getBlock(tailBlock.get).get.message.state_root, beaconState[],
+      noRollback):
+    doAssert false, "could not load tail state"
+  if not dry_run:
+    copyDb.putState(beaconState[])
+
+  for signedBlock in getAncestors(db, headBlock.get):
+    if not dry_run:
+      copyDb.putBlock(signedBlock)
+    if verbose:
+      echo "copied block at slot ", signedBlock.message.slot
+
+    for slot in countdown(prevBlockSlot, signedBlock.message.slot + 1):
+      if slot mod SLOTS_PER_EPOCH != 0 or
+          ((not keepOldStates) and slot.epoch < finalizedEpoch):
+        continue
+
+      # Could also only copy these states, head and finalized, plus tail state
+      let stateRequired = slot.epoch in [finalizedEpoch, headEpoch]
+
+      let sr = db.getStateRoot(signedBlock.root, slot)
+      if sr.isErr:
+        if stateRequired:
+          doAssert false, "state root and state required"
+        continue
+
+      if not db.getState(sr.get, beaconState[], noRollback):
+        # Don't copy dangling stateroot pointers
+        if stateRequired:
+          doAssert false, "state root and state required"
+        continue
+
+      finalizedEpoch = max(
+        finalizedEpoch, beaconState.finalized_checkpoint.epoch)
+
+      if not dry_run:
+        copyDb.putStateRoot(signedBlock.root, slot, sr.get)
+        copyDb.putState(beaconState[])
+      if verbose:
+        echo "copied state at slot ", slot, " from block at ", shortLog(signedBlock.message.slot)
+
+    prevBlockSlot = signedBlock.message.slot
+
+  if not dry_run:
+    copyDb.putHeadBlock(headBlock.get)
+    copyDb.putTailBlock(tailBlock.get)
+
+proc cmdPrune(conf: DbConf) =
+  let
+    db = BeaconChainDB.init(
+      kvStore SqStoreRef.init(conf.databaseDir.string, "nbc").tryGet())
+    copyDb = BeaconChainDB.init(
+      kvStore SqStoreRef.init(conf.databaseDir.string, "nbc_pruned").tryGet())
+
+  db.copyPrunedDatabase(copyDb, conf.dryRun, conf.verbose, conf.keepOldStates)
+
 proc cmdRewindState(conf: DbConf, runtimePreset: RuntimePreset) =
   echo "Opening database..."
   let
@@ -191,5 +282,7 @@ when isMainModule:
     cmdDumpState(conf)
   of dumpBlock:
     cmdDumpBlock(conf)
+  of pruneDatabase:
+    cmdPrune(conf)
   of rewindState:
     cmdRewindState(conf, runtimePreset)