slashing import integrated in NBC

beacon_chain/conf.nim

@@ -38,6 +38,7 @@ type
     wallets
     record
     web3
+    slashingdb
 
   WalletsCmd* {.pure.} = enum
     create  = "Creates a new EIP-2386 wallet"
@@ -74,6 +75,13 @@ type
     sql
     file
 
+  SlashProtCmd* = enum
+    exportAll = "Export the whole validator slashing protection DB to json."
+    `import` = "Import the slashing protection json to the node."
+    `export` = "Export specified validators slashing protection data to json"
+    # migrateAll = "Export and remove the whole validator slashing protection DB."
+    # migrate = "Export and remove specified validators from Nimbus."
+
   BeaconNodeConf* = object
     logLevel* {.
       defaultValue: "INFO"
@@ -487,6 +495,20 @@ type
           desc: "The web3 provider URL to test"
           name: "url" }: Uri
 
+    of slashingdb:
+      # TODO: when this is not set, confutils crashes
+      interchangeFile* {.
+        desc: "Path to the slashing interchange file"
+        name: "interchange" .}: Option[string]
+
+      case slashingdbCmd* {.command.}: SlashProtCmd
+      of SlashProtCmd.exportAll, SlashProtCmd.`import`:
+        discard
+      of SlashProtCmd.`export`:
+        validators* {.
+          argument
+          desc: "One or more validators to export".}: seq[string]
+
   ValidatorClientConf* = object
     logLevel* {.
       defaultValue: "INFO"

beacon_chain/nimbus_beacon_node.nim

@@ -90,6 +90,9 @@ declareGauge next_action_wait,
 
 logScope: topics = "beacnde"
 
+const SlashingDbName = "slashing_protection"
+  # changing this requires physical file rename as well or history is lost.
+
 proc init*(T: type BeaconNode,
            runtimePreset: RuntimePreset,
            rng: ref BrHmacDrbgContext,
@@ -328,8 +331,7 @@ proc init*(T: type BeaconNode,
     slashingProtectionDB =
       SlashingProtectionDB.init(
           getStateField(chainDag.headState, genesis_validators_root),
-          config.validatorsDir(), "slashing_protection"
-        )
+          config.validatorsDir(), SlashingDbName)
     validatorPool = newClone(ValidatorPool.init(slashingProtectionDB))
 
     consensusManager = ConsensusManager.new(
@@ -1927,6 +1929,87 @@ proc doWeb3Cmd(config: BeaconNodeConf) {.raises: [Defect, CatchableError].} =
     waitFor testWeb3Provider(config.web3TestUrl,
                              metadata.depositContractAddress)
 
+proc doSlashingExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].}=
+  let
+    dir = conf.validatorsDir()
+    filetrunc = SlashingDbName
+  # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
+  let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
+
+  let interchange = block:
+    if conf.interchangeFile.isSome():
+      string(conf.interchangeFile.unsafeGet())
+    else:
+      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
+
+  db.exportSlashingInterchange(interchange)
+  echo "Export finished: '", dir/filetrunc & ".sqlite3" , "' into '", interchange, "'"
+
+proc doSlashingPartialExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].} =
+  let
+    dir = conf.validatorsDir()
+    filetrunc = SlashingDbName
+  # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
+  let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
+
+  let interchange = block:
+    if conf.interchangeFile.isSome():
+      string(conf.interchangeFile.unsafeGet())
+    else:
+      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
+
+  db.exportPartialSlashingInterchange(conf.validators, interchange)
+  echo "Partial export finished: '", dir/filetrunc/".sqlite3" , "' into '", interchange, "'"
+
+proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOError, Defect].} =
+  let
+    dir = conf.validatorsDir()
+    filetrunc = SlashingDbName
+  # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
+
+  let interchange = block:
+    if conf.interchangeFile.isSome():
+      string(conf.interchangeFile.unsafeGet())
+    else:
+      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
+
+  var spdir: SPDIR
+  try:
+    spdir = JSON.loadFile(interchange, SPDIR)
+  except SerializationError as err:
+    writeStackTrace()
+    stderr.write $JSON & " load issue for file \"", interchange, "\"\n"
+    stderr.write err.formatMsg(interchange), "\n"
+    quit 1
+
+  # Open DB and handle migration from v1 to v2 if needed
+  let db = SlashingProtectionDB.init(
+    genesis_validators_root = Eth2Digest spdir.metadata.genesis_validators_root,
+    basePath = dir,
+    dbname = filetrunc,
+    modes = {kCompleteArchiveV2},
+    disagreementBehavior = kChooseV2
+  )
+
+  # Now import the slashing interchange file
+  # Failures mode:
+  # - siError can only happen with invalid genesis_validators_root which would be caught above
+  # - siPartial can happen for invalid public keys, slashable blocks, slashable votes
+  let status = db.inclSPDIR(spdir)
+  doAssert status in {siSuccess, siPartial}
+
+  echo "Import finished: '", interchange, "' into '", dir/filetrunc & ".sqlite3", "'"
+
+proc doSlashingInterchange(conf: BeaconNodeConf) {.raises: [Defect, CatchableError].} =
+  doAssert conf.cmd == slashingdb
+  case conf.slashingdbCmd
+  of SlashProtCmd.exportAll:
+    conf.doSlashingExport()
+  of SlashProtCmd.`export`:
+    conf.doSlashingPartialExport()
+  of SlashProtCmd.`import`:
+    conf.doSlashingImport()
+
 {.pop.} # TODO moduletests exceptions
 programMain:
   var
@@ -1975,3 +2058,4 @@ programMain:
   of wallets: doWallets(config, rng[])
   of record: doRecord(config, rng[])
   of web3: doWeb3Cmd(config)
+  of slashingdb: doSlashingInterchange(config)

beacon_chain/validators/slashing_protection_common.nim

@@ -9,7 +9,7 @@
 
 import
   # Stdlib
-  std/[typetraits, strutils, algorithm],
+  std/[typetraits, strutils, algorithm, sequtils],
   # Status
   eth/db/[kvstore, kvstore_sqlite3],
   stew/results,
@@ -230,6 +230,24 @@ proc exportSlashingInterchange*(
   Json.saveFile(path, spdir, prettify)
   echo "Exported slashing protection DB to '", path, "'"
 
+proc exportPartialSlashingInterchange*(
+       db: SlashingProtectionDB_Concept,
+       validators: seq[string],
+       path: string, prettify = true) {.raises: [Defect, IOError].} =
+  ## Export a database to the Slashing Protection Database Interchange Format
+  # We could modify toSPDIR to do the filtering directly
+  # but this is not a performance sensitive operation.
+  # so it's better to keep it simple.
+  var spdir = db.toSPDIR()
+
+  # O(a log b) with b the number of validators to keep
+  #        and a the total number of validators in DB
+  let validators = validators.sorted()
+  spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1)
+
+  Json.saveFile(path, spdir, prettify)
+  echo "Exported slashing protection DB to '", path, "'"
+
 proc importSlashingInterchange*(
        db: auto,
        path: string): SlashingImportStatus {.raises: [Defect, IOError, SerializationError].} =

docs/slashing_interchange.md

@@ -0,0 +1,75 @@
+# Slashing interchange
+
+Importing and exporting validators is available via the following commands:
+
+- `path/to/nimbus_beacon_node slashingdb import --interchange=infile.json`
+- `path/to/nimbus_beacon_node slashingdb exportAll --interchange=outfile.json`
+- `path/to/nimbus_beacon_node slashingdb export --interchange=outfile.json --validators=0xAAAA...AAA --validators=0xBBBB...BBBB --validators=0xCCCC...CCCC`
+
+## Importing new validators
+
+## Importing validators
+
+The default command for import into the database is:
+
+```
+build/nimbus_beacon_node slashingdb import --interchange=interchange.json
+```
+
+### With specified validators folder
+
+The validators folder contains the valdiators setup.
+By default it is `path/to/datadir/validators`
+
+```
+build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
+```
+
+### With the data-dir folder
+
+The data-dir contains the beacon node setup.
+
+```
+build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
+```
+
+## Exporting all validators
+
+The default command for exporting the database is:
+
+```
+build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json
+```
+
+On success you will have a message similar to:
+
+```
+Exported slashing protection DB to 'interchange.json'
+Export finished: '$HOME/.cache/nimbus/BeaconNode/validators/slashing_protection.sqlite3' into 'interchange.json'
+```
+
+### With specified validators folder
+
+The validators folder contains the valdiators setup.
+By default it is `path/to/datadir/validators`
+
+```
+build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
+```
+
+### With the data-dir folder
+
+The data-dir contains the beacon node setup.
+
+```
+build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
+```
+
+## Partial exports
+
+Partial export can be done by specifying the public keys of the relevant validators.
+The `--validators` command can be specified multiple time, once per validator.
+
+```
+build/nimbus_beacon_node slashingdb export --interchange=interchange.json --validators=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615
+```

ncli/ncli_slashing.nim

@@ -20,7 +20,7 @@ type
     dump = "Dump the validator slashing protection DB to json"
     restore = "Restore the validator slashing protection DB from json"
 
-  SlashProtConf = object
+  SlashProtConf* = object
 
     db{.desc: "The path to the database .sqlite3 file" .}: string