Simplify the slashing db import/export CLI
This commit is contained in:
parent
dacc508992
commit
5c313b958e
|
@ -6,10 +6,10 @@ other clients and bringing further performance optimizations.
|
|||
|
||||
**New features:**
|
||||
|
||||
* A new `slashingProtection` sub-command offering import and export options
|
||||
for the EIP-3076 slashing protection interchange format. Please see the
|
||||
the prepared [migration guides](https://nimbus.guide/migrate.html) outlining
|
||||
the safest way to migrate to Nimbus from other clients.
|
||||
* A new `slashingdb` sub-command offering import and export options for the
|
||||
EIP-3076 slashing protection interchange format. Please see the the prepared
|
||||
[migration guides](https://nimbus.guide/migrate.html) outlining the safest
|
||||
way to migrate to Nimbus from other clients.
|
||||
|
||||
* Pruning of the slashing protection database and transition to more optimal
|
||||
queries *->* significant reduction of disk and CPU usage on nodes running
|
||||
|
|
|
@ -76,9 +76,8 @@ type
|
|||
file
|
||||
|
||||
SlashProtCmd* = enum
|
||||
exportAll = "Export the whole validator slashing protection DB to json."
|
||||
`import` = "Import the slashing protection json to the node."
|
||||
`export` = "Export specified validators slashing protection data to json"
|
||||
`import` = "Import a EIP-3076 slashing protection interchange file"
|
||||
`export` = "Export a EIP-3076 slashing protection interchange file"
|
||||
# migrateAll = "Export and remove the whole validator slashing protection DB."
|
||||
# migrate = "Export and remove specified validators from Nimbus."
|
||||
|
||||
|
@ -496,18 +495,20 @@ type
|
|||
name: "url" }: Uri
|
||||
|
||||
of slashingdb:
|
||||
# TODO: when this is not set, confutils crashes
|
||||
interchangeFile* {.
|
||||
desc: "Path to the slashing interchange file"
|
||||
name: "interchange" .}: Option[string]
|
||||
|
||||
case slashingdbCmd* {.command.}: SlashProtCmd
|
||||
of SlashProtCmd.exportAll, SlashProtCmd.`import`:
|
||||
discard
|
||||
of SlashProtCmd.`import`:
|
||||
importedInterchangeFile* {.
|
||||
desc: "EIP-3076 slashing protection interchange file to import"
|
||||
argument .}: InputFile
|
||||
of SlashProtCmd.`export`:
|
||||
validators* {.
|
||||
argument
|
||||
desc: "One or more validators to export".}: seq[string]
|
||||
exportedValidators* {.
|
||||
desc: "Limit the export to specific validators " &
|
||||
"(specified as numeric indices or public keys)"
|
||||
abbr: "v"
|
||||
name: "validator" }: seq[string]
|
||||
exportedInterchangeFile* {.
|
||||
desc: "EIP-3076 slashing protection interchange file to export"
|
||||
argument }: OutFile
|
||||
|
||||
ValidatorClientConf* = object
|
||||
logLevel* {.
|
||||
|
|
|
@ -1936,42 +1936,17 @@ proc doSlashingExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].}=
|
|||
# TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
|
||||
let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
|
||||
|
||||
let interchange = block:
|
||||
if conf.interchangeFile.isSome():
|
||||
string(conf.interchangeFile.unsafeGet())
|
||||
else:
|
||||
conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
|
||||
|
||||
db.exportSlashingInterchange(interchange)
|
||||
let interchange = conf.exportedInterchangeFile.string
|
||||
db.exportSlashingInterchange(interchange, conf.exportedValidators)
|
||||
echo "Export finished: '", dir/filetrunc & ".sqlite3" , "' into '", interchange, "'"
|
||||
|
||||
proc doSlashingPartialExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].} =
|
||||
let
|
||||
dir = conf.validatorsDir()
|
||||
filetrunc = SlashingDbName
|
||||
# TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
|
||||
let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
|
||||
|
||||
let interchange = block:
|
||||
if conf.interchangeFile.isSome():
|
||||
string(conf.interchangeFile.unsafeGet())
|
||||
else:
|
||||
conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
|
||||
|
||||
db.exportPartialSlashingInterchange(conf.validators, interchange)
|
||||
echo "Partial export finished: '", dir/filetrunc/".sqlite3" , "' into '", interchange, "'"
|
||||
|
||||
proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOError, Defect].} =
|
||||
let
|
||||
dir = conf.validatorsDir()
|
||||
filetrunc = SlashingDbName
|
||||
# TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
|
||||
|
||||
let interchange = block:
|
||||
if conf.interchangeFile.isSome():
|
||||
string(conf.interchangeFile.unsafeGet())
|
||||
else:
|
||||
conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
|
||||
let interchange = conf.importedInterchangeFile.string
|
||||
|
||||
var spdir: SPDIR
|
||||
try:
|
||||
|
@ -2003,10 +1978,8 @@ proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOErr
|
|||
proc doSlashingInterchange(conf: BeaconNodeConf) {.raises: [Defect, CatchableError].} =
|
||||
doAssert conf.cmd == slashingdb
|
||||
case conf.slashingdbCmd
|
||||
of SlashProtCmd.exportAll:
|
||||
conf.doSlashingExport()
|
||||
of SlashProtCmd.`export`:
|
||||
conf.doSlashingPartialExport()
|
||||
conf.doSlashingExport()
|
||||
of SlashProtCmd.`import`:
|
||||
conf.doSlashingImport()
|
||||
|
||||
|
|
|
@ -9,10 +9,11 @@
|
|||
|
||||
import
|
||||
# stdlib
|
||||
std/os,
|
||||
std/[os, algorithm, sequtils],
|
||||
# Status
|
||||
eth/db/[kvstore, kvstore_sqlite3],
|
||||
stew/results, chronicles,
|
||||
stew/[results, byteutils],
|
||||
chronicles,
|
||||
# Internal
|
||||
../spec/[datatypes, digest, crypto],
|
||||
./slashing_protection_common,
|
||||
|
@ -167,13 +168,6 @@ proc loadUnchecked*(
|
|||
except:
|
||||
result.disagreementBehavior = kChooseV1
|
||||
|
||||
try:
|
||||
result.db_v1.fromRawDB(kvstore result.db_v2.getRawDBHandle())
|
||||
result.modes.incl(kCompleteArchiveV1)
|
||||
except:
|
||||
doAssert result.modes.card > 0, "Couldn't open the DB."
|
||||
result.disagreementBehavior = kChooseV2
|
||||
|
||||
proc close*(db: SlashingProtectionDB) =
|
||||
## Close a slashing protection database
|
||||
db.db_v2.close()
|
||||
|
@ -307,3 +301,24 @@ proc inclSPDIR*(db: SlashingProtectionDB, spdir: SPDIR): SlashingImportStatus
|
|||
proc toSPDIR*(db: SlashingProtectionDB): SPDIR
|
||||
{.raises: [IOError, Defect].} =
|
||||
db.db_v2.toSPDIR()
|
||||
|
||||
proc exportSlashingInterchange*(
|
||||
db: SlashingProtectionDB,
|
||||
path: string,
|
||||
validatorsWhiteList: seq[string] = @[],
|
||||
prettify = true) {.raises: [Defect, IOError].} =
|
||||
## Export a database to the Slashing Protection Database Interchange Format
|
||||
# We could modify toSPDIR to do the filtering directly
|
||||
# but this is not a performance sensitive operation.
|
||||
# so it's better to keep it simple.
|
||||
var spdir = db.toSPDIR()
|
||||
|
||||
if validatorsWhiteList.len > 0:
|
||||
# O(a log b) with b the number of validators to keep
|
||||
# and a the total number of validators in DB
|
||||
let validators = validatorsWhiteList.sorted()
|
||||
spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1)
|
||||
|
||||
Json.saveFile(path, spdir, prettify)
|
||||
echo "Exported slashing protection DB to '", path, "'"
|
||||
|
||||
|
|
|
@ -222,32 +222,6 @@ proc readValue*(r: var JsonReader, a: var (SlotString or EpochString))
|
|||
except ValueError:
|
||||
raiseUnexpectedValue(r, "Integer in a string expected")
|
||||
|
||||
proc exportSlashingInterchange*(
|
||||
db: auto,
|
||||
path: string, prettify = true) {.raises: [Defect, IOError].} =
|
||||
## Export a database to the Slashing Protection Database Interchange Format
|
||||
let spdir = db.toSPDIR()
|
||||
Json.saveFile(path, spdir, prettify)
|
||||
echo "Exported slashing protection DB to '", path, "'"
|
||||
|
||||
proc exportPartialSlashingInterchange*(
|
||||
db: SlashingProtectionDB_Concept,
|
||||
validators: seq[string],
|
||||
path: string, prettify = true) {.raises: [Defect, IOError].} =
|
||||
## Export a database to the Slashing Protection Database Interchange Format
|
||||
# We could modify toSPDIR to do the filtering directly
|
||||
# but this is not a performance sensitive operation.
|
||||
# so it's better to keep it simple.
|
||||
var spdir = db.toSPDIR()
|
||||
|
||||
# O(a log b) with b the number of validators to keep
|
||||
# and a the total number of validators in DB
|
||||
let validators = validators.sorted()
|
||||
spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1)
|
||||
|
||||
Json.saveFile(path, spdir, prettify)
|
||||
echo "Exported slashing protection DB to '", path, "'"
|
||||
|
||||
proc importSlashingInterchange*(
|
||||
db: auto,
|
||||
path: string): SlashingImportStatus {.raises: [Defect, IOError, SerializationError].} =
|
||||
|
|
|
@ -2,9 +2,9 @@
|
|||
|
||||
Importing and exporting validators is available via the following commands:
|
||||
|
||||
- `path/to/nimbus_beacon_node slashingdb import --interchange=infile.json`
|
||||
- `path/to/nimbus_beacon_node slashingdb exportAll --interchange=outfile.json`
|
||||
- `path/to/nimbus_beacon_node slashingdb export --interchange=outfile.json --validators=0xAAAA...AAA --validators=0xBBBB...BBBB --validators=0xCCCC...CCCC`
|
||||
- `path/to/nimbus_beacon_node slashingdb import infile.json`
|
||||
- `path/to/nimbus_beacon_node slashingdb export outfile.json`
|
||||
- `path/to/nimbus_beacon_node slashingdb export outfile.json --validator=0xAAAA...AAA --validator=0xBBBB...BBBB --validator=0xCCCC...CCCC`
|
||||
|
||||
## Importing new validators
|
||||
|
||||
|
@ -13,7 +13,7 @@ Importing and exporting validators is available via the following commands:
|
|||
The default command for import into the database is:
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb import --interchange=interchange.json
|
||||
build/nimbus_beacon_node slashingdb import interchange.json
|
||||
```
|
||||
|
||||
### With specified validators folder
|
||||
|
@ -22,7 +22,7 @@ The validators folder contains the valdiators setup.
|
|||
By default it is `path/to/datadir/validators`
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
|
||||
build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/
|
||||
```
|
||||
|
||||
### With the data-dir folder
|
||||
|
@ -30,7 +30,7 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --v
|
|||
The data-dir contains the beacon node setup.
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
|
||||
build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/
|
||||
```
|
||||
|
||||
## Exporting all validators
|
||||
|
@ -38,7 +38,7 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --d
|
|||
The default command for exporting the database is:
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json
|
||||
build/nimbus_beacon_node slashingdb export interchange.json
|
||||
```
|
||||
|
||||
On success you will have a message similar to:
|
||||
|
@ -54,7 +54,7 @@ The validators folder contains the valdiators setup.
|
|||
By default it is `path/to/datadir/validators`
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
|
||||
build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/
|
||||
```
|
||||
|
||||
### With the data-dir folder
|
||||
|
@ -62,14 +62,14 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --v
|
|||
The data-dir contains the beacon node setup.
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
|
||||
build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/
|
||||
```
|
||||
|
||||
## Partial exports
|
||||
|
||||
Partial export can be done by specifying the public keys of the relevant validators.
|
||||
The `--validators` command can be specified multiple time, once per validator.
|
||||
The `--validator` command can be specified multiple time, once per validator.
|
||||
|
||||
```
|
||||
build/nimbus_beacon_node slashingdb export --interchange=interchange.json --validators=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615
|
||||
build/nimbus_beacon_node slashingdb export interchange.json --validator=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615
|
||||
```
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit f091a70a5bf95ec772c8b4d9978e81b8ae89af0c
|
||||
Subproject commit d1a45cfa9aa222cceb9b8365f8aeedeb4d51df36
|
Loading…
Reference in New Issue