From 0a91c3c592614f7cc651aea37a56cd02b243dfb6 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 12 May 2021 16:27:03 +0300 Subject: [PATCH 01/14] Prepare for the v1.3.0 release (more to come) --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ beacon_chain/version.nim | 4 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4a4cb0f0..6d85da8e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,39 @@ +2021-05-17 v1.3.0 +================= + +A new release offering safer and easier options for migrating to Nimbus from +other clients and bringing further performance optimizations. + +**New features:** + +* A new `slashingProtection` sub-command offering import and export options + for the EIP-3076 slashing protection interchange format. Please see the + the prepared [migration guides](https://nimbus.guide/migrate.html) outlining + the safest way to migrate to Nimbus from other clients. + +* Pruning of the slashing protection database and transition to more optimal + queries *->* significant reduction of disk and CPU usage on nodes running + large number of validators. + +* More consistent level of validation for the attestations received from + third-party sources and the JSON-RPC and REST APIs, preventing invalid + attestations to be broadcasted to the network. + +* Performance tuning of attestation subnet transition timings and state + snapshotting intervals *->* improved CPU and bandwidth usage. + +**We've fixed:** + +* Problems in the GossipSub subnet walking logic leading to unnecessary + bandwidth and CPU costs. + +**New tools:** + +* A new `ncli_db validatorPerf` command for producing a textual report for the + attestation performance of a particular validator (please note that `ncli_db` + is available only when compiling from source). + + 2021-05-03 v1.2.2 ================= diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 2853fc74a..397f10d81 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -15,8 +15,8 @@ when not defined(nimscript): const versionMajor* = 1 - versionMinor* = 2 - versionBuild* = 2 + versionMinor* = 3 + versionBuild* = 0 versionBlob* = "stateofus" # Single word - ends up in the default graffitti From 895ccd1c955ee1cda791279f3f05178743162fde Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 12 May 2021 14:31:02 +0200 Subject: [PATCH 02/14] clean up imports (#2557) --- beacon_chain/validators/validator_duties.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_chain/validators/validator_duties.nim b/beacon_chain/validators/validator_duties.nim index e26c12528..cb673c93e 100644 --- a/beacon_chain/validators/validator_duties.nim +++ b/beacon_chain/validators/validator_duties.nim @@ -17,12 +17,11 @@ import chronicles, json_serialization/std/[options, sets, net], serialization/errors, eth/db/kvstore, - eth/[keys, async_utils], eth/p2p/discoveryv5/[protocol, enr], + eth/keys, eth/p2p/discoveryv5/[protocol, enr], # Local modules ../spec/[ - datatypes, digest, crypto, helpers, network, signatures, state_transition, - validator], + datatypes, digest, crypto, helpers, network, signatures, state_transition], ../conf, ../beacon_clock, ../consensus_object_pools/[ spec_cache, blockchain_dag, block_clearance, From 2c257438a03cea0038729168d9c4e0cdccbf9c0d Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 14 May 2021 14:27:28 +0200 Subject: [PATCH 03/14] Upgrade database schema This is a minimal performance hotfix for the kvstore table, such that new databases get a kvstore table with rowid, fixing the most urgent performance problem we have with pruning. --- vendor/nim-eth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-eth b/vendor/nim-eth index 16802c0e5..bf5339b30 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 16802c0e5218cce405cd623a554ce95549dd5181 +Subproject commit bf5339b3033045852e849746fd127b9f3312e77d From 0574531c43c2072cfd9ac6f516cfc4224867b93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Tue, 30 Mar 2021 17:37:03 +0200 Subject: [PATCH 04/14] add restore from slashing DB --- Makefile | 1 + .../validators/slashing_protection.nim | 21 ++++++-- .../validators/slashing_protection_common.nim | 2 +- ncli/ncli_slashing.nim | 50 +++++++++++++++++-- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 408e4c44e..6fcde205f 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ TOOLS := \ nbench_spec_scenarios \ ncli \ ncli_db \ + ncli_slashing \ process_dashboard \ stack_sizes \ nimbus_validator_client \ diff --git a/beacon_chain/validators/slashing_protection.nim b/beacon_chain/validators/slashing_protection.nim index 9b6b9ad88..208f126c6 100644 --- a/beacon_chain/validators/slashing_protection.nim +++ b/beacon_chain/validators/slashing_protection.nim @@ -105,6 +105,7 @@ proc init*( fatal "The slashing database refers to another chain/mainnet/testnet", path = basePath/dbname, genesis_validators_root = genesis_validators_root + quit 1 db_v1.fromRawDB(rawdb) if requiresMigration: @@ -153,15 +154,25 @@ proc loadUnchecked*( ## this doesn't check the genesis validator root ## ## Does not handle migration + new result - result.modes = {kCompleteArchiveV1, kCompleteArchiveV2} + result.modes = {} result.disagreementBehavior = kCrash - result.db_v2 = SlashingProtectionDB_v2.loadUnchecked( - basePath, dbname, readOnly - ) + try: + result.db_v2 = SlashingProtectionDB_v2.loadUnchecked( + basePath, dbname, readOnly + ) + result.modes.incl(kCompleteArchiveV2) + except: + result.disagreementBehavior = kChooseV1 - result.db_v1.fromRawDB(kvstore result.db_v2.getRawDBHandle()) + try: + result.db_v1.fromRawDB(kvstore result.db_v2.getRawDBHandle()) + result.modes.incl(kCompleteArchiveV1) + except: + doAssert result.modes.card > 0, "Couldn't open the DB." + result.disagreementBehavior = kChooseV2 proc close*(db: SlashingProtectionDB) = ## Close a slashing protection database diff --git a/beacon_chain/validators/slashing_protection_common.nim b/beacon_chain/validators/slashing_protection_common.nim index 3fdab4d07..dda1391df 100644 --- a/beacon_chain/validators/slashing_protection_common.nim +++ b/beacon_chain/validators/slashing_protection_common.nim @@ -196,7 +196,7 @@ proc writeValue*(writer: var JsonWriter, value: PubKey0x) proc readValue*(reader: var JsonReader, value: var PubKey0x) {.raises: [SerializationError, IOError, Defect].} = try: - value = PubKey0x reader.readValue(string).hexToByteArray(RawPubKeySize) + value = PubKey0x hexToByteArray(reader.readValue(string), RawPubKeySize) except ValueError: raiseUnexpectedValue(reader, "Hex string expected") diff --git a/ncli/ncli_slashing.nim b/ncli/ncli_slashing.nim index 8bf02b7cc..e824cce73 100644 --- a/ncli/ncli_slashing.nim +++ b/ncli/ncli_slashing.nim @@ -10,6 +10,7 @@ import std/[os, strutils], confutils, + serialization, json_serialization, eth/db/[kvstore, kvstore_sqlite3], ../beacon_chain/validators/slashing_protection, ../beacon_chain/spec/digest @@ -21,24 +22,63 @@ type SlashProtConf = object + db{.desc: "The path to the database .sqlite3 file" .}: string + case cmd {. command, - desc: "Dump database or restore" .}: SlashProtCmd - of dump, restore: - infile {.argument.}: string + desc: "Dump database to or restore from a slashing interchange file" .}: SlashProtCmd + of dump: outfile {.argument.}: string + of restore: + infile {.argument.}: string proc doDump(conf: SlashProtConf) = - let (dir, file) = splitPath(conf.infile) + let (dir, file) = splitPath(conf.db) # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 # TODO: why is sqlite3 always appending .sqlite3 ? let filetrunc = file.changeFileExt("") let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) db.exportSlashingInterchange(conf.outfile) + echo "Export finished: '", conf.db, "' into '", conf.outfile, "'" + +proc doRestore(conf: SlashProtConf) = + let (dir, file) = splitPath(conf.db) + # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 + # TODO: why is sqlite3 always appending .sqlite3 ? + let filetrunc = file.changeFileExt("") + + var spdir: SPDIR + try: + spdir = JSON.loadFile(conf.infile, SPDIR) + except SerializationError as err: + writeStackTrace() + stderr.write $JSON & " load issue for file \"", conf.infile, "\"\n" + stderr.write err.formatMsg(conf.infile), "\n" + quit 1 + + # Open DB and handle migration from v1 to v2 if needed + let db = SlashingProtectionDB.init( + genesis_validators_root = Eth2Digest spdir.metadata.genesis_validators_root, + basePath = dir, + dbname = filetrunc, + modes = {kCompleteArchiveV2}, + disagreementBehavior = kChooseV2 + ) + + # Now import the slashing interchange file + # Failures mode: + # - siError can only happen with invalid genesis_validators_root which would be caught above + # - siPartial can happen for invalid public keys, slashable blocks, slashable votes + let status = db.inclSPDIR(spdir) + doAssert status in {siSuccess, siPartial} + + echo "Import finished: '", conf.infile, "' into '", conf.db, "'" + # TODO: do we mention that v2 MUST be used? + # normally this has always been a hidden option. when isMainModule: let conf = SlashProtConf.load() case conf.cmd: of dump: conf.doDump() - of restore: doAssert false, "unimplemented" + of restore: conf.doRestore() From dacc5089926f29900888ec411236efddeff3ce15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Thu, 13 May 2021 11:09:14 +0200 Subject: [PATCH 05/14] slashing import integrated in NBC --- beacon_chain/conf.nim | 22 +++++ beacon_chain/nimbus_beacon_node.nim | 88 ++++++++++++++++++- .../validators/slashing_protection_common.nim | 20 ++++- docs/slashing_interchange.md | 75 ++++++++++++++++ ncli/ncli_slashing.nim | 2 +- 5 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 docs/slashing_interchange.md diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index de0911e00..6cd8c1ead 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -38,6 +38,7 @@ type wallets record web3 + slashingdb WalletsCmd* {.pure.} = enum create = "Creates a new EIP-2386 wallet" @@ -74,6 +75,13 @@ type sql file + SlashProtCmd* = enum + exportAll = "Export the whole validator slashing protection DB to json." + `import` = "Import the slashing protection json to the node." + `export` = "Export specified validators slashing protection data to json" + # migrateAll = "Export and remove the whole validator slashing protection DB." + # migrate = "Export and remove specified validators from Nimbus." + BeaconNodeConf* = object logLevel* {. defaultValue: "INFO" @@ -487,6 +495,20 @@ type desc: "The web3 provider URL to test" name: "url" }: Uri + of slashingdb: + # TODO: when this is not set, confutils crashes + interchangeFile* {. + desc: "Path to the slashing interchange file" + name: "interchange" .}: Option[string] + + case slashingdbCmd* {.command.}: SlashProtCmd + of SlashProtCmd.exportAll, SlashProtCmd.`import`: + discard + of SlashProtCmd.`export`: + validators* {. + argument + desc: "One or more validators to export".}: seq[string] + ValidatorClientConf* = object logLevel* {. defaultValue: "INFO" diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index c00de56e9..117f552bf 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -90,6 +90,9 @@ declareGauge next_action_wait, logScope: topics = "beacnde" +const SlashingDbName = "slashing_protection" + # changing this requires physical file rename as well or history is lost. + proc init*(T: type BeaconNode, runtimePreset: RuntimePreset, rng: ref BrHmacDrbgContext, @@ -328,8 +331,7 @@ proc init*(T: type BeaconNode, slashingProtectionDB = SlashingProtectionDB.init( getStateField(chainDag.headState, genesis_validators_root), - config.validatorsDir(), "slashing_protection" - ) + config.validatorsDir(), SlashingDbName) validatorPool = newClone(ValidatorPool.init(slashingProtectionDB)) consensusManager = ConsensusManager.new( @@ -1927,6 +1929,87 @@ proc doWeb3Cmd(config: BeaconNodeConf) {.raises: [Defect, CatchableError].} = waitFor testWeb3Provider(config.web3TestUrl, metadata.depositContractAddress) +proc doSlashingExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].}= + let + dir = conf.validatorsDir() + filetrunc = SlashingDbName + # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 + let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) + + let interchange = block: + if conf.interchangeFile.isSome(): + string(conf.interchangeFile.unsafeGet()) + else: + conf.validatorsDir() & "/" & "interchange-" & "" & ".json" + + db.exportSlashingInterchange(interchange) + echo "Export finished: '", dir/filetrunc & ".sqlite3" , "' into '", interchange, "'" + +proc doSlashingPartialExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].} = + let + dir = conf.validatorsDir() + filetrunc = SlashingDbName + # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 + let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) + + let interchange = block: + if conf.interchangeFile.isSome(): + string(conf.interchangeFile.unsafeGet()) + else: + conf.validatorsDir() & "/" & "interchange-" & "" & ".json" + + db.exportPartialSlashingInterchange(conf.validators, interchange) + echo "Partial export finished: '", dir/filetrunc/".sqlite3" , "' into '", interchange, "'" + +proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOError, Defect].} = + let + dir = conf.validatorsDir() + filetrunc = SlashingDbName + # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 + + let interchange = block: + if conf.interchangeFile.isSome(): + string(conf.interchangeFile.unsafeGet()) + else: + conf.validatorsDir() & "/" & "interchange-" & "" & ".json" + + var spdir: SPDIR + try: + spdir = JSON.loadFile(interchange, SPDIR) + except SerializationError as err: + writeStackTrace() + stderr.write $JSON & " load issue for file \"", interchange, "\"\n" + stderr.write err.formatMsg(interchange), "\n" + quit 1 + + # Open DB and handle migration from v1 to v2 if needed + let db = SlashingProtectionDB.init( + genesis_validators_root = Eth2Digest spdir.metadata.genesis_validators_root, + basePath = dir, + dbname = filetrunc, + modes = {kCompleteArchiveV2}, + disagreementBehavior = kChooseV2 + ) + + # Now import the slashing interchange file + # Failures mode: + # - siError can only happen with invalid genesis_validators_root which would be caught above + # - siPartial can happen for invalid public keys, slashable blocks, slashable votes + let status = db.inclSPDIR(spdir) + doAssert status in {siSuccess, siPartial} + + echo "Import finished: '", interchange, "' into '", dir/filetrunc & ".sqlite3", "'" + +proc doSlashingInterchange(conf: BeaconNodeConf) {.raises: [Defect, CatchableError].} = + doAssert conf.cmd == slashingdb + case conf.slashingdbCmd + of SlashProtCmd.exportAll: + conf.doSlashingExport() + of SlashProtCmd.`export`: + conf.doSlashingPartialExport() + of SlashProtCmd.`import`: + conf.doSlashingImport() + {.pop.} # TODO moduletests exceptions programMain: var @@ -1975,3 +2058,4 @@ programMain: of wallets: doWallets(config, rng[]) of record: doRecord(config, rng[]) of web3: doWeb3Cmd(config) + of slashingdb: doSlashingInterchange(config) diff --git a/beacon_chain/validators/slashing_protection_common.nim b/beacon_chain/validators/slashing_protection_common.nim index dda1391df..606fa3264 100644 --- a/beacon_chain/validators/slashing_protection_common.nim +++ b/beacon_chain/validators/slashing_protection_common.nim @@ -9,7 +9,7 @@ import # Stdlib - std/[typetraits, strutils, algorithm], + std/[typetraits, strutils, algorithm, sequtils], # Status eth/db/[kvstore, kvstore_sqlite3], stew/results, @@ -230,6 +230,24 @@ proc exportSlashingInterchange*( Json.saveFile(path, spdir, prettify) echo "Exported slashing protection DB to '", path, "'" +proc exportPartialSlashingInterchange*( + db: SlashingProtectionDB_Concept, + validators: seq[string], + path: string, prettify = true) {.raises: [Defect, IOError].} = + ## Export a database to the Slashing Protection Database Interchange Format + # We could modify toSPDIR to do the filtering directly + # but this is not a performance sensitive operation. + # so it's better to keep it simple. + var spdir = db.toSPDIR() + + # O(a log b) with b the number of validators to keep + # and a the total number of validators in DB + let validators = validators.sorted() + spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1) + + Json.saveFile(path, spdir, prettify) + echo "Exported slashing protection DB to '", path, "'" + proc importSlashingInterchange*( db: auto, path: string): SlashingImportStatus {.raises: [Defect, IOError, SerializationError].} = diff --git a/docs/slashing_interchange.md b/docs/slashing_interchange.md new file mode 100644 index 000000000..7561e5f7d --- /dev/null +++ b/docs/slashing_interchange.md @@ -0,0 +1,75 @@ +# Slashing interchange + +Importing and exporting validators is available via the following commands: + +- `path/to/nimbus_beacon_node slashingdb import --interchange=infile.json` +- `path/to/nimbus_beacon_node slashingdb exportAll --interchange=outfile.json` +- `path/to/nimbus_beacon_node slashingdb export --interchange=outfile.json --validators=0xAAAA...AAA --validators=0xBBBB...BBBB --validators=0xCCCC...CCCC` + +## Importing new validators + +## Importing validators + +The default command for import into the database is: + +``` +build/nimbus_beacon_node slashingdb import --interchange=interchange.json +``` + +### With specified validators folder + +The validators folder contains the valdiators setup. +By default it is `path/to/datadir/validators` + +``` +build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/ +``` + +### With the data-dir folder + +The data-dir contains the beacon node setup. + +``` +build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/ +``` + +## Exporting all validators + +The default command for exporting the database is: + +``` +build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json +``` + +On success you will have a message similar to: + +``` +Exported slashing protection DB to 'interchange.json' +Export finished: '$HOME/.cache/nimbus/BeaconNode/validators/slashing_protection.sqlite3' into 'interchange.json' +``` + +### With specified validators folder + +The validators folder contains the valdiators setup. +By default it is `path/to/datadir/validators` + +``` +build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/ +``` + +### With the data-dir folder + +The data-dir contains the beacon node setup. + +``` +build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/ +``` + +## Partial exports + +Partial export can be done by specifying the public keys of the relevant validators. +The `--validators` command can be specified multiple time, once per validator. + +``` +build/nimbus_beacon_node slashingdb export --interchange=interchange.json --validators=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615 +``` diff --git a/ncli/ncli_slashing.nim b/ncli/ncli_slashing.nim index e824cce73..8e3ed65ce 100644 --- a/ncli/ncli_slashing.nim +++ b/ncli/ncli_slashing.nim @@ -20,7 +20,7 @@ type dump = "Dump the validator slashing protection DB to json" restore = "Restore the validator slashing protection DB from json" - SlashProtConf = object + SlashProtConf* = object db{.desc: "The path to the database .sqlite3 file" .}: string From 5c313b958e2381247f6b17b60e698fd35b0606ec Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 May 2021 01:24:18 +0300 Subject: [PATCH 06/14] Simplify the slashing db import/export CLI --- CHANGELOG.md | 8 ++--- beacon_chain/conf.nim | 27 +++++++------- beacon_chain/nimbus_beacon_node.nim | 35 +++---------------- .../validators/slashing_protection.nim | 33 ++++++++++++----- .../validators/slashing_protection_common.nim | 26 -------------- docs/slashing_interchange.md | 22 ++++++------ vendor/nim-confutils | 2 +- 7 files changed, 58 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d85da8e2..8e87db119 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,10 @@ other clients and bringing further performance optimizations. **New features:** -* A new `slashingProtection` sub-command offering import and export options - for the EIP-3076 slashing protection interchange format. Please see the - the prepared [migration guides](https://nimbus.guide/migrate.html) outlining - the safest way to migrate to Nimbus from other clients. +* A new `slashingdb` sub-command offering import and export options for the + EIP-3076 slashing protection interchange format. Please see the the prepared + [migration guides](https://nimbus.guide/migrate.html) outlining the safest + way to migrate to Nimbus from other clients. * Pruning of the slashing protection database and transition to more optimal queries *->* significant reduction of disk and CPU usage on nodes running diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 6cd8c1ead..a70ec071f 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -76,9 +76,8 @@ type file SlashProtCmd* = enum - exportAll = "Export the whole validator slashing protection DB to json." - `import` = "Import the slashing protection json to the node." - `export` = "Export specified validators slashing protection data to json" + `import` = "Import a EIP-3076 slashing protection interchange file" + `export` = "Export a EIP-3076 slashing protection interchange file" # migrateAll = "Export and remove the whole validator slashing protection DB." # migrate = "Export and remove specified validators from Nimbus." @@ -496,18 +495,20 @@ type name: "url" }: Uri of slashingdb: - # TODO: when this is not set, confutils crashes - interchangeFile* {. - desc: "Path to the slashing interchange file" - name: "interchange" .}: Option[string] - case slashingdbCmd* {.command.}: SlashProtCmd - of SlashProtCmd.exportAll, SlashProtCmd.`import`: - discard + of SlashProtCmd.`import`: + importedInterchangeFile* {. + desc: "EIP-3076 slashing protection interchange file to import" + argument .}: InputFile of SlashProtCmd.`export`: - validators* {. - argument - desc: "One or more validators to export".}: seq[string] + exportedValidators* {. + desc: "Limit the export to specific validators " & + "(specified as numeric indices or public keys)" + abbr: "v" + name: "validator" }: seq[string] + exportedInterchangeFile* {. + desc: "EIP-3076 slashing protection interchange file to export" + argument }: OutFile ValidatorClientConf* = object logLevel* {. diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 117f552bf..08a23b44a 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1936,42 +1936,17 @@ proc doSlashingExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].}= # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) - let interchange = block: - if conf.interchangeFile.isSome(): - string(conf.interchangeFile.unsafeGet()) - else: - conf.validatorsDir() & "/" & "interchange-" & "" & ".json" - - db.exportSlashingInterchange(interchange) + let interchange = conf.exportedInterchangeFile.string + db.exportSlashingInterchange(interchange, conf.exportedValidators) echo "Export finished: '", dir/filetrunc & ".sqlite3" , "' into '", interchange, "'" -proc doSlashingPartialExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].} = - let - dir = conf.validatorsDir() - filetrunc = SlashingDbName - # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 - let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) - - let interchange = block: - if conf.interchangeFile.isSome(): - string(conf.interchangeFile.unsafeGet()) - else: - conf.validatorsDir() & "/" & "interchange-" & "" & ".json" - - db.exportPartialSlashingInterchange(conf.validators, interchange) - echo "Partial export finished: '", dir/filetrunc/".sqlite3" , "' into '", interchange, "'" - proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOError, Defect].} = let dir = conf.validatorsDir() filetrunc = SlashingDbName # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 - let interchange = block: - if conf.interchangeFile.isSome(): - string(conf.interchangeFile.unsafeGet()) - else: - conf.validatorsDir() & "/" & "interchange-" & "" & ".json" + let interchange = conf.importedInterchangeFile.string var spdir: SPDIR try: @@ -2003,10 +1978,8 @@ proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOErr proc doSlashingInterchange(conf: BeaconNodeConf) {.raises: [Defect, CatchableError].} = doAssert conf.cmd == slashingdb case conf.slashingdbCmd - of SlashProtCmd.exportAll: - conf.doSlashingExport() of SlashProtCmd.`export`: - conf.doSlashingPartialExport() + conf.doSlashingExport() of SlashProtCmd.`import`: conf.doSlashingImport() diff --git a/beacon_chain/validators/slashing_protection.nim b/beacon_chain/validators/slashing_protection.nim index 208f126c6..4dd686e3d 100644 --- a/beacon_chain/validators/slashing_protection.nim +++ b/beacon_chain/validators/slashing_protection.nim @@ -9,10 +9,11 @@ import # stdlib - std/os, + std/[os, algorithm, sequtils], # Status eth/db/[kvstore, kvstore_sqlite3], - stew/results, chronicles, + stew/[results, byteutils], + chronicles, # Internal ../spec/[datatypes, digest, crypto], ./slashing_protection_common, @@ -167,13 +168,6 @@ proc loadUnchecked*( except: result.disagreementBehavior = kChooseV1 - try: - result.db_v1.fromRawDB(kvstore result.db_v2.getRawDBHandle()) - result.modes.incl(kCompleteArchiveV1) - except: - doAssert result.modes.card > 0, "Couldn't open the DB." - result.disagreementBehavior = kChooseV2 - proc close*(db: SlashingProtectionDB) = ## Close a slashing protection database db.db_v2.close() @@ -307,3 +301,24 @@ proc inclSPDIR*(db: SlashingProtectionDB, spdir: SPDIR): SlashingImportStatus proc toSPDIR*(db: SlashingProtectionDB): SPDIR {.raises: [IOError, Defect].} = db.db_v2.toSPDIR() + +proc exportSlashingInterchange*( + db: SlashingProtectionDB, + path: string, + validatorsWhiteList: seq[string] = @[], + prettify = true) {.raises: [Defect, IOError].} = + ## Export a database to the Slashing Protection Database Interchange Format + # We could modify toSPDIR to do the filtering directly + # but this is not a performance sensitive operation. + # so it's better to keep it simple. + var spdir = db.toSPDIR() + + if validatorsWhiteList.len > 0: + # O(a log b) with b the number of validators to keep + # and a the total number of validators in DB + let validators = validatorsWhiteList.sorted() + spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1) + + Json.saveFile(path, spdir, prettify) + echo "Exported slashing protection DB to '", path, "'" + diff --git a/beacon_chain/validators/slashing_protection_common.nim b/beacon_chain/validators/slashing_protection_common.nim index 606fa3264..659f29a5d 100644 --- a/beacon_chain/validators/slashing_protection_common.nim +++ b/beacon_chain/validators/slashing_protection_common.nim @@ -222,32 +222,6 @@ proc readValue*(r: var JsonReader, a: var (SlotString or EpochString)) except ValueError: raiseUnexpectedValue(r, "Integer in a string expected") -proc exportSlashingInterchange*( - db: auto, - path: string, prettify = true) {.raises: [Defect, IOError].} = - ## Export a database to the Slashing Protection Database Interchange Format - let spdir = db.toSPDIR() - Json.saveFile(path, spdir, prettify) - echo "Exported slashing protection DB to '", path, "'" - -proc exportPartialSlashingInterchange*( - db: SlashingProtectionDB_Concept, - validators: seq[string], - path: string, prettify = true) {.raises: [Defect, IOError].} = - ## Export a database to the Slashing Protection Database Interchange Format - # We could modify toSPDIR to do the filtering directly - # but this is not a performance sensitive operation. - # so it's better to keep it simple. - var spdir = db.toSPDIR() - - # O(a log b) with b the number of validators to keep - # and a the total number of validators in DB - let validators = validators.sorted() - spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1) - - Json.saveFile(path, spdir, prettify) - echo "Exported slashing protection DB to '", path, "'" - proc importSlashingInterchange*( db: auto, path: string): SlashingImportStatus {.raises: [Defect, IOError, SerializationError].} = diff --git a/docs/slashing_interchange.md b/docs/slashing_interchange.md index 7561e5f7d..5a6b0ce42 100644 --- a/docs/slashing_interchange.md +++ b/docs/slashing_interchange.md @@ -2,9 +2,9 @@ Importing and exporting validators is available via the following commands: -- `path/to/nimbus_beacon_node slashingdb import --interchange=infile.json` -- `path/to/nimbus_beacon_node slashingdb exportAll --interchange=outfile.json` -- `path/to/nimbus_beacon_node slashingdb export --interchange=outfile.json --validators=0xAAAA...AAA --validators=0xBBBB...BBBB --validators=0xCCCC...CCCC` +- `path/to/nimbus_beacon_node slashingdb import infile.json` +- `path/to/nimbus_beacon_node slashingdb export outfile.json` +- `path/to/nimbus_beacon_node slashingdb export outfile.json --validator=0xAAAA...AAA --validator=0xBBBB...BBBB --validator=0xCCCC...CCCC` ## Importing new validators @@ -13,7 +13,7 @@ Importing and exporting validators is available via the following commands: The default command for import into the database is: ``` -build/nimbus_beacon_node slashingdb import --interchange=interchange.json +build/nimbus_beacon_node slashingdb import interchange.json ``` ### With specified validators folder @@ -22,7 +22,7 @@ The validators folder contains the valdiators setup. By default it is `path/to/datadir/validators` ``` -build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/ +build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/ ``` ### With the data-dir folder @@ -30,7 +30,7 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --v The data-dir contains the beacon node setup. ``` -build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/ +build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/ ``` ## Exporting all validators @@ -38,7 +38,7 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --d The default command for exporting the database is: ``` -build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json +build/nimbus_beacon_node slashingdb export interchange.json ``` On success you will have a message similar to: @@ -54,7 +54,7 @@ The validators folder contains the valdiators setup. By default it is `path/to/datadir/validators` ``` -build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/ +build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/ ``` ### With the data-dir folder @@ -62,14 +62,14 @@ build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --v The data-dir contains the beacon node setup. ``` -build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/ +build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/ ``` ## Partial exports Partial export can be done by specifying the public keys of the relevant validators. -The `--validators` command can be specified multiple time, once per validator. +The `--validator` command can be specified multiple time, once per validator. ``` -build/nimbus_beacon_node slashingdb export --interchange=interchange.json --validators=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615 +build/nimbus_beacon_node slashingdb export interchange.json --validator=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615 ``` diff --git a/vendor/nim-confutils b/vendor/nim-confutils index f091a70a5..d1a45cfa9 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit f091a70a5bf95ec772c8b4d9978e81b8ae89af0c +Subproject commit d1a45cfa9aa222cceb9b8365f8aeedeb4d51df36 From 5fadaa247c75962ae0812644008f33bedaba63d2 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 May 2021 18:17:42 +0300 Subject: [PATCH 07/14] Remove ncli_slashing --- Makefile | 1 - ncli/ncli_slashing.nim | 84 ------------------------------------------ 2 files changed, 85 deletions(-) delete mode 100644 ncli/ncli_slashing.nim diff --git a/Makefile b/Makefile index 6fcde205f..408e4c44e 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,6 @@ TOOLS := \ nbench_spec_scenarios \ ncli \ ncli_db \ - ncli_slashing \ process_dashboard \ stack_sizes \ nimbus_validator_client \ diff --git a/ncli/ncli_slashing.nim b/ncli/ncli_slashing.nim deleted file mode 100644 index 8e3ed65ce..000000000 --- a/ncli/ncli_slashing.nim +++ /dev/null @@ -1,84 +0,0 @@ -# beacon_chain -# Copyright (c) 2018-2020 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -# Import/export the validator slashing protection database - -import - std/[os, strutils], - confutils, - serialization, json_serialization, - eth/db/[kvstore, kvstore_sqlite3], - ../beacon_chain/validators/slashing_protection, - ../beacon_chain/spec/digest - -type - SlashProtCmd = enum - dump = "Dump the validator slashing protection DB to json" - restore = "Restore the validator slashing protection DB from json" - - SlashProtConf* = object - - db{.desc: "The path to the database .sqlite3 file" .}: string - - case cmd {. - command, - desc: "Dump database to or restore from a slashing interchange file" .}: SlashProtCmd - of dump: - outfile {.argument.}: string - of restore: - infile {.argument.}: string - -proc doDump(conf: SlashProtConf) = - let (dir, file) = splitPath(conf.db) - # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 - # TODO: why is sqlite3 always appending .sqlite3 ? - let filetrunc = file.changeFileExt("") - let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false) - db.exportSlashingInterchange(conf.outfile) - echo "Export finished: '", conf.db, "' into '", conf.outfile, "'" - -proc doRestore(conf: SlashProtConf) = - let (dir, file) = splitPath(conf.db) - # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312 - # TODO: why is sqlite3 always appending .sqlite3 ? - let filetrunc = file.changeFileExt("") - - var spdir: SPDIR - try: - spdir = JSON.loadFile(conf.infile, SPDIR) - except SerializationError as err: - writeStackTrace() - stderr.write $JSON & " load issue for file \"", conf.infile, "\"\n" - stderr.write err.formatMsg(conf.infile), "\n" - quit 1 - - # Open DB and handle migration from v1 to v2 if needed - let db = SlashingProtectionDB.init( - genesis_validators_root = Eth2Digest spdir.metadata.genesis_validators_root, - basePath = dir, - dbname = filetrunc, - modes = {kCompleteArchiveV2}, - disagreementBehavior = kChooseV2 - ) - - # Now import the slashing interchange file - # Failures mode: - # - siError can only happen with invalid genesis_validators_root which would be caught above - # - siPartial can happen for invalid public keys, slashable blocks, slashable votes - let status = db.inclSPDIR(spdir) - doAssert status in {siSuccess, siPartial} - - echo "Import finished: '", conf.infile, "' into '", conf.db, "'" - # TODO: do we mention that v2 MUST be used? - # normally this has always been a hidden option. - -when isMainModule: - let conf = SlashProtConf.load() - - case conf.cmd: - of dump: conf.doDump() - of restore: conf.doRestore() From b9924214ab2cfcd6cf0c98e9b485ea06820b3f47 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 May 2021 19:55:46 +0300 Subject: [PATCH 08/14] Better error-handling for the slashingdb import/export feature * Error when specifying an invalid --data-dir (or --validator-dir) * Error when entering an invalid validator public key (e.g. invalid hex value) * Warning when attempting to export a validator not present in the local database Some unnecessary remains of the v1 mode has been removed as well --- beacon_chain/conf.nim | 23 ++++-- beacon_chain/nimbus_beacon_node.nim | 3 +- .../validators/slashing_protection.nim | 44 +++++------ .../validators/slashing_protection_common.nim | 11 ++- .../validators/slashing_protection_v1.nim | 2 +- docs/slashing_interchange.md | 75 ------------------- vendor/nim-confutils | 2 +- 7 files changed, 50 insertions(+), 110 deletions(-) delete mode 100644 docs/slashing_interchange.md diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index a70ec071f..837aacd11 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -9,15 +9,18 @@ import strutils, os, options, unicode, uri, + chronicles, chronicles/options as chroniclesOptions, confutils, confutils/defs, confutils/std/net, stew/shims/net as stewNet, - stew/io2, unicodedb/properties, normalize, + stew/[io2, byteutils], unicodedb/properties, normalize, eth/common/eth_types as commonEthTypes, eth/net/nat, eth/p2p/discoveryv5/enr, json_serialization, web3/[ethtypes, confutils_defs], - spec/[crypto, keystore, digest, datatypes, network], + + ./spec/[crypto, keystore, digest, datatypes, network], ./networking/network_metadata, - filepath + ./validators/slashing_protection_common, + ./filepath export uri, @@ -505,7 +508,7 @@ type desc: "Limit the export to specific validators " & "(specified as numeric indices or public keys)" abbr: "v" - name: "validator" }: seq[string] + name: "validator" }: seq[PubKey0x] exportedInterchangeFile* {. desc: "EIP-3076 slashing protection interchange file to export" argument }: OutFile @@ -559,8 +562,9 @@ type name: "rpc-port" }: Port rpcAddress* {. - defaultValue: defaultAdminListenAddress - desc: "Address of the server to connect to for RPC [=127.0.0.1]" + desc: "Address of the server to connect to for RPC" + defaultValue: init(ValidIpAddress, "127.0.0.1") + defaultValueDesc: "127.0.0.1" name: "rpc-address" }: ValidIpAddress retryDelay* {. @@ -623,6 +627,13 @@ func parseCmdArg*(T: type Uri, input: TaintedString): T func completeCmdArg*(T: type Uri, input: TaintedString): seq[string] = return @[] +func parseCmdArg*(T: type PubKey0x, input: TaintedString): T + {.raises: [ValueError, Defect].} = + PubKey0x(hexToPaddedByteArray[RawPubKeySize](input.string)) + +func completeCmdArg*(T: type PubKey0x, input: TaintedString): seq[string] = + return @[] + func parseCmdArg*(T: type Checkpoint, input: TaintedString): T {.raises: [ValueError, Defect].} = let sepIdx = find(input.string, ':') diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 08a23b44a..c8bc57aad 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1962,8 +1962,7 @@ proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOErr genesis_validators_root = Eth2Digest spdir.metadata.genesis_validators_root, basePath = dir, dbname = filetrunc, - modes = {kCompleteArchiveV2}, - disagreementBehavior = kChooseV2 + modes = {kCompleteArchive} ) # Now import the slashing interchange file diff --git a/beacon_chain/validators/slashing_protection.nim b/beacon_chain/validators/slashing_protection.nim index 4dd686e3d..71274fa21 100644 --- a/beacon_chain/validators/slashing_protection.nim +++ b/beacon_chain/validators/slashing_protection.nim @@ -45,9 +45,8 @@ export chronicles type SlashProtDBMode* = enum - kCompleteArchiveV1 # Complete Format V1 backend (saves all attestations) - kCompleteArchiveV2 # Complete Format V2 backend (saves all attestations) - kLowWatermarkV2 # Low-Watermark Format V2 backend (prunes attestations) + kCompleteArchive # Complete Format V2 backend (saves all attestations) + kLowWatermark # Low-Watermark Format V2 backend (prunes attestations) SlashingProtectionDB* = ref object ## Database storing the blocks attested @@ -55,13 +54,6 @@ type ## or validator client. db_v2*: SlashingProtectionDB_v2 modes: set[SlashProtDBMode] - disagreementBehavior: DisagreementBehavior - - DisagreementBehavior* = enum - ## How to handle disagreement between DB versions - kCrash - kChooseV1 - kChooseV2 # DB Multiversioning # ------------------------------------------------------------- @@ -78,7 +70,6 @@ proc init*( genesis_validators_root: Eth2Digest, basePath, dbname: string, modes: set[SlashProtDBMode], - disagreementBehavior: DisagreementBehavior ): T = ## Initialize or load a slashing protection DB ## This is for Beacon Node usage @@ -86,12 +77,11 @@ proc init*( doAssert modes.card >= 1, "No slashing protection mode chosen. Choose a v1, a v2 or v1 and v2 slashing DB mode." doAssert not( - kCompleteArchiveV2 in modes and - kLowWatermarkV2 in modes), "Mode(s): " & $modes & ". Choose only one of V2 DB modes." + kCompleteArchive in modes and + kLowWatermark in modes), "Mode(s): " & $modes & ". Choose only one of V2 DB modes." new result result.modes = modes - result.disagreementBehavior = disagreementBehavior let (db, requiresMigration) = SlashingProtectionDB_v2.initCompatV1( genesis_validators_root, @@ -142,8 +132,7 @@ proc init*( ## Does not handle migration init( T, genesis_validators_root, basePath, dbname, - modes = {kLowWatermarkV2}, - disagreementBehavior = kChooseV2 + modes = {kLowWatermark} ) proc loadUnchecked*( @@ -158,15 +147,15 @@ proc loadUnchecked*( new result result.modes = {} - result.disagreementBehavior = kCrash try: result.db_v2 = SlashingProtectionDB_v2.loadUnchecked( basePath, dbname, readOnly ) - result.modes.incl(kCompleteArchiveV2) - except: - result.disagreementBehavior = kChooseV1 + result.modes.incl(kCompleteArchive) + except CatchableError as err: + error "Failed to load the Slashing protection database", err = err.msg + quit 1 proc close*(db: SlashingProtectionDB) = ## Close a slashing protection database @@ -280,10 +269,10 @@ proc pruneAfterFinalization*( ## This ensures that even if pruning is called with an incorrect epoch ## slashing protection can fallback to the minimal / high-watermark protection mode. ## - ## Pruning is only done if pruning is enabled (DB in kLowWatermarkV2 mode) + ## Pruning is only done if pruning is enabled (DB in kLowWatermark mode) ## Pruning is only triggered on v2 database. - if kLowWatermarkV2 in db.modes: + if kLowWatermark in db.modes: db.db_v2.pruneAfterFinalization(finalizedEpoch) # The high-level import/export functions are @@ -305,7 +294,7 @@ proc toSPDIR*(db: SlashingProtectionDB): SPDIR proc exportSlashingInterchange*( db: SlashingProtectionDB, path: string, - validatorsWhiteList: seq[string] = @[], + validatorsWhiteList: seq[PubKey0x] = @[], prettify = true) {.raises: [Defect, IOError].} = ## Export a database to the Slashing Protection Database Interchange Format # We could modify toSPDIR to do the filtering directly @@ -317,7 +306,14 @@ proc exportSlashingInterchange*( # O(a log b) with b the number of validators to keep # and a the total number of validators in DB let validators = validatorsWhiteList.sorted() - spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1) + spdir.data.keepItIf(validators.binarySearch(it.pubkey) != -1) + + if spdir.data.len != validatorsWhiteList.len: + let exportedKeys = spdir.data.mapIt(it.pubkey).sorted() + for v in validators: + if exportedKeys.binarySearch(v) == -1: + warn "Specified validator key not found in the slashing database", + key = v.PubKeyBytes.toHex Json.saveFile(path, spdir, prettify) echo "Exported slashing protection DB to '", path, "'" diff --git a/beacon_chain/validators/slashing_protection_common.nim b/beacon_chain/validators/slashing_protection_common.nim index 659f29a5d..9a7eee0a9 100644 --- a/beacon_chain/validators/slashing_protection_common.nim +++ b/beacon_chain/validators/slashing_protection_common.nim @@ -9,7 +9,7 @@ import # Stdlib - std/[typetraits, strutils, algorithm, sequtils], + std/[typetraits, strutils, algorithm], # Status eth/db/[kvstore, kvstore_sqlite3], stew/results, @@ -170,6 +170,15 @@ func `==`*(a, b: BadVote): bool = of BadVoteKind.DatabaseError: true +template `==`*(a, b: PubKey0x): bool = + PubKeyBytes(a) == PubKeyBytes(b) + +template `<`*(a, b: PubKey0x): bool = + PubKeyBytes(a) < PubKeyBytes(b) + +template cmp*(a, b: PubKey0x): bool = + cmp(PubKeyBytes(a), PubKeyBytes(b)) + func `==`*(a, b: BadProposal): bool = ## Comparison operator. ## Used implictily by Result when comparing the diff --git a/beacon_chain/validators/slashing_protection_v1.nim b/beacon_chain/validators/slashing_protection_v1.nim index 015040b46..caedeaacc 100644 --- a/beacon_chain/validators/slashing_protection_v1.nim +++ b/beacon_chain/validators/slashing_protection_v1.nim @@ -398,7 +398,7 @@ proc loadUnchecked*( subkey(kGenesisValidatorsRoot) ).get(), "The Slashing DB is missing genesis information" - result = T(backend: backend) + T(backend: backend) proc close*(db: SlashingProtectionDB_v1) = discard db.backend.close() diff --git a/docs/slashing_interchange.md b/docs/slashing_interchange.md deleted file mode 100644 index 5a6b0ce42..000000000 --- a/docs/slashing_interchange.md +++ /dev/null @@ -1,75 +0,0 @@ -# Slashing interchange - -Importing and exporting validators is available via the following commands: - -- `path/to/nimbus_beacon_node slashingdb import infile.json` -- `path/to/nimbus_beacon_node slashingdb export outfile.json` -- `path/to/nimbus_beacon_node slashingdb export outfile.json --validator=0xAAAA...AAA --validator=0xBBBB...BBBB --validator=0xCCCC...CCCC` - -## Importing new validators - -## Importing validators - -The default command for import into the database is: - -``` -build/nimbus_beacon_node slashingdb import interchange.json -``` - -### With specified validators folder - -The validators folder contains the valdiators setup. -By default it is `path/to/datadir/validators` - -``` -build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/ -``` - -### With the data-dir folder - -The data-dir contains the beacon node setup. - -``` -build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/ -``` - -## Exporting all validators - -The default command for exporting the database is: - -``` -build/nimbus_beacon_node slashingdb export interchange.json -``` - -On success you will have a message similar to: - -``` -Exported slashing protection DB to 'interchange.json' -Export finished: '$HOME/.cache/nimbus/BeaconNode/validators/slashing_protection.sqlite3' into 'interchange.json' -``` - -### With specified validators folder - -The validators folder contains the valdiators setup. -By default it is `path/to/datadir/validators` - -``` -build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/ -``` - -### With the data-dir folder - -The data-dir contains the beacon node setup. - -``` -build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/ -``` - -## Partial exports - -Partial export can be done by specifying the public keys of the relevant validators. -The `--validator` command can be specified multiple time, once per validator. - -``` -build/nimbus_beacon_node slashingdb export interchange.json --validator=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615 -``` diff --git a/vendor/nim-confutils b/vendor/nim-confutils index d1a45cfa9..5f2f88215 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit d1a45cfa9aa222cceb9b8365f8aeedeb4d51df36 +Subproject commit 5f2f882151fee87471514bb74c1311856c636594 From f6a1f602b4409f633f224992ab6fb89c32beaad9 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 May 2021 20:44:57 +0300 Subject: [PATCH 09/14] Handle changes in the latest version of Confutils --- beacon_chain/conf.nim | 114 ++++++++++++++++++++++++------------------ vendor/nim-confutils | 2 +- 2 files changed, 66 insertions(+), 50 deletions(-) diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 837aacd11..069934125 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -86,8 +86,8 @@ type BeaconNodeConf* = object logLevel* {. + desc: "Sets the log level for process and topics (e.g. \"DEBUG; TRACE:discv5,libp2p; REQUIRED:none; DISABLED:none\")" defaultValue: "INFO" - desc: "Sets the log level for process and topics (e.g. \"DEBUG; TRACE:discv5,libp2p; REQUIRED:none; DISABLED:none\") [=INFO]" name: "log-level" }: string logFile* {. @@ -95,12 +95,14 @@ type name: "log-file" }: Option[OutFile] eth2Network* {. - desc: "The Eth2 network to join [=mainnet]" + desc: "The Eth2 network to join" + defaultValueDesc: "mainnet" name: "network" }: Option[string] dataDir* {. - defaultValue: config.defaultDataDir() desc: "The directory where nimbus will store all blockchain data" + defaultValue: config.defaultDataDir() + defaultValueDesc: "" abbr: "d" name: "data-dir" }: OutDir @@ -131,15 +133,14 @@ type name: "non-interactive" }: bool netKeyFile* {. - defaultValue: "random", desc: "Source of network (secp256k1) private key file " & - "(random|) [=random]" + "(random|)" + defaultValue: "random", name: "netkey-file" }: string netKeyInsecurePassword* {. + desc: "Use pre-generated INSECURE password for network private key file" defaultValue: false, - desc: "Use pre-generated INSECURE password for network private key " & - "file [=false]" name: "insecure-netkey-password" }: bool agentString* {. @@ -155,13 +156,14 @@ type slashingDbKind* {. hidden defaultValue: SlashingDbKind.v2 - desc: "The slashing DB flavour to use (v2) [=v2]" + desc: "The slashing DB flavour to use" name: "slashing-db-kind" }: SlashingDbKind stateDbKind* {. hidden + desc: "State DB kind (sql, file)" defaultValue: StateDbKind.sql - desc: "State DB kind (sql, file) [=sql]" + defaultValueDesc: "sql" name: "state-db-kind" }: StateDbKind case cmd* {. @@ -175,42 +177,45 @@ type name: "bootstrap-node" }: seq[string] bootstrapNodesFile* {. - defaultValue: "" desc: "Specifies a line-delimited file of bootstrap Ethereum network addresses" + defaultValue: "" name: "bootstrap-file" }: InputFile listenAddress* {. + desc: "Listening address for the Ethereum LibP2P and Discovery v5 traffic" defaultValue: defaultListenAddress - desc: "Listening address for the Ethereum LibP2P and Discovery v5 " & - "traffic [=0.0.0.0]" + defaultValueDesc: "0.0.0.0" name: "listen-address" }: ValidIpAddress tcpPort* {. + desc: "Listening TCP port for Ethereum LibP2P traffic" defaultValue: defaultEth2TcpPort - desc: "Listening TCP port for Ethereum LibP2P traffic [=9000]" + defaultValueDesc: "9000" name: "tcp-port" }: Port udpPort* {. + desc: "Listening UDP port for node discovery" defaultValue: defaultEth2TcpPort - desc: "Listening UDP port for node discovery [=9000]" + # defaultValueDesc: 9000 name: "udp-port" }: Port maxPeers* {. + desc: "The maximum number of peers to connect to" defaultValue: 160 # 5 (fanout) * 64 (subnets) / 2 (subs) for a heathy mesh - desc: "The maximum number of peers to connect to [=160]" name: "max-peers" }: int nat* {. desc: "Specify method to use for determining public address. " & "Must be one of: any, none, upnp, pmp, extip:" defaultValue: NatConfig(hasExtIp: false, nat: NatAny) + defaultValueDesc: "any" name: "nat" .}: NatConfig enrAutoUpdate* {. - defaultValue: false desc: "Discovery can automatically update its ENR with the IP address " & "and UDP port as seen by other nodes it communicates with. " & "This option allows to enable/disable this functionality" + defaultValue: false name: "enr-auto-update" .}: bool weakSubjectivityCheckpoint* {. @@ -226,9 +231,9 @@ type name: "finalized-checkpoint-block" }: Option[InputFile] nodeName* {. - defaultValue: "" desc: "A name for this node that will appear in the logs. " & "If you set this to 'auto', a persistent automatically generated ID will be selected for each --data-dir folder" + defaultValue: "" name: "node-name" }: string graffiti* {. @@ -237,88 +242,94 @@ type name: "graffiti" }: Option[GraffitiBytes] verifyFinalization* {. - defaultValue: false desc: "Specify whether to verify finalization occurs on schedule, for testing" + defaultValue: false name: "verify-finalization" }: bool stopAtEpoch* {. - defaultValue: 0 desc: "A positive epoch selects the epoch at which to stop" + defaultValue: 0 name: "stop-at-epoch" }: uint64 metricsEnabled* {. + desc: "Enable the metrics server" defaultValue: false - desc: "Enable the metrics server [=false]" name: "metrics" }: bool metricsAddress* {. + desc: "Listening address of the metrics server" defaultValue: defaultAdminListenAddress - desc: "Listening address of the metrics server [=127.0.0.1]" + defaultValueDesc: "127.0.0.1" name: "metrics-address" }: ValidIpAddress metricsPort* {. + desc: "Listening HTTP port of the metrics server" defaultValue: 8008 - desc: "Listening HTTP port of the metrics server [=8008]" name: "metrics-port" }: Port statusBarEnabled* {. - defaultValue: true desc: "Display a status bar at the bottom of the terminal screen" + defaultValue: true name: "status-bar" }: bool statusBarContents* {. + desc: "Textual template for the contents of the status bar" defaultValue: "peers: $connected_peers;" & "finalized: $finalized_root:$finalized_epoch;" & "head: $head_root:$head_epoch:$head_epoch_slot;" & "time: $epoch:$epoch_slot ($slot);" & "sync: $sync_status|" & "ETH: $attached_validators_balance" - desc: "Textual template for the contents of the status bar" + defaultValueDesc: "" name: "status-bar-contents" }: string rpcEnabled* {. + desc: "Enable the JSON-RPC server" defaultValue: false - desc: "Enable the JSON-RPC server [=false]" name: "rpc" }: bool rpcPort* {. + desc: "HTTP port for the JSON-RPC service" defaultValue: defaultEth2RpcPort - desc: "HTTP port for the JSON-RPC service [=9190]" + defaultValueDesc: "9190" name: "rpc-port" }: Port rpcAddress* {. + desc: "Listening address of the RPC server" defaultValue: defaultAdminListenAddress - desc: "Listening address of the RPC server [=127.0.0.1]" + defaultValueDesc: "127.0.0.1" name: "rpc-address" }: ValidIpAddress restEnabled* {. + desc: "Enable the REST (BETA version) server" defaultValue: false - desc: "Enable the REST (BETA version) server [=false]" name: "rest" }: bool restPort* {. + desc: "Port for the REST (BETA version) server" defaultValue: DefaultEth2RestPort - desc: "Port for the REST (BETA version) server [=5052]" + defaultValueDesc: "5052" name: "rest-port" }: Port restAddress* {. + desc: "Listening address of the REST (BETA version) server" defaultValue: defaultAdminListenAddress - desc: "Listening address of the REST (BETA version) server [=127.0.0.1]" + defaultValueDesc: "127.0.0.1" name: "rest-address" }: ValidIpAddress inProcessValidators* {. - defaultValue: true # the use of the nimbus_signing_process binary by default will be delayed until async I/O over stdin/stdout is developed for the child process. desc: "Disable the push model (the beacon node tells a signing process with the private keys of the validators what to sign and when) and load the validators in the beacon node itself" + defaultValue: true # the use of the nimbus_signing_process binary by default will be delayed until async I/O over stdin/stdout is developed for the child process. name: "in-process-validators" }: bool discv5Enabled* {. + desc: "Enable Discovery v5" defaultValue: true - desc: "Enable Discovery v5 [=true]" name: "discv5" }: bool dumpEnabled* {. + desc: "Write SSZ dumps of blocks, attestations and states to data dir" defaultValue: false - desc: "Write SSZ dumps of blocks, attestations and states to data dir [=false]" name: "dump" }: bool directPeers* {. @@ -326,8 +337,8 @@ type name: "direct-peer" .}: seq[string] doppelgangerDetection* {. + desc: "Whether to detect whether another validator is be running the same validator keys" defaultValue: true - desc: "Whether to detect whether another validator is be running the same validator keys [=true]" name: "doppelganger-detection" }: bool @@ -341,18 +352,20 @@ type name: "total-validators" }: uint64 bootstrapAddress* {. - defaultValue: init(ValidIpAddress, "127.0.0.1") desc: "The public IP address that will be advertised as a bootstrap node for the testnet" + defaultValue: init(ValidIpAddress, "127.0.0.1") + defaultValueDesc: "127.0.0.1" name: "bootstrap-address" }: ValidIpAddress bootstrapPort* {. - defaultValue: defaultEth2TcpPort desc: "The TCP/UDP port that will be used by the bootstrap node" + defaultValue: defaultEth2TcpPort + defaultValueDesc: "9000" name: "bootstrap-port" }: Port genesisOffset* {. - defaultValue: 5 desc: "Seconds from now to add to genesis time" + defaultValue: 5 name: "genesis-offset" }: int outputGenesis* {. @@ -360,8 +373,8 @@ type name: "output-genesis" }: OutFile withGenesisRoot* {. - defaultValue: false desc: "Include a genesis root in 'network.json'" + defaultValue: false name: "with-genesis-root" }: bool outputBootstrapFile* {. @@ -405,8 +418,8 @@ type case depositsCmd* {.command.}: DepositsCmd of DepositsCmd.createTestnetDeposits: totalDeposits* {. - defaultValue: 1 desc: "Number of deposits to generate" + defaultValue: 1 name: "count" }: int existingWalletId* {. @@ -414,13 +427,13 @@ type name: "wallet" }: Option[WalletName] outValidatorsDir* {. - defaultValue: "validators" desc: "Output folder for validator keystores" + defaultValue: "validators" name: "out-validators-dir" }: string outSecretsDir* {. - defaultValue: "secrets" desc: "Output folder for randomly generated keystore passphrases" + defaultValue: "secrets" name: "out-secrets-dir" }: string outDepositsFile* {. @@ -451,9 +464,10 @@ type desc: "Validator index or a public key of the exited validator" }: string rpcUrlForExit* {. - name: "rpc-url" + desc: "URL of the beacon node JSON-RPC service" defaultValue: parseUri("http://localhost:" & $defaultEth2RpcPort) - desc: "URL of the beacon node JSON-RPC service" }: Uri + defaultValueDesc: "http://localhost:9190" + name: "rpc-url" }: Uri exitAtEpoch* {. name: "epoch" @@ -475,8 +489,8 @@ type name: "udp-port" .}: Port seqNumber* {. - defaultValue: 1, desc: "Record sequence number" + defaultValue: 1, name: "seq-number" .}: uint fields* {. @@ -515,8 +529,8 @@ type ValidatorClientConf* = object logLevel* {. + desc: "Sets the log level" defaultValue: "INFO" - desc: "Sets the log level [=INFO]" name: "log-level" }: string logFile* {. @@ -524,8 +538,9 @@ type name: "log-file" }: Option[OutFile] dataDir* {. - defaultValue: config.defaultDataDir() desc: "The directory where nimbus will store all blockchain data" + defaultValue: config.defaultDataDir() + defaultValueDesc: "" abbr: "d" name: "data-dir" }: OutDir @@ -552,13 +567,14 @@ type name: "graffiti" }: Option[GraffitiBytes] stopAtEpoch* {. - defaultValue: 0 desc: "A positive epoch selects the epoch at which to stop" + defaultValue: 0 name: "stop-at-epoch" }: uint64 rpcPort* {. + desc: "HTTP port of the server to connect to for RPC" defaultValue: defaultEth2RpcPort - desc: "HTTP port of the server to connect to for RPC [=9190]" + defaultValueDesc: "9190" name: "rpc-port" }: Port rpcAddress* {. @@ -568,8 +584,8 @@ type name: "rpc-address" }: ValidIpAddress retryDelay* {. - defaultValue: 10 desc: "Delay in seconds between retries after unsuccessful attempts to connect to a beacon node [=10]" + defaultValue: 10 name: "retry-delay" }: int proc defaultDataDir*(config: BeaconNodeConf|ValidatorClientConf): string = diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 5f2f88215..5f7cfa8d9 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 5f2f882151fee87471514bb74c1311856c636594 +Subproject commit 5f7cfa8d98f6333fc835d49d77b8786d73cd11bb From 1f1367d09a17c7c6622a76e3421bca197e2e36ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 13 May 2021 20:58:52 +0200 Subject: [PATCH 10/14] macOS binary distribution (both AMD64 and ARM64) --- .github/workflows/ci.yml | 9 ++++ .github/workflows/release.yml | 66 ++++++++++++++++++++++- CHANGELOG.md | 42 +++++++-------- Makefile | 12 +++++ config.nims | 27 ++++++++-- docker/dist/Dockerfile.macos | 18 +++++++ docker/dist/Dockerfile.macos-arm64 | 18 +++++++ docker/dist/README.md | 2 +- docker/dist/base_image/Dockerfile.macos | 24 +++++++++ docker/dist/base_image/Makefile | 10 +++- docker/dist/base_image/build_osxcross.sh | 27 ++++++++++ docker/dist/entry_point.sh | 68 ++++++++++++++++++++++++ scripts/compile_nim_program.sh | 4 ++ scripts/make_dist.sh | 3 +- vendor/nim-libbacktrace | 2 +- 15 files changed, 301 insertions(+), 31 deletions(-) create mode 100644 docker/dist/Dockerfile.macos create mode 100644 docker/dist/Dockerfile.macos-arm64 create mode 100644 docker/dist/base_image/Dockerfile.macos create mode 100755 docker/dist/base_image/build_osxcross.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00fabe488..43ca4e8ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,6 +233,9 @@ jobs: shell: bash working-directory: nim-beacon-chain run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + ulimit -n 1024 + fi make -j$ncpu ARCH_OVERRIDE=$PLATFORM CI_CACHE=NimBinaries QUICK_AND_DIRTY_COMPILER=1 update - name: Get latest fixtures commit hash @@ -268,6 +271,9 @@ jobs: shell: bash working-directory: nim-beacon-chain run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + ulimit -n 1024 + fi make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node nimbus_validator_client - name: Run nim-beacon-chain tests @@ -275,6 +281,9 @@ jobs: shell: bash working-directory: nim-beacon-chain run: | + if [[ "${{ runner.os }}" == "macOS" ]]; then + ulimit -n 1024 + fi make -j$ncpu ARCH_OVERRIDE=$PLATFORM DISABLE_TEST_FIXTURES_SCRIPT=1 test # The upload creates a combined report that gets posted as a comment on the PR diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe81ef47e..4e6d89758 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -178,9 +178,63 @@ jobs: name: Windows_amd64_checksum path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum retention-days: 2 + build-macos-amd64: + name: macOS AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-macos + cd dist + ARCHIVE=$(echo nimbus-eth2_macOS_amd64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus_beacon_node.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 + build-macos-arm64: + name: macOS ARM64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-macos-arm64 + cd dist + ARCHIVE=$(echo nimbus-eth2_macOS_arm64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus_beacon_node.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus_beacon_node.sha512sum + retention-days: 2 prepare-release: name: Prepare release draft - needs: [build-amd64, build-arm64, build-arm, build-win64] + needs: [build-amd64, build-arm64, build-arm, build-win64, build-macos-amd64, build-macos-arm64] runs-on: ubuntu-latest steps: - name: Download artefacts @@ -202,6 +256,10 @@ jobs: cat Linux_arm_checksum/* >> release_notes.md echo '# Windows AMD64' >> release_notes.md cat Windows_amd64_checksum/* >> release_notes.md + echo '# macOS AMD64' >> release_notes.md + cat macOS_amd64_checksum/* >> release_notes.md + echo '# macOS ARM64' >> release_notes.md + cat macOS_arm64_checksum/* >> release_notes.md echo '```' >> release_notes.md - name: Create release id: create_release @@ -217,6 +275,8 @@ jobs: Linux_arm64_archive/* Linux_arm_archive/* Windows_amd64_archive/* + macOS_amd64_archive/* + macOS_arm64_archive/* - name: Delete artefacts uses: geekyeggo/delete-artifact@v1 with: @@ -230,4 +290,8 @@ jobs: Linux_arm_checksum Windows_amd64_archive Windows_amd64_checksum + macOS_amd64_archive + macOS_amd64_checksum + macOS_arm64_archive + macOS_arm64_checksum diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e87db119..21ccdec15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,37 +1,37 @@ 2021-05-17 v1.3.0 ================= -A new release offering safer and easier options for migrating to Nimbus from -other clients and bringing further performance optimizations. +This release offers safer and easier options to migrate to Nimbus from other clients. +It also brings further performance optimizations. -**New features:** +**We've added:** -* A new `slashingdb` sub-command offering import and export options for the - EIP-3076 slashing protection interchange format. Please see the the prepared - [migration guides](https://nimbus.guide/migrate.html) outlining the safest - way to migrate to Nimbus from other clients. +* A new `slashingdb` sub-command with `import` and `export` options. This allows for + safely migrating to Nimbus from another client (as per the [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076) slashing + protection interchange format). + Please see the the newly prepared [migration guides](https://nimbus.guide/migrate.html) for the details. -* Pruning of the slashing protection database and transition to more optimal - queries *->* significant reduction of disk and CPU usage on nodes running - large number of validators. +* Pruning of the slashing protection database and a transition to more optimal + queries. This results in a significant reduction in both disk and CPU usage + on nodes running a large number of validators. * More consistent level of validation for the attestations received from - third-party sources and the JSON-RPC and REST APIs, preventing invalid - attestations to be broadcasted to the network. + third-party sources and the JSON-RPC and REST APIs. This prevents invalid + attestations from being broadcasted to the network. * Performance tuning of attestation subnet transition timings and state - snapshotting intervals *->* improved CPU and bandwidth usage. + snapshotting intervals. This results in improved CPU and bandwidth usage. + +* A new `ncli_db validatorPerf` command. This can be used to perform a textual + report for the attestation performance of a particular validator + (please note that `ncli_db` is available only when compiling from source). + +* Official binaries for macOS (AMD64 and ARM64). **We've fixed:** -* Problems in the GossipSub subnet walking logic leading to unnecessary - bandwidth and CPU costs. - -**New tools:** - -* A new `ncli_db validatorPerf` command for producing a textual report for the - attestation performance of a particular validator (please note that `ncli_db` - is available only when compiling from source). +* Problems in the GossipSub subnet walking logic leading to unnecessary bandwidth + and CPU costs. 2021-05-03 v1.2.2 diff --git a/Makefile b/Makefile index 408e4c44e..ce980b552 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,8 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) dist-arm64 \ dist-arm \ dist-win64 \ + dist-macos \ + dist-macos-arm64 \ dist \ benchmarks @@ -582,11 +584,21 @@ dist-win64: + MAKE="$(MAKE)" \ scripts/make_dist.sh win64 +dist-macos: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh macos + +dist-macos-arm64: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh macos-arm64 + dist: + $(MAKE) dist-amd64 + $(MAKE) dist-arm64 + $(MAKE) dist-arm + $(MAKE) dist-win64 + + $(MAKE) dist-macos + + $(MAKE) dist-macos-arm64 #- this simple test will show any missing dynamically-linked Glibc symbols in the target distro dist-test: diff --git a/config.nims b/config.nims index b479ecd65..9ca740380 100644 --- a/config.nims +++ b/config.nims @@ -1,3 +1,5 @@ +import strutils + const nimCachePathOverride {.strdefine.} = "" when nimCachePathOverride == "": when defined(release): @@ -79,8 +81,7 @@ switch("define", "withoutPCRE") switch("import", "testutils/moduletests") -const useLibStackTrace = not defined(macosx) and - not defined(windows) and +const useLibStackTrace = not defined(windows) and not defined(disable_libbacktrace) when useLibStackTrace: @@ -90,9 +91,25 @@ else: --stacktrace:on --linetrace:on -# the default open files limit is too low on macOS (512), breaking the -# "--debugger:native" build. It can be increased with `ulimit -n 1024`. -if not defined(macosx): +var canEnableDebuggingSymbols = true +if defined(macosx): + # The default open files limit is too low on macOS (512), breaking the + # "--debugger:native" build. It can be increased with `ulimit -n 1024`. + let openFilesLimitTarget = 1024 + var openFilesLimit = 0 + try: + openFilesLimit = staticExec("ulimit -n").strip(chars = Whitespace + Newlines).parseInt() + if openFilesLimit < openFilesLimitTarget: + echo "Open files limit too low to enable debugging symbols and lightweight stack traces." + echo "Increase it with \"ulimit -n " & $openFilesLimitTarget & "\"" + canEnableDebuggingSymbols = false + except: + echo "ulimit error" +# We ignore this resource limit on Windows, where a default `ulimit -n` of 256 +# in Git Bash is apparently ignored by the OS, and on Linux where the default of +# 1024 is good enough for us. + +if canEnableDebuggingSymbols: # add debugging symbols and original files and line numbers --debugger:native diff --git a/docker/dist/Dockerfile.macos b/docker/dist/Dockerfile.macos new file mode 100644 index 000000000..37b30035b --- /dev/null +++ b/docker/dist/Dockerfile.macos @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus_beacon_node:dist_base_20210513160553_macos@sha256:eef4aff594307c0ff615160aa7184b3660648ce929bb670a409428fc32bd04ed + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "macOS_amd64"] + diff --git a/docker/dist/Dockerfile.macos-arm64 b/docker/dist/Dockerfile.macos-arm64 new file mode 100644 index 000000000..42bb8f85c --- /dev/null +++ b/docker/dist/Dockerfile.macos-arm64 @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus_beacon_node:dist_base_20210513160553_macos@sha256:eef4aff594307c0ff615160aa7184b3660648ce929bb670a409428fc32bd04ed + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "macOS_arm64"] + diff --git a/docker/dist/README.md b/docker/dist/README.md index 192c145b3..b954f7e00 100644 --- a/docker/dist/README.md +++ b/docker/dist/README.md @@ -1,7 +1,7 @@ # Binary Nimbus beacon node distribution This binary distribution of the Nimbus eth2 package is compiled -in [reproducible way](https://reproducible-builds.org/) from source files +in a [reproducible way](https://reproducible-builds.org/) from source files hosted at https://github.com/status-im/nimbus-eth2. The tarball containing this README uses the following naming scheme: diff --git a/docker/dist/base_image/Dockerfile.macos b/docker/dist/base_image/Dockerfile.macos new file mode 100644 index 000000000..8b606a547 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.macos @@ -0,0 +1,24 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential git clang-11 llvm-11-dev cmake curl libssl-dev lzma-dev libxml2-dev &>/dev/null \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 100 \ + && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-11 100 \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY "build_osxcross.sh" "/root/" +RUN cd /root \ + && ./build_osxcross.sh + diff --git a/docker/dist/base_image/Makefile b/docker/dist/base_image/Makefile index b3f4e9bc2..53be2a60f 100644 --- a/docker/dist/base_image/Makefile +++ b/docker/dist/base_image/Makefile @@ -8,10 +8,12 @@ IMAGE_NAME := statusteam/nimbus_beacon_node:$(IMAGE_TAG) build-arm64 \ build-arm \ build-win64 \ + build-macos \ push-amd64 \ push-arm64 \ push-arm \ - push-win64 + push-win64 \ + push-macos build-amd64: $(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)" @@ -25,6 +27,9 @@ build-arm: build-win64: $(CURDIR)/make_base_image.sh win64 "$(IMAGE_NAME)_win64" +build-macos: + $(CURDIR)/make_base_image.sh macos "$(IMAGE_NAME)_macos" + # You probably don't want to recreate and push these base images to Docker Hub, # because when older images expire and get deleted, it will no longer be possible # to reproduce old releases. @@ -41,3 +46,6 @@ build-win64: #push-win64: build-win64 # docker push $(IMAGE_NAME)_win64 +#push-macos: build-macos + #docker push $(IMAGE_NAME)_macos + diff --git a/docker/dist/base_image/build_osxcross.sh b/docker/dist/base_image/build_osxcross.sh new file mode 100755 index 000000000..e65384a15 --- /dev/null +++ b/docker/dist/base_image/build_osxcross.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + + +git clone https://github.com/tpoechtrager/osxcross.git + +# macOS SDK +cd osxcross/tarballs +MACOS_SDK_VER="11.3" +MACOS_SDK_TARBALL="MacOSX${MACOS_SDK_VER}.sdk.tar.xz" +curl -OLsS https://github.com/phracker/MacOSX-SDKs/releases/download/${MACOS_SDK_VER}/${MACOS_SDK_TARBALL} +cd .. + +# build OSXCross toolchain +export TARGET_DIR="/opt/osxcross" +UNATTENDED=1 ./build.sh +# "tools/osxcross_conf.sh" ignores TARGET_DIR and uses "target" instead, so do a symlink +ln -s ${TARGET_DIR} target +./build_llvm_dsymutil.sh +# ridiculous amount of uncompressed man pages +rm -rf ${TARGET_DIR}/SDK/MacOSX${MACOS_SDK_VER}.sdk/usr/share + +# cleanup +cd .. +rm -rf osxcross + diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh index 0ac7c1511..5d2349d9d 100755 --- a/docker/dist/entry_point.sh +++ b/docker/dist/entry_point.sh @@ -90,6 +90,66 @@ elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --cpu:arm64 --gcc.exe=${CC} --gcc.linkerexe=${CC}" \ PARTIAL_STATIC_LINKING=1 \ ${BINARIES} +elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then + export PATH="/opt/osxcross/bin:${PATH}" + export OSXCROSS_MP_INC=1 # sets up include and library paths + export ZERO_AR_DATE=1 # avoid timestamps in binaries + DARWIN_VER="20.4" + CC="o64-clang" + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common build/generate_makefile + make \ + -j$(nproc) \ + CC="${CC}" \ + LIBTOOL="x86_64-apple-darwin${DARWIN_VER}-libtool" \ + OS="darwin" \ + NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:macosx --clang.exe=${CC}" \ + nat-libs + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + AR="x86_64-apple-darwin${DARWIN_VER}-ar" \ + RANLIB="x86_64-apple-darwin${DARWIN_VER}-ranlib" \ + CMAKE="x86_64-apple-darwin${DARWIN_VER}-cmake" \ + DSYMUTIL="x86_64-apple-darwin${DARWIN_VER}-dsymutil" \ + FORCE_DSYMUTIL=1 \ + USE_VENDORED_LIBUNWIND=1 \ + NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:macosx --clang.exe=${CC} --clang.linkerexe=${CC}" \ + ${BINARIES} +elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then + export PATH="/opt/osxcross/bin:${PATH}" + export OSXCROSS_MP_INC=1 # sets up include and library paths + export ZERO_AR_DATE=1 # avoid timestamps in binaries + DARWIN_VER="20.4" + CC="oa64-clang" + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common build/generate_makefile + make \ + -j$(nproc) \ + CC="${CC}" \ + LIBTOOL="arm64-apple-darwin${DARWIN_VER}-libtool" \ + OS="darwin" \ + NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:macosx --cpu:arm64 --clang.exe=${CC}" \ + nat-libs + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + AR="arm64-apple-darwin${DARWIN_VER}-ar" \ + RANLIB="arm64-apple-darwin${DARWIN_VER}-ranlib" \ + CMAKE="arm64-apple-darwin${DARWIN_VER}-cmake" \ + DSYMUTIL="arm64-apple-darwin${DARWIN_VER}-dsymutil" \ + FORCE_DSYMUTIL=1 \ + USE_VENDORED_LIBUNWIND=1 \ + NIMFLAGS="-d:disableMarchNative -d:chronicles_sinks=textlines -d:chronicles_colors=none --os:macosx --cpu:arm64 --clang.exe=${CC} --clang.linkerexe=${CC}" \ + ${BINARIES} else make \ -j$(nproc) \ @@ -119,6 +179,10 @@ mkdir "${DIST_PATH}/build" # copy and checksum binaries, copy scripts and docs for BINARY in ${BINARIES}; do cp -a "./build/${BINARY}" "${DIST_PATH}/build/" + if [[ "${PLATFORM}" =~ macOS ]]; then + # debug info + cp -a "./build/${BINARY}.dSYM" "${DIST_PATH}/build/" + fi cd "${DIST_PATH}/build" sha512sum "${BINARY}" > "${BINARY}.sha512sum" if [[ "${PLATFORM}" == "Windows_amd64" ]]; then @@ -137,6 +201,10 @@ elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then elif [[ "${PLATFORM}" == "Windows_amd64" ]]; then sed -i -e 's/^make dist$/make dist-win64/' "${DIST_PATH}/README.md" cp -a docker/dist/README-Windows.md "${DIST_PATH}/" +elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then + sed -i -e 's/^make dist$/make dist-macos/' "${DIST_PATH}/README.md" +elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then + sed -i -e 's/^make dist$/make dist-macos-arm64/' "${DIST_PATH}/README.md" fi cp -a scripts/run-beacon-node.sh "${DIST_PATH}/scripts" diff --git a/scripts/compile_nim_program.sh b/scripts/compile_nim_program.sh index fbdfd9127..4bb1ba540 100755 --- a/scripts/compile_nim_program.sh +++ b/scripts/compile_nim_program.sh @@ -26,3 +26,7 @@ build/generate_makefile "nimcache/release/${BINARY}/${PROJECT_NAME}.json" "nimca [[ "$V" == "0" ]] && exec &>/dev/null "${MAKE}" -f "nimcache/release/${BINARY}/${BINARY}.makefile" --no-print-directory build +if uname | grep -qi darwin || [[ -n "${FORCE_DSYMUTIL}" ]]; then + [[ -z "${DSYMUTIL}" ]] && DSYMUTIL="dsymutil" + "${DSYMUTIL}" build/${BINARY} +fi diff --git a/scripts/make_dist.sh b/scripts/make_dist.sh index 4c5fc909c..ac3f83b2c 100755 --- a/scripts/make_dist.sh +++ b/scripts/make_dist.sh @@ -30,7 +30,8 @@ DOCKER_BUILDKIT=1 \ --build-arg GROUP_ID=$(id -g) \ -f Dockerfile.${ARCH} . -docker run --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth2 ${DOCKER_TAG} +# seccomp can have some serious overhead, so we disable it with "--privileged" - https://pythonspeed.com/articles/docker-performance-overhead/ +docker run --privileged --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth2 ${DOCKER_TAG} cd - &>/dev/null diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index ce966b1c4..63196b062 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit ce966b1c469dda179b54346feaaf1a62202c984f +Subproject commit 63196b0628fd6742a9467842f771d3b1ed1fb803 From 2cb139696947191e2f6c1ef979d838e4385f0d32 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 May 2021 21:33:32 +0300 Subject: [PATCH 11/14] Log the slashing DB pruning time --- beacon_chain/validators/slashing_protection.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beacon_chain/validators/slashing_protection.nim b/beacon_chain/validators/slashing_protection.nim index 71274fa21..24e6a8948 100644 --- a/beacon_chain/validators/slashing_protection.nim +++ b/beacon_chain/validators/slashing_protection.nim @@ -13,7 +13,7 @@ import # Status eth/db/[kvstore, kvstore_sqlite3], stew/[results, byteutils], - chronicles, + chronicles, chronicles/timings, # Internal ../spec/[datatypes, digest, crypto], ./slashing_protection_common, @@ -273,7 +273,8 @@ proc pruneAfterFinalization*( ## Pruning is only triggered on v2 database. if kLowWatermark in db.modes: - db.db_v2.pruneAfterFinalization(finalizedEpoch) + debug.logTime "Pruning slashing DB": + db.db_v2.pruneAfterFinalization(finalizedEpoch) # The high-level import/export functions are # - importSlashingInterchange From 6b14d33ad6bab8bb034e8f0c9348746e56dfecc6 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 18 May 2021 11:02:01 +0300 Subject: [PATCH 12/14] Bump Confutils This fixes an issue with the --bootstrap-file parameter which was accidentally considered mandatory due to an upstream issue. --- vendor/nim-confutils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-confutils b/vendor/nim-confutils index 5f7cfa8d9..6036a4700 160000 --- a/vendor/nim-confutils +++ b/vendor/nim-confutils @@ -1 +1 @@ -Subproject commit 5f7cfa8d98f6333fc835d49d77b8786d73cd11bb +Subproject commit 6036a47000e342ba3b31eb74dac995e3cf922b56 From 06dca32786ad9d52ae0817b037dd5889af30d328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Tue, 18 May 2021 19:18:22 +0200 Subject: [PATCH 13/14] CI: daily cron job testing 3 Nim branches --- .github/workflows/cron.yml | 190 +++++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 .github/workflows/cron.yml diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml new file mode 100644 index 000000000..429847f37 --- /dev/null +++ b/.github/workflows/cron.yml @@ -0,0 +1,190 @@ +name: Daily +on: + schedule: + - cron: "45 3 * * *" + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 20 + matrix: + target: + - os: linux + cpu: amd64 + - os: linux + cpu: i386 + - os: macos + cpu: amd64 + - os: windows + cpu: amd64 + - os: windows + cpu: i386 + branch: [version-1-2, version-1-4, devel] + include: + - target: + os: linux + builder: ubuntu-18.04 + shell: bash + - target: + os: macos + builder: macos-10.15 + shell: bash + - target: + os: windows + builder: windows-2019 + shell: msys2 {0} + + defaults: + run: + shell: ${{ matrix.shell }} + + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' + runs-on: ${{ matrix.builder }} + continue-on-error: ${{ matrix.branch }} == 'devel' + steps: + - name: Checkout nimbus-eth2 + uses: actions/checkout@v2 + with: + ref: unstable + + - name: Derive environment variables + shell: bash + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + PLATFORM=x64 + else + PLATFORM=x86 + fi + echo "PLATFORM=${PLATFORM}" >> $GITHUB_ENV + + # libminiupnp / natpmp + if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'i386' ]]; then + export CFLAGS="${CFLAGS} -m32 -mno-adx" + echo "CFLAGS=${CFLAGS}" >> $GITHUB_ENV + fi + + ncpu="" + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=${NUMBER_OF_PROCESSORS} + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=${ncpu}" >> $GITHUB_ENV + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-fast update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \ + --no-install-recommends -yq gcc-multilib g++-multilib + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 -mno-adx "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 -mno-adx "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo "${{ github.workspace }}/external/bin" >> $GITHUB_PATH + + - name: MSYS2 (Windows i386) + if: runner.os == 'Windows' && matrix.target.cpu == 'i386' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + msystem: MINGW32 + install: >- + base-devel + git + mingw-w64-i686-toolchain + + - name: MSYS2 (Windows amd64) + if: runner.os == 'Windows' && matrix.target.cpu == 'amd64' + uses: msys2/setup-msys2@v2 + with: + path-type: inherit + install: >- + base-devel + git + mingw-w64-x86_64-toolchain + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v2 + with: + path: external/dlls-${{ matrix.target.cpu }} + key: 'dlls-${{ matrix.target.cpu }}' + + - name: Install DLLs dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + shell: bash + run: | + mkdir -p external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + + - name: Path to cached dependencies (Windows) + if: > + runner.os == 'Windows' + shell: bash + run: | + echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + + - name: Install build dependencies (macOS) + if: runner.os == 'macOS' + shell: bash + run: | + brew install gnu-getopt + brew link --force gnu-getopt + + - name: Get latest fixtures commit hash + id: fixtures_version + shell: bash + run: | + getHash() { + git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1 + } + fixturesHash=$(getHash status-im/nim-eth2-scenarios) + echo "::set-output name=fixtures::${fixturesHash}" + + - name: Restore Ethereum Foundation fixtures from cache + id: fixtures-cache + uses: actions/cache@v2 + with: + path: fixturesCache + key: 'eth2-scenarios-${{ steps.fixtures_version.outputs.fixtures }}' + + - name: Get the Ethereum Foundation fixtures + shell: bash + run: | + scripts/setup_official_tests.sh fixturesCache + + - name: Build Nim and Nimbus dependencies + shell: bash + run: | + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update + + - name: Smoke test the Beacon Node and Validator Client with all tracing enabled + shell: bash + run: | + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node nimbus_validator_client + + - name: Run nimbus-eth2 tests + shell: bash + run: | + make -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} DISABLE_TEST_FIXTURES_SCRIPT=1 test + From eb55fe241908b35db179aea04ae1db37a7d18b9e Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 19 May 2021 09:25:22 +0300 Subject: [PATCH 14/14] v1.3.0 --- CHANGELOG.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ccdec15..685eab1ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,15 @@ It also brings further performance optimizations. **We've added:** * A new `slashingdb` sub-command with `import` and `export` options. This allows for - safely migrating to Nimbus from another client (as per the [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076) slashing - protection interchange format). - Please see the the newly prepared [migration guides](https://nimbus.guide/migrate.html) for the details. + safely migrating to Nimbus from another client (as per the [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076) + slashing protection interchange format). + Please see the the newly prepared [migration guides](https://nimbus.guide/migration.html) for the details. + +* A new `ncli_db validatorPerf` command. This can be used to perform a textual + report for the attestation performance of a particular validator + (please note that `ncli_db` is available only when compiling from source). + +* Official binaries for macOS (AMD64 and ARM64). * Pruning of the slashing protection database and a transition to more optimal queries. This results in a significant reduction in both disk and CPU usage @@ -22,12 +28,6 @@ It also brings further performance optimizations. * Performance tuning of attestation subnet transition timings and state snapshotting intervals. This results in improved CPU and bandwidth usage. -* A new `ncli_db validatorPerf` command. This can be used to perform a textual - report for the attestation performance of a particular validator - (please note that `ncli_db` is available only when compiling from source). - -* Official binaries for macOS (AMD64 and ARM64). - **We've fixed:** * Problems in the GossipSub subnet walking logic leading to unnecessary bandwidth