From be756454a4a907d3bb05f65208561775e809a5ed Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 29 Jun 2021 15:53:36 +0200 Subject: [PATCH 1/6] flush stdout logs (#2669) else pipes end up not getting log output until much later --- beacon_chain/nimbus_binary_common.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_chain/nimbus_binary_common.nim b/beacon_chain/nimbus_binary_common.nim index 7a6378f7f..0ce0cad41 100644 --- a/beacon_chain/nimbus_binary_common.nim +++ b/beacon_chain/nimbus_binary_common.nim @@ -31,6 +31,7 @@ proc setupStdoutLogging*(logLevel: string) = proc (logLevel: LogLevel, msg: LogOutputStr) {.gcsafe, raises: [Defect].} = try: stdout.write(msg) + stdout.flushFile() except IOError as err: logLoggingFailure(cstring(msg), err) From bd684d0eeae5e2893e7213b4ad2c0376c9f1e5dc Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 6 Jul 2021 15:11:18 +0200 Subject: [PATCH 2/6] increase block attestation wait time (#2705) We generally send out attestations 250 ms after the block arrives. Recent efficiency improvements have led to a slightly increased incidence of "slot 0" issues where attestations are dropped by other nodes because they have not yet had time to process the block due to epoch processing taking time. This PR mitigates the problem by increasing the window between receiving the block and sending out attestations. --- beacon_chain/validators/validator_duties.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/beacon_chain/validators/validator_duties.nim b/beacon_chain/validators/validator_duties.nim index b4caacc88..6d0804ffe 100644 --- a/beacon_chain/validators/validator_duties.nim +++ b/beacon_chain/validators/validator_duties.nim @@ -688,14 +688,18 @@ proc handleValidatorDuties*(node: BeaconNode, lastSlot, slot: Slot) {.async.} = # An opposite case is that we received (or produced) a block that has # not yet reached our neighbours. To protect against our attestations # being dropped (because the others have not yet seen the block), we'll - # impose a minimum delay of 250ms. The delay is enforced only when we're + # impose a minimum delay of 1000ms. The delay is enforced only when we're # not hitting the "normal" cutoff time for sending out attestations. + # An earlier delay of 250ms has proven to be not enough, increasing the + # risk of losing attestations. + # Regardless, because we "just" received the block, we'll impose the + # delay. - const afterBlockDelay = 250 + const afterBlockDelay = 1000 let afterBlockTime = node.beaconClock.now() + millis(afterBlockDelay) afterBlockCutoff = node.beaconClock.fromNow( - min(afterBlockTime, attestationCutoffTime)) + min(afterBlockTime, attestationCutoffTime + millis(afterBlockDelay))) if afterBlockCutoff.inFuture: debug "Got block, waiting to send attestations", From bbced08f6362d9d212eb489194105d858d7046fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 7 Jul 2021 10:19:41 +0200 Subject: [PATCH 3/6] fix metrics on Windows (#2707) --- vendor/nim-eth | 2 +- vendor/nim-web3 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/nim-eth b/vendor/nim-eth index 3514ee648..2dfd352fd 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 3514ee6484038b7e66540f5b5853831198562412 +Subproject commit 2dfd352fd0635171f8cb9a86db94b352d63012e5 diff --git a/vendor/nim-web3 b/vendor/nim-web3 index 57f86f752..97e05aea6 160000 --- a/vendor/nim-web3 +++ b/vendor/nim-web3 @@ -1 +1 @@ -Subproject commit 57f86f752b0b4c4ec1f6caf0b8d2eb9870112f4e +Subproject commit 97e05aea6573d2630e318e7777a54d95db6ec40e From 6f441191e40253f6c1d00ff6cccf9ff9d79ae2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mamy=20Andr=C3=A9-Ratsimbazafy?= Date: Sat, 10 Jul 2021 15:18:32 +0200 Subject: [PATCH 4/6] v1.4.1 changelog and version --- CHANGELOG.md | 24 ++++++++++++++++++++++++ beacon_chain/version.nim | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66fcdf116..fee4a440a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +2021-07-10 v1.4.1 +================= + +Nimbus v1.4.1 - "Every attestation counts" + +This release is marked as `low-urgency` + +## Notable changes + +Nimbus `v1.4.0` users might have noticed that they are missing a small number of (seemingly random) attestations since the update. Our investigation into the matter has showed that, due to `v1.4.0`'s significant performance improvements, Nimbus validators occasionally send their first attestation for a new epoch before some peers are ready. These "slow" peers end up dropping early attestations because they're busy with the epoch transition. + +It's a rare occurrence, since it requires a validator to be scheduled to attest in the first slot of an epoch *and* for the beacon node to only be connected to "slow" peers for the respective libp2p topic. If both these conditions are true, a premature attestation may be lost in time, like tears in the rain. + +As a fix, we are using a larger send delay: [#2705](https://github.com/status-im/nimbus-eth2/pull/2705). + +Fo those Nimbus `v1.4.0` users who are concerned about reaching optimal attestation effectiveness, we encourage you to upgrade as soon as possible. + +Other changes include log flushing and metrics fixes. + +Full list: +- increase attestation wait time ([#2705](https://github.com/status-im/nimbus-eth2/pull/2705)) +- ensure logs are printed without delays ([#2669](https://github.com/status-im/nimbus-eth2/pull/2669)) +- fix metrics on Windows ([#2707](https://github.com/status-im/nimbus-eth2/pull/2707)) + 2021-06-21 v1.4.0 ================= diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 95ea89568..e1505f51d 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -16,7 +16,7 @@ when not defined(nimscript): const versionMajor* = 1 versionMinor* = 4 - versionBuild* = 0 + versionBuild* = 1 versionBlob* = "stateofus" # Single word - ends up in the default graffitti From ee79c10a7dd9c0d9dcdd50ee40cc72bcd678dacf Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 5 Aug 2021 10:26:10 +0200 Subject: [PATCH 5/6] update validator key cache on startup (#2760) * update validator key cache on startup Versions prior to 1.1.0 do not write a validator key cache at all. Versions from 1.4.0 and upwards require an immutable validator key cache to verify blocks - normally, block verification fills the cache but that assumes that at least one block was verified by a version that has the key cache. Taken together, this breaks direct upgrades from anything <1.1.0 to 1.4.0. The fix is simply to refresh fill the cache from an existing state on startup. * also log serious block validation failures at info level --- beacon_chain/beacon_chain_db.nim | 12 ++++++ .../block_clearance.nim | 16 ++++++-- .../consensus_object_pools/blockchain_dag.nim | 4 ++ tests/test_block_pool.nim | 41 ++++++++++++++++++- 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 1f872a70b..3de7f7552 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -799,3 +799,15 @@ iterator getAncestorSummaries*(db: BeaconChainDB, root: Eth2Digest): newSummaries.add(res) res.root = res.summary.parent_root + +# Test operations used to create broken and/or legacy database + +proc putStateV0*(db: BeaconChainDB, key: Eth2Digest, value: BeaconState) = + # Writes to KVStore, as done in 1.0.12 and earlier + db.v0.backend.putSnappySSZ(subkey(type value, key), value) + +proc putBlockV0*(db: BeaconChainDB, value: TrustedSignedBeaconBlock) = + # Write to KVStore, as done in 1.0.12 and earlier + # In particular, no summary is written here - it should be recreated + # automatically + db.v0.backend.putSnappySSZ(subkey(SignedBeaconBlock, value.root), value) diff --git a/beacon_chain/consensus_object_pools/block_clearance.nim b/beacon_chain/consensus_object_pools/block_clearance.nim index c1510f65f..ac24771c1 100644 --- a/beacon_chain/consensus_object_pools/block_clearance.nim +++ b/beacon_chain/consensus_object_pools/block_clearance.nim @@ -205,11 +205,15 @@ proc addRawBlockKnownParent( onBlockAdded: OnBlockAdded ): Result[BlockRef, (ValidationResult, BlockError)] = ## Add a block whose parent is known, after performing validity checks + logScope: + blck = shortLog(signedBlock.message) + blockRoot = shortLog(signedBlock.root) + signature = shortLog(signedBlock.signature) if parent.slot >= signedBlock.message.slot: # A block whose parent is newer than the block itself is clearly invalid - # discard it immediately - debug "Invalid block slot", + info "Block with invalid parent, dropping", parentBlock = shortLog(parent) return err((ValidationResult.Reject, Invalid)) @@ -225,7 +229,7 @@ proc addRawBlockKnownParent( # correct - from their point of view, the head block they have is the # latest thing that happened on the chain and they're performing their # duty correctly. - debug "Unviable block, dropping", + info "Unviable block, dropping", finalizedHead = shortLog(dag.finalizedHead), tail = shortLog(dag.tail) @@ -249,11 +253,15 @@ proc addRawBlockKnownParent( # TODO: remove skipBLSValidation var sigs: seq[SignatureSet] - if sigs.collectSignatureSets( - signedBlock, dag.db.immutableValidators, dag.clearanceState, cache).isErr(): + if (let e = sigs.collectSignatureSets( + signedBlock, dag.db.immutableValidators, dag.clearanceState, cache); e.isErr()): + info "Unable to load signature sets", + err = e.error() + # A PublicKey or Signature isn't on the BLS12-381 curve return err((ValidationResult.Reject, Invalid)) if not quarantine.batchVerify(sigs): + info "Block signature verification failed" return err((ValidationResult.Reject, Invalid)) let sigVerifyTick = Moment.now() diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index aaf44ebf9..3ffa0975e 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -440,6 +440,10 @@ proc init*(T: type ChainDAGRef, # Pruning metadata dag.lastPrunePoint = dag.finalizedHead + # Fill validator key cache in case we're loading an old database that doesn't + # have a cache + dag.updateValidatorKeys(getStateField(dag.headState, validators).asSeq()) + info "Block dag initialized", head = shortLog(headRef), finalizedHead = shortLog(dag.finalizedHead), diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim index 27bd47b6f..97920a6c8 100644 --- a/tests/test_block_pool.nim +++ b/tests/test_block_pool.nim @@ -12,9 +12,10 @@ import unittest2, stew/assign2, eth/keys, - ../beacon_chain/spec/[datatypes, digest, helpers, state_transition, presets], + ../beacon_chain/spec/[ + beaconstate, datatypes, digest, helpers, state_transition, presets], ../beacon_chain/beacon_node_types, - ../beacon_chain/[beacon_chain_db, ssz], + ../beacon_chain/[beacon_chain_db, ssz, extras], ../beacon_chain/consensus_object_pools/[ blockchain_dag, block_quarantine, block_clearance, statedata_helpers], ./testutil, ./testdbutil, ./testblockutil @@ -523,3 +524,39 @@ suite "chain DAG finalization tests" & preset(): dag2.finalizedHead.blck.root == dag.finalizedHead.blck.root dag2.finalizedHead.slot == dag.finalizedHead.slot hash_tree_root(dag2.headState) == hash_tree_root(dag.headState) + +suite "Old database versions" & preset(): + setup: + let + genState = initialize_beacon_state_from_eth1( + defaultRuntimePreset, + Eth2Digest(), + 0, + makeInitialDeposits(SLOTS_PER_EPOCH.uint64, flags = {skipBlsValidation}), + {skipBlsValidation}) + genBlock = get_initial_beacon_block(genState[]) + quarantine = QuarantineRef.init(keys.newRng()) + + test "pre-1.1.0": + # only kvstore, no immutable validator keys + + let db = BeaconChainDB.new(defaultRuntimePreset, "", inMemory = true) + + # preInit a database to a v1.0.12 state + db.putStateV0(genBlock.message.state_root, genState[]) + db.putBlockV0(genBlock) + db.putTailBlock(genBlock.root) + db.putHeadBlock(genBlock.root) + db.putStateRoot(genBlock.root, genState.slot, genBlock.message.state_root) + db.putGenesisBlockRoot(genBlock.root) + + var + dag = init(ChainDAGRef, defaultRuntimePreset, db) + state = newClone(dag.headState.data) + cache = StateCache() + att0 = makeFullAttestations(state[], dag.tail.root, 0.Slot, cache) + b1 = addTestBlock(state[], dag.tail.root, cache, attestations = att0) + b1Add = dag.addRawBlock(quarantine, b1, nil) + + check: + b1Add.isOk() From 1e958cfb0a15122206773cb815fd3c58b8e98158 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Thu, 5 Aug 2021 14:28:17 +0300 Subject: [PATCH 6/6] version 1.4.2 --- CHANGELOG.md | 22 ++++++++++++++++++++++ beacon_chain/version.nim | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fee4a440a..012c707d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +2021-07-10 v1.4.2 +================= + +Nimbus `v1.4.2` - "Upgrade procedure: Hotfix release" + +This release is marked as `low-urgency` for all Nimbus users other than those who have recently updated to `v1.4.1` from a version earlier than `v1.1.0` - for these users this is a `high-urgency` release. + +## Notable changes +This release fixes an issue in the upgrade procedure when upgrading from a version earlier than `1.1.0` to `1.4.x`. + +**How can I tell if I've been affected?** + +If you've already upgraded to `1.4.1`, you can tell that you've been affected if you're seeing the following` WRN log`: + +``` + Received invalid sequence of blocks +``` + +To re-iterate, this issue only affects users who are upgrading from `1.0.12` or earlier (released on `2021-03-10`), **and have not run any release in between**. Everyone else can ignore this release. + + 2021-07-10 v1.4.1 ================= @@ -22,6 +43,7 @@ Full list: - ensure logs are printed without delays ([#2669](https://github.com/status-im/nimbus-eth2/pull/2669)) - fix metrics on Windows ([#2707](https://github.com/status-im/nimbus-eth2/pull/2707)) + 2021-06-21 v1.4.0 ================= diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index e1505f51d..d88c6ec0b 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -16,7 +16,7 @@ when not defined(nimscript): const versionMajor* = 1 versionMinor* = 4 - versionBuild* = 1 + versionBuild* = 2 versionBlob* = "stateofus" # Single word - ends up in the default graffitti