Merge branch 'unstable' into feat/splitview

This commit is contained in:
Etan Kissling 2024-03-27 16:14:13 +01:00
commit ce19875583
No known key found for this signature in database
GPG Key ID: B21DA824C5A3D03D
13 changed files with 44 additions and 174 deletions

View File

@ -370,7 +370,7 @@ proc checkBloblessSignature(
let proposer = getProposer(
dag, parent, signed_beacon_block.message.slot).valueOr:
return err("checkBloblessSignature: Cannot compute proposer")
if uint64(proposer) != signed_beacon_block.message.proposer_index:
if distinctBase(proposer) != signed_beacon_block.message.proposer_index:
return err("checkBloblessSignature: Incorrect proposer")
if not verify_block_signature(
dag.forkAtEpoch(signed_beacon_block.message.slot.epoch),
@ -537,6 +537,7 @@ proc storeBlock(
if NewPayloadStatus.invalid == payloadStatus:
self.consensusManager.quarantine[].addUnviable(signedBlock.root)
self[].dumpInvalidBlock(signedBlock)
return err((VerifierError.UnviableFork, ProcessingStatus.completed))
if NewPayloadStatus.noResponse == payloadStatus:

View File

@ -2295,14 +2295,11 @@ proc createEth2Node*(rng: ref HmacDrbgContext,
let phase0Prefix = "/eth2/" & $forkDigests.phase0
func msgIdProvider(m: messages.Message): Result[seq[byte], ValidationResult] =
template topic: untyped =
if m.topicIds.len > 0: m.topicIds[0] else: ""
try:
# This doesn't have to be a tight bound, just enough to avoid denial of
# service attacks.
let decoded = snappy.decode(m.data, static(GOSSIP_MAX_SIZE.uint32))
ok(gossipId(decoded, phase0Prefix, topic))
ok(gossipId(decoded, phase0Prefix, m.topic))
except CatchableError:
err(ValidationResult.Reject)

View File

@ -8,7 +8,7 @@
{.push raises: [].}
import
std/[sequtils, strutils, os],
std/os,
stew/[byteutils, objects], stew/shims/macros, nimcrypto/hash,
web3/[conversions],
web3/primitives as web3types,
@ -16,6 +16,11 @@ import
eth/common/eth_types_json_serialization,
../spec/[eth2_ssz_serialization, forks]
from std/sequtils import deduplicate, filterIt, mapIt
from std/strutils import
escape, parseBiggestUInt, replace, splitLines, startsWith, strip,
toLowerAscii
# TODO(zah):
# We can compress the embedded states with snappy before embedding them here.
@ -236,7 +241,7 @@ when const_preset == "gnosis":
chiadoGenesisSize* {.importc: "gnosis_chiado_genesis_size".}: int
# let `.incbin` in assembly file find the binary file through search path
{.passc: "-I" & vendorDir.}
{.passc: "-I" & escape(vendorDir).}
{.compile: "network_metadata_gnosis.S".}
else:
@ -263,9 +268,6 @@ when const_preset == "gnosis":
checkForkConsistency(network.cfg)
for network in [gnosisMetadata, chiadoMetadata]:
doAssert network.cfg.ALTAIR_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.BELLATRIX_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.CAPELLA_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.DENEB_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.ELECTRA_FORK_EPOCH == FAR_FUTURE_EPOCH
static: doAssert ConsensusFork.high == ConsensusFork.Deneb
@ -287,7 +289,7 @@ elif const_preset == "mainnet":
{.pop.}
# let `.incbin` in assembly file find the binary file through search path
{.passc: "-I" & vendorDir.}
{.passc: "-I" & escape(vendorDir).}
{.compile: "network_metadata_mainnet.S".}
else:
@ -329,9 +331,6 @@ elif const_preset == "mainnet":
checkForkConsistency(network.cfg)
for network in [mainnetMetadata, praterMetadata, sepoliaMetadata, holeskyMetadata]:
doAssert network.cfg.ALTAIR_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.BELLATRIX_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.CAPELLA_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.DENEB_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.ELECTRA_FORK_EPOCH == FAR_FUTURE_EPOCH
static: doAssert ConsensusFork.high == ConsensusFork.Deneb

View File

@ -11,13 +11,11 @@ import
stew/assign2,
json_serialization/std/sets,
chronicles,
../extras,
./datatypes/[phase0, altair, bellatrix],
"."/[eth2_merkleization, forks, signatures, validator]
from std/algorithm import fill
from std/sequtils import anyIt, mapIt, toSeq
from ./datatypes/capella import BeaconState, ExecutionPayloadHeader, Withdrawal
export extras, forks, validator, chronicles
@ -104,30 +102,31 @@ func initiate_validator_exit*(
# Return if validator already initiated exit
let validator = addr state.validators.mitem(index)
trace "Validator exiting",
index = index,
num_validators = state.validators.len,
current_epoch = get_current_epoch(state),
validator_slashed = validator.slashed,
validator_withdrawable_epoch = validator.withdrawable_epoch,
validator_exit_epoch = validator.exit_epoch,
validator_effective_balance = validator.effective_balance
var exit_queue_epoch = compute_activation_exit_epoch(get_current_epoch(state))
var
exit_queue_epoch = compute_activation_exit_epoch(get_current_epoch(state))
exit_queue_churn: uint64
# Compute max exit epoch
for idx in 0..<state.validators.len:
let exit_epoch = state.validators.item(idx).exit_epoch
if exit_epoch != FAR_FUTURE_EPOCH and exit_epoch > exit_queue_epoch:
exit_queue_epoch = exit_epoch
var
exit_queue_churn: int
for idx in 0..<state.validators.len:
if state.validators.item(idx).exit_epoch == exit_queue_epoch:
exit_queue_churn += 1
# Reset exit queue churn counter as the expected exit_queue_epoch updates
# via this essentially max()-but-not-FAR_FUTURE_EPOCH loop to restart the
# counting the second for loop in spec version does. Only the last count,
# the one corresponding to the ultimately correct exit_queue_epoch, won't
# be reset.
exit_queue_churn = 0
if exit_queue_churn.uint64 >= get_validator_churn_limit(cfg, state, cache):
exit_queue_epoch += 1
# Second spec loop body, which there is responsible for taking the already
# known exit_queue_epoch, scanning for all validators with that exit epoch
# and checking if they'll reach validator_churn_limit(state). Do that here
# incrementally to fuse the two loops and save an all-validator iteration.
if exit_epoch == exit_queue_epoch:
inc exit_queue_churn
if exit_queue_churn >= get_validator_churn_limit(cfg, state, cache):
inc exit_queue_epoch
# Set validator exit epoch and withdrawable epoch
validator.exit_epoch = exit_queue_epoch

View File

@ -1213,137 +1213,6 @@ proc proposeBlockAux(
blobsBundle.proofs, blobsBundle.blobs))
else:
Opt.none(seq[BlobSidecar])
# BIG BUG SOURCE: The `let` below cannot be combined with the others above!
# If combined, there are sometimes `SIGSEGV` during `test_keymanager_api`.
# This has only been observed on macOS (aarch64) in Jenkins, not on GitHub.
#
# - macOS 14.2.1 (23C71)
# - Xcode 15.1 (15C65)
# - Nim v1.6.18 (a749a8b742bd0a4272c26a65517275db4720e58a)
#
# Issue has started occuring around 12 Jan 2024, in a CI run for PR #5731.
# The PR did not change anything related to this, suggesting an environment
# or hardware change. The issue is flaky; could have been introduced earlier
# before surfacing in the aforementioned PR. About 30% to hit bug.
#
# [2024-01-12T11:54:21.011Z] Wrote test_keymanager_api/bootstrap_node.enr
# [2024-01-12T11:54:29.294Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
# [2024-01-12T11:54:29.294Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
# [2024-01-12T11:54:34.870Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
# [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(1016) main
# [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(1006) NimMain
# [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(997) PreMain
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1502) atmtest_keymanager_apidotnim_Init000
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1475) main
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1481) main
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(307) startBeaconNode
# [2024-01-12T11:54:34.870Z] beacon_chain/nimbus_beacon_node.nim(1900) start
# [2024-01-12T11:54:34.870Z] beacon_chain/nimbus_beacon_node.nim(1847) run
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1465) delayedTests
# [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(392) runTests
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
# [2024-01-12T11:54:34.870Z] vendor/nim-unittest2/unittest2.nim(1147) runTests
# [2024-01-12T11:54:34.870Z] vendor/nim-unittest2/unittest2.nim(1086) runDirect
# [2024-01-12T11:54:34.870Z] vendor/nim-testutils/testutils/unittests.nim(16) runTestX60gensym2933
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(656) waitFor
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(631) pollFor
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
# [2024-01-12T11:54:34.870Z] beacon_chain/validators/beacon_validators.nim(82) proposeBlockAux
# [2024-01-12T11:54:34.870Z] vendor/nimbus-build-system/vendor/Nim/lib/system/excpt.nim(631) signalHandler
# [2024-01-12T11:54:34.870Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
#
# This appeared again around 25 Feb 2024, in a CI run for PR #5959,
# despite the extra `let` having been applied -- once more observed on
# macOS (aarch64) in Jenkins, and much rarer than before.
#
# [2024-02-25T23:21:24.533Z] Wrote test_keymanager_api/bootstrap_node.enr
# [2024-02-25T23:21:32.756Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
# [2024-02-25T23:21:32.756Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
# [2024-02-25T23:21:37.219Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
# [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1068) main
# [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1058) NimMain
# [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1049) PreMain
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1501) atmtest_keymanager_apidotnim_Init000
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1474) main
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1480) main
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(307) startBeaconNode
# [2024-02-25T23:21:37.219Z] beacon_chain/nimbus_beacon_node.nim(1916) start
# [2024-02-25T23:21:37.219Z] beacon_chain/nimbus_beacon_node.nim(1863) run
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1464) delayedTests
# [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(391) runTests
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
# [2024-02-25T23:21:37.219Z] vendor/nim-unittest2/unittest2.nim(1151) runTests
# [2024-02-25T23:21:37.219Z] vendor/nim-unittest2/unittest2.nim(1086) runDirect
# [2024-02-25T23:21:37.219Z] vendor/nim-testutils/testutils/unittests.nim(16) runTestX60gensym3188
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(660) waitFor
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(635) pollFor
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
# [2024-02-25T23:21:37.219Z] vendor/nim-chronicles/chronicles.nim(251) proposeBlockAux
# [2024-02-25T23:21:37.219Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
#
# One theory is that PR #5946 may increase the frequency, as there were
# times where the Jenkins CI failed almost every time using a shorter trace.
# However, the problem was once more flaky, with some passes in-between.
# For now, PR #5946 was reverted (low priority), and the problem is gone,
# whether related or not.
#
# [2024-02-23T23:11:47.700Z] Wrote test_keymanager_api/bootstrap_node.enr
# [2024-02-23T23:11:54.728Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
# [2024-02-23T23:11:54.728Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
# [2024-02-23T23:11:59.523Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
# [2024-02-23T23:11:59.523Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1067) main
# [2024-02-23T23:11:59.523Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1057) NimMain
# [2024-02-23T23:11:59.523Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-02-23T23:11:59.523Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
# [2024-02-23T23:11:59.523Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
#
# The generated `nimcache` differs slightly if the `let` are separated from
# a single block; separation introduces an additional state in closure iter.
# This change, maybe combined with some macOS specific compiler specifics,
# could this trigger the `SIGSEGV`? Maybe the extra state adds just enough
# complexity to the function to disable certain problematic optimizations?
# The change in size of the environment changes a number of things such as
# alignment and which parts of an environment contain pointers and so on,
# which in turn may have surprising behavioural effects, ie most likely this
# extra state masks some underlying issue. Furthermore, the combination of
# `(await xyz).valueOr: return` is not very commonly used with other `await`
# in the same `let` block, which could explain this not being more common.
#
# Note that when compiling for Wasm, there are similar bugs with `results`
# when inlining unwraps, e.g., in `eth2_rest_serialization.nim`.
# These have not been investigated thoroughly so far as that project uses
# Nim 2.0 with --mm:orc and is just a prototype for Wasm, no production use.
# But maybe there is something weird going on with `results` related to the
# random `SIGSEGV` that we are now observing here, related to doing too much
# inline logic without defining intermediate isolated `let` statements.
#
# if mediaType == ApplicationJsonMediaType:
# try:
# - ok RestJson.decode(value, T,
# - requireAllFields = true,
# - allowUnknownFields = true)
# + let r = RestJson.decode(value, T,
# + requireAllFields = true,
# + allowUnknownFields = true)
# + ok r
# except SerializationError as exc:
# debug "Failed to deserialize REST JSON data",
# err = exc.formatMsg("<data>"),
#
# At this time we can only speculate about the trigger of these issues.
# Until a shared pattern can be identified, it is better to apply
# workarounds that at least avoid the known to be reachable triggers.
# The solution is hacky and far from desirable; it is what it is.
let
newBlockRef = (
await node.router.routeSignedBeaconBlock(signedBlock, blobsOpt)
).valueOr:

11
env.sh
View File

@ -1,9 +1,15 @@
#!/usr/bin/env bash
# beacon_chain
# Copyright (c) 2020-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
# and we fall back to a Zsh-specific special var to also support Zsh.
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
ABS_PATH="$(cd ${REL_PATH}; pwd)"
ABS_PATH="$(cd "${REL_PATH}"; pwd)"
# Activate nvm only when this file is sourced without arguments:
if [ -z "$*" ]; then
@ -32,5 +38,4 @@ if [ -f "${USER_ENV_FILE}" ]; then
set +o allexport
fi
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
source "${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh"

@ -1 +1 @@
Subproject commit ab3ab545be0b550cca1c2529f7e97fbebf5eba81
Subproject commit 5b79c5ed5e460b19d8d8afc241b5f5a02de628a6

2
vendor/nim-chronos vendored

@ -1 +1 @@
Subproject commit 47cc17719f4293bf80a22ebe28e3bfc54b2a59a1
Subproject commit ef1b077adfdc803fcce880e81a5740b964bac0bc

2
vendor/nim-kzg4844 vendored

@ -1 +1 @@
Subproject commit 057f7c653e1abe91cca9aac2f94832f39228ea98
Subproject commit 4fbcfbe4c452313bd440936318a87ed708987d8b

@ -1 +1 @@
Subproject commit 027570111c161d8378bca9e84b5f75500a8c38a3
Subproject commit 9f4fd37153a84a7039b7e6a355a1c3374b0ba4ce

2
vendor/nim-libp2p vendored

@ -1 +1 @@
Subproject commit 28609597d104a9be880ed5e1648e1ce18ca9dc38
Subproject commit 458b0885dd276afeb81b9a6d431dde79e99f2b01

2
vendor/nim-web3 vendored

@ -1 +1 @@
Subproject commit 285d97c2b05bbe2a13dab4b52ea878157fb1a1a1
Subproject commit 85e34e8ab2767f3da1d5c166d695666d42ff0c96

@ -1 +1 @@
Subproject commit 3866a8ab98fc6e0e6d406b88800aed72163d5fd4
Subproject commit 14e0c55e89f0bd529f39b259b0f88196277ac08a