Merge branch 'unstable' into feat/splitview

beacon_chain/gossip_processing/block_processor.nim

@@ -370,7 +370,7 @@ proc checkBloblessSignature(
   let proposer = getProposer(
         dag, parent, signed_beacon_block.message.slot).valueOr:
     return err("checkBloblessSignature: Cannot compute proposer")
-  if uint64(proposer) != signed_beacon_block.message.proposer_index:
+  if distinctBase(proposer) != signed_beacon_block.message.proposer_index:
     return err("checkBloblessSignature: Incorrect proposer")
   if not verify_block_signature(
       dag.forkAtEpoch(signed_beacon_block.message.slot.epoch),
@@ -537,6 +537,7 @@ proc storeBlock(
 
   if NewPayloadStatus.invalid == payloadStatus:
     self.consensusManager.quarantine[].addUnviable(signedBlock.root)
+    self[].dumpInvalidBlock(signedBlock)
     return err((VerifierError.UnviableFork, ProcessingStatus.completed))
 
   if NewPayloadStatus.noResponse == payloadStatus:

beacon_chain/networking/eth2_network.nim

@@ -2295,14 +2295,11 @@ proc createEth2Node*(rng: ref HmacDrbgContext,
   let phase0Prefix = "/eth2/" & $forkDigests.phase0
 
   func msgIdProvider(m: messages.Message): Result[seq[byte], ValidationResult] =
-    template topic: untyped =
-      if m.topicIds.len > 0: m.topicIds[0] else: ""
-
     try:
       # This doesn't have to be a tight bound, just enough to avoid denial of
       # service attacks.
       let decoded = snappy.decode(m.data, static(GOSSIP_MAX_SIZE.uint32))
-      ok(gossipId(decoded, phase0Prefix, topic))
+      ok(gossipId(decoded, phase0Prefix, m.topic))
     except CatchableError:
       err(ValidationResult.Reject)
 

beacon_chain/networking/network_metadata.nim

@@ -8,7 +8,7 @@
 {.push raises: [].}
 
 import
-  std/[sequtils, strutils, os],
+  std/os,
   stew/[byteutils, objects], stew/shims/macros, nimcrypto/hash,
   web3/[conversions],
   web3/primitives as web3types,
@@ -16,6 +16,11 @@ import
   eth/common/eth_types_json_serialization,
   ../spec/[eth2_ssz_serialization, forks]
 
+from std/sequtils import deduplicate, filterIt, mapIt
+from std/strutils import
+  escape, parseBiggestUInt, replace, splitLines, startsWith, strip,
+  toLowerAscii
+
 # TODO(zah):
 # We can compress the embedded states with snappy before embedding them here.
 
@@ -236,7 +241,7 @@ when const_preset == "gnosis":
       chiadoGenesisSize* {.importc: "gnosis_chiado_genesis_size".}: int
 
     # let `.incbin` in assembly file find the binary file through search path
-    {.passc: "-I" & vendorDir.}
+    {.passc: "-I" & escape(vendorDir).}
     {.compile: "network_metadata_gnosis.S".}
 
   else:
@@ -263,9 +268,6 @@ when const_preset == "gnosis":
       checkForkConsistency(network.cfg)
 
     for network in [gnosisMetadata, chiadoMetadata]:
-      doAssert network.cfg.ALTAIR_FORK_EPOCH < FAR_FUTURE_EPOCH
-      doAssert network.cfg.BELLATRIX_FORK_EPOCH < FAR_FUTURE_EPOCH
-      doAssert network.cfg.CAPELLA_FORK_EPOCH < FAR_FUTURE_EPOCH
       doAssert network.cfg.DENEB_FORK_EPOCH < FAR_FUTURE_EPOCH
       doAssert network.cfg.ELECTRA_FORK_EPOCH == FAR_FUTURE_EPOCH
       static: doAssert ConsensusFork.high == ConsensusFork.Deneb
@@ -287,7 +289,7 @@ elif const_preset == "mainnet":
     {.pop.}
 
     # let `.incbin` in assembly file find the binary file through search path
-    {.passc: "-I" & vendorDir.}
+    {.passc: "-I" & escape(vendorDir).}
     {.compile: "network_metadata_mainnet.S".}
 
   else:
@@ -329,9 +331,6 @@ elif const_preset == "mainnet":
       checkForkConsistency(network.cfg)
 
     for network in [mainnetMetadata, praterMetadata, sepoliaMetadata, holeskyMetadata]:
-      doAssert network.cfg.ALTAIR_FORK_EPOCH < FAR_FUTURE_EPOCH
-      doAssert network.cfg.BELLATRIX_FORK_EPOCH < FAR_FUTURE_EPOCH
-      doAssert network.cfg.CAPELLA_FORK_EPOCH < FAR_FUTURE_EPOCH
       doAssert network.cfg.DENEB_FORK_EPOCH < FAR_FUTURE_EPOCH
       doAssert network.cfg.ELECTRA_FORK_EPOCH == FAR_FUTURE_EPOCH
       static: doAssert ConsensusFork.high == ConsensusFork.Deneb

beacon_chain/spec/beaconstate.nim

@@ -11,13 +11,11 @@ import
   stew/assign2,
   json_serialization/std/sets,
   chronicles,
-  ../extras,
   ./datatypes/[phase0, altair, bellatrix],
   "."/[eth2_merkleization, forks, signatures, validator]
 
 from std/algorithm import fill
 from std/sequtils import anyIt, mapIt, toSeq
-
 from ./datatypes/capella import BeaconState, ExecutionPayloadHeader, Withdrawal
 
 export extras, forks, validator, chronicles
@@ -104,30 +102,31 @@ func initiate_validator_exit*(
   # Return if validator already initiated exit
   let validator = addr state.validators.mitem(index)
 
-  trace "Validator exiting",
+  var
-    index = index,
+    exit_queue_epoch = compute_activation_exit_epoch(get_current_epoch(state))
-    num_validators = state.validators.len,
+    exit_queue_churn: uint64
-    current_epoch = get_current_epoch(state),
-    validator_slashed = validator.slashed,
-    validator_withdrawable_epoch = validator.withdrawable_epoch,
-    validator_exit_epoch = validator.exit_epoch,
-    validator_effective_balance = validator.effective_balance
-
-  var exit_queue_epoch = compute_activation_exit_epoch(get_current_epoch(state))
   # Compute max exit epoch
   for idx in 0..<state.validators.len:
     let exit_epoch = state.validators.item(idx).exit_epoch
     if exit_epoch != FAR_FUTURE_EPOCH and exit_epoch > exit_queue_epoch:
       exit_queue_epoch = exit_epoch
 
-  var
+      # Reset exit queue churn counter as the expected exit_queue_epoch updates
-    exit_queue_churn: int
+      # via this essentially max()-but-not-FAR_FUTURE_EPOCH loop to restart the
-  for idx in 0..<state.validators.len:
+      # counting the second for loop in spec version does. Only the last count,
-    if state.validators.item(idx).exit_epoch == exit_queue_epoch:
+      # the one corresponding to the ultimately correct exit_queue_epoch, won't
-      exit_queue_churn += 1
+      # be reset.
+      exit_queue_churn = 0
 
-  if exit_queue_churn.uint64 >= get_validator_churn_limit(cfg, state, cache):
+    # Second spec loop body, which there is responsible for taking the already
-    exit_queue_epoch += 1
+    # known exit_queue_epoch, scanning for all validators with that exit epoch
+    # and checking if they'll reach validator_churn_limit(state). Do that here
+    # incrementally to fuse the two loops and save an all-validator iteration.
+    if exit_epoch == exit_queue_epoch:
+      inc exit_queue_churn
+
+  if exit_queue_churn >= get_validator_churn_limit(cfg, state, cache):
+    inc exit_queue_epoch
 
   # Set validator exit epoch and withdrawable epoch
   validator.exit_epoch = exit_queue_epoch

beacon_chain/validators/beacon_validators.nim

@@ -1213,137 +1213,6 @@ proc proposeBlockAux(
             blobsBundle.proofs, blobsBundle.blobs))
         else:
           Opt.none(seq[BlobSidecar])
-
-    # BIG BUG SOURCE: The `let` below cannot be combined with the others above!
-    # If combined, there are sometimes `SIGSEGV` during `test_keymanager_api`.
-    # This has only been observed on macOS (aarch64) in Jenkins, not on GitHub.
-    #
-    # - macOS 14.2.1 (23C71)
-    # - Xcode 15.1 (15C65)
-    # - Nim v1.6.18 (a749a8b742bd0a4272c26a65517275db4720e58a)
-    #
-    # Issue has started occuring around 12 Jan 2024, in a CI run for PR #5731.
-    # The PR did not change anything related to this, suggesting an environment
-    # or hardware change. The issue is flaky; could have been introduced earlier
-    # before surfacing in the aforementioned PR. About 30% to hit bug.
-    #
-    # [2024-01-12T11:54:21.011Z] Wrote test_keymanager_api/bootstrap_node.enr
-    # [2024-01-12T11:54:29.294Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
-    # [2024-01-12T11:54:29.294Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
-    # [2024-01-12T11:54:34.870Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
-    # [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(1016) main
-    # [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(1006) NimMain
-    # [2024-01-12T11:54:34.870Z] vendor/nim-libp2p/libp2p/protocols/rendezvous.nim(997) PreMain
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1502) atmtest_keymanager_apidotnim_Init000
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1475) main
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1481) main
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(307) startBeaconNode
-    # [2024-01-12T11:54:34.870Z] beacon_chain/nimbus_beacon_node.nim(1900) start
-    # [2024-01-12T11:54:34.870Z] beacon_chain/nimbus_beacon_node.nim(1847) run
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(1465) delayedTests
-    # [2024-01-12T11:54:34.870Z] tests/test_keymanager_api.nim(392) runTests
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
-    # [2024-01-12T11:54:34.870Z] vendor/nim-unittest2/unittest2.nim(1147) runTests
-    # [2024-01-12T11:54:34.870Z] vendor/nim-unittest2/unittest2.nim(1086) runDirect
-    # [2024-01-12T11:54:34.870Z] vendor/nim-testutils/testutils/unittests.nim(16) runTestX60gensym2933
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(656) waitFor
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(631) pollFor
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-01-12T11:54:34.870Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(378) futureContinue
-    # [2024-01-12T11:54:34.870Z] beacon_chain/validators/beacon_validators.nim(82) proposeBlockAux
-    # [2024-01-12T11:54:34.870Z] vendor/nimbus-build-system/vendor/Nim/lib/system/excpt.nim(631) signalHandler
-    # [2024-01-12T11:54:34.870Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
-    #
-    # This appeared again around 25 Feb 2024, in a CI run for PR #5959,
-    # despite the extra `let` having been applied -- once more observed on
-    # macOS (aarch64) in Jenkins, and much rarer than before.
-    #
-    # [2024-02-25T23:21:24.533Z] Wrote test_keymanager_api/bootstrap_node.enr
-    # [2024-02-25T23:21:32.756Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
-    # [2024-02-25T23:21:32.756Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
-    # [2024-02-25T23:21:37.219Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
-    # [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1068) main
-    # [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1058) NimMain
-    # [2024-02-25T23:21:37.219Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1049) PreMain
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1501) atmtest_keymanager_apidotnim_Init000
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1474) main
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1480) main
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(307) startBeaconNode
-    # [2024-02-25T23:21:37.219Z] beacon_chain/nimbus_beacon_node.nim(1916) start
-    # [2024-02-25T23:21:37.219Z] beacon_chain/nimbus_beacon_node.nim(1863) run
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(1464) delayedTests
-    # [2024-02-25T23:21:37.219Z] tests/test_keymanager_api.nim(391) runTests
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
-    # [2024-02-25T23:21:37.219Z] vendor/nim-unittest2/unittest2.nim(1151) runTests
-    # [2024-02-25T23:21:37.219Z] vendor/nim-unittest2/unittest2.nim(1086) runDirect
-    # [2024-02-25T23:21:37.219Z] vendor/nim-testutils/testutils/unittests.nim(16) runTestX60gensym3188
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(660) waitFor
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(635) pollFor
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronos/chronos/internal/asyncfutures.nim(382) futureContinue
-    # [2024-02-25T23:21:37.219Z] vendor/nim-chronicles/chronicles.nim(251) proposeBlockAux
-    # [2024-02-25T23:21:37.219Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
-    #
-    # One theory is that PR #5946 may increase the frequency, as there were
-    # times where the Jenkins CI failed almost every time using a shorter trace.
-    # However, the problem was once more flaky, with some passes in-between.
-    # For now, PR #5946 was reverted (low priority), and the problem is gone,
-    # whether related or not.
-    #
-    # [2024-02-23T23:11:47.700Z] Wrote test_keymanager_api/bootstrap_node.enr
-    # [2024-02-23T23:11:54.728Z] Serialization/deserialization [Beacon Node] [Preset: mainnet] . (0.00s)
-    # [2024-02-23T23:11:54.728Z] ListKeys requests [Beacon Node] [Preset: mainnet] .... (0.01s)
-    # [2024-02-23T23:11:59.523Z] ImportKeystores requests [Beacon Node] [Preset: mainnet] Traceback (most recent call last, using override)
-    # [2024-02-23T23:11:59.523Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1067) main
-    # [2024-02-23T23:11:59.523Z] vendor/nim-libp2p/libp2p/protocols/pubsub/pubsub.nim(1057) NimMain
-    # [2024-02-23T23:11:59.523Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-02-23T23:11:59.523Z] vendor/nim-chronos/chronos/internal/asyncengine.nim(150) poll
-    # [2024-02-23T23:11:59.523Z] SIGSEGV: Illegal storage access. (Attempt to read from nil?)
-    #
-    # The generated `nimcache` differs slightly if the `let` are separated from
-    # a single block; separation introduces an additional state in closure iter.
-    # This change, maybe combined with some macOS specific compiler specifics,
-    # could this trigger the `SIGSEGV`? Maybe the extra state adds just enough
-    # complexity to the function to disable certain problematic optimizations?
-    # The change in size of the environment changes a number of things such as
-    # alignment and which parts of an environment contain pointers and so on,
-    # which in turn may have surprising behavioural effects, ie most likely this
-    # extra state masks some underlying issue. Furthermore, the combination of
-    # `(await xyz).valueOr: return` is not very commonly used with other `await`
-    # in the same `let` block, which could explain this not being more common.
-    #
-    # Note that when compiling for Wasm, there are similar bugs with `results`
-    # when inlining unwraps, e.g., in `eth2_rest_serialization.nim`.
-    # These have not been investigated thoroughly so far as that project uses
-    # Nim 2.0 with --mm:orc and is just a prototype for Wasm, no production use.
-    # But maybe there is something weird going on with `results` related to the
-    # random `SIGSEGV` that we are now observing here, related to doing too much
-    # inline logic without defining intermediate isolated `let` statements.
-    #
-    #    if mediaType == ApplicationJsonMediaType:
-    #      try:
-    # -      ok RestJson.decode(value, T,
-    # -                         requireAllFields = true,
-    # -                         allowUnknownFields = true)
-    # +      let r = RestJson.decode(value, T,
-    # +                              requireAllFields = true,
-    # +                              allowUnknownFields = true)
-    # +      ok r
-    #      except SerializationError as exc:
-    #        debug "Failed to deserialize REST JSON data",
-    #              err = exc.formatMsg("<data>"),
-    #
-    # At this time we can only speculate about the trigger of these issues.
-    # Until a shared pattern can be identified, it is better to apply
-    # workarounds that at least avoid the known to be reachable triggers.
-    # The solution is hacky and far from desirable; it is what it is.
-    let
       newBlockRef = (
         await node.router.routeSignedBeaconBlock(signedBlock, blobsOpt)
       ).valueOr:

env.sh

@@ -1,9 +1,15 @@
 #!/usr/bin/env bash
+# beacon_chain
+# Copyright (c) 2020-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 # We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
 # and we fall back to a Zsh-specific special var to also support Zsh.
 REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-ABS_PATH="$(cd ${REL_PATH}; pwd)"
+ABS_PATH="$(cd "${REL_PATH}"; pwd)"
 
 # Activate nvm only when this file is sourced without arguments:
 if [ -z "$*" ]; then
@@ -32,5 +38,4 @@ if [ -f "${USER_ENV_FILE}" ]; then
   set +o allexport
 fi
 
-source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
+source "${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh"
-

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit ab3ab545be0b550cca1c2529f7e97fbebf5eba81
+Subproject commit 5b79c5ed5e460b19d8d8afc241b5f5a02de628a6

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 47cc17719f4293bf80a22ebe28e3bfc54b2a59a1
+Subproject commit ef1b077adfdc803fcce880e81a5740b964bac0bc

vendor/nim-kzg4844

@@ -1 +1 @@
-Subproject commit 057f7c653e1abe91cca9aac2f94832f39228ea98
+Subproject commit 4fbcfbe4c452313bd440936318a87ed708987d8b

vendor/nim-libbacktrace

@@ -1 +1 @@
-Subproject commit 027570111c161d8378bca9e84b5f75500a8c38a3
+Subproject commit 9f4fd37153a84a7039b7e6a355a1c3374b0ba4ce

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit 28609597d104a9be880ed5e1648e1ce18ca9dc38
+Subproject commit 458b0885dd276afeb81b9a6d431dde79e99f2b01

vendor/nim-web3

@@ -1 +1 @@
-Subproject commit 285d97c2b05bbe2a13dab4b52ea878157fb1a1a1
+Subproject commit 85e34e8ab2767f3da1d5c166d695666d42ff0c96

vendor/nimbus-build-system

@@ -1 +1 @@
-Subproject commit 3866a8ab98fc6e0e6d406b88800aed72163d5fd4
+Subproject commit 14e0c55e89f0bd529f39b259b0f88196277ac08a