Merge branch 'unstable' into feat/splitview

AllTests-mainnet.md

@@ -459,8 +459,9 @@ OK: 5/5 Fail: 0/5 Skip: 0/5
 + Roundtrip engine RPC V1 and bellatrix ExecutionPayload representations                     OK
 + Roundtrip engine RPC V2 and capella ExecutionPayload representations                       OK
 + Roundtrip engine RPC V3 and deneb ExecutionPayload representations                         OK
++ Roundtrip engine RPC V4 and electra ExecutionPayload representations                       OK
 ```
-OK: 5/5 Fail: 0/5 Skip: 0/5
+OK: 6/6 Fail: 0/6 Skip: 0/6
 ## Eth2 specific discovery tests
 ```diff
 + Invalid attnets field                                                                      OK
@@ -1018,4 +1019,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 683/688 Fail: 0/688 Skip: 5/688
+OK: 684/689 Fail: 0/689 Skip: 5/689

CHANGELOG.md

@@ -1,3 +1,40 @@
+2024-03-29 v24.3.0
+==================
+
+Nimbus `v24.3.0` is a `low-urgency` upgrade bringing additional beacon API support and resilience to suboptimal network conditions.
+
+### Improvements
+
+* Add keymanager API graffiti endpoints:
+  https://github.com/status-im/nimbus-eth2/pull/6054
+
+* Remember gossip messages longer to avoid potentially slow handling of irrelevant messages:
+  https://github.com/status-im/nimbus-eth2/pull/6098
+
+* Nimbus processes blocks with deposits in a more optimized way:
+  https://github.com/status-im/nimbus-eth2/pull/5982
+
+* Fork choice performance during periods of nonfinality has been improved:
+  https://github.com/status-im/nimbus-eth2/pull/6076
+
+* Nimbus will continue validating even without external chain progression:
+  https://github.com/status-im/nimbus-eth2/pull/6101
+
+* Locally built blocks via the engine API are preferentially selected by default over similarly valuable builder API blocks:
+  https://github.com/status-im/nimbus-eth2/pull/6103
+
+### Fixes
+
+* Add required header `eth-consensus-block-value` in produceBlockV3 REST beacon API call in beacon node:
+  https://github.com/status-im/nimbus-eth2/pull/5873
+
+* Restore usage of certain mainnet bootstrap nodes to enable faster and more reliable node connectivity at startup:
+  https://github.com/status-im/nimbus-eth2/pull/6052
+
+* The network configuration `INACTIVITY_SCORE_RECOVERY_RATE` can now be overridden:
+  https://github.com/status-im/nimbus-eth2/pull/6091
+
+
 2023-02-27 v24.2.2
 ==================
 

beacon_chain/el/el_manager.nim

@@ -482,6 +482,54 @@ func asConsensusType*(payload: engine_api.GetPayloadV3Response):
       blobs: Blobs.init(
         payload.blobsBundle.blobs.mapIt(it.bytes))))
 
+func asConsensusType*(rpcExecutionPayload: ExecutionPayloadV4):
+    electra.ExecutionPayload =
+  template getTransaction(tt: TypedTransaction): bellatrix.Transaction =
+    bellatrix.Transaction.init(tt.distinctBase)
+
+  template getDepositReceipt(dr: DepositReceiptV1): DepositReceipt =
+    DepositReceipt(
+      pubkey: ValidatorPubKey(blob: dr.pubkey.distinctBase),
+      withdrawal_credentials: dr.withdrawalCredentials.asEth2Digest,
+      amount: dr.amount.Gwei,
+      signature: ValidatorSig(blob: dr.signature.distinctBase),
+      index: dr.index.uint64)
+
+  template getExecutionLayerExit(ele: ExitV1): ExecutionLayerExit =
+    ExecutionLayerExit(
+      source_address: ExecutionAddress(data: ele.sourceAddress.distinctBase),
+      validator_pubkey: ValidatorPubKey(
+        blob: ele.validatorPublicKey.distinctBase))
+
+  electra.ExecutionPayload(
+    parent_hash: rpcExecutionPayload.parentHash.asEth2Digest,
+    feeRecipient:
+      ExecutionAddress(data: rpcExecutionPayload.feeRecipient.distinctBase),
+    state_root: rpcExecutionPayload.stateRoot.asEth2Digest,
+    receipts_root: rpcExecutionPayload.receiptsRoot.asEth2Digest,
+    logs_bloom: BloomLogs(data: rpcExecutionPayload.logsBloom.distinctBase),
+    prev_randao: rpcExecutionPayload.prevRandao.asEth2Digest,
+    block_number: rpcExecutionPayload.blockNumber.uint64,
+    gas_limit: rpcExecutionPayload.gasLimit.uint64,
+    gas_used: rpcExecutionPayload.gasUsed.uint64,
+    timestamp: rpcExecutionPayload.timestamp.uint64,
+    extra_data: List[byte, MAX_EXTRA_DATA_BYTES].init(
+      rpcExecutionPayload.extraData.bytes),
+    base_fee_per_gas: rpcExecutionPayload.baseFeePerGas,
+    block_hash: rpcExecutionPayload.blockHash.asEth2Digest,
+    transactions: List[bellatrix.Transaction, MAX_TRANSACTIONS_PER_PAYLOAD].init(
+      mapIt(rpcExecutionPayload.transactions, it.getTransaction)),
+    withdrawals: List[capella.Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD].init(
+      mapIt(rpcExecutionPayload.withdrawals, it.asConsensusWithdrawal)),
+    blob_gas_used: rpcExecutionPayload.blobGasUsed.uint64,
+    excess_blob_gas: rpcExecutionPayload.excessBlobGas.uint64,
+    deposit_receipts:
+      List[electra.DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD].init(
+        mapIt(rpcExecutionPayload.depositReceipts, it.getDepositReceipt)),
+    exits:
+      List[electra.ExecutionLayerExit, MAX_EXECUTION_LAYER_EXITS_PER_PAYLOAD].init(
+        mapIt(rpcExecutionPayload.exits, it.getExecutionLayerExit)))
+
 func asEngineExecutionPayload*(executionPayload: bellatrix.ExecutionPayload):
     ExecutionPayloadV1 =
   template getTypedTransaction(tt: bellatrix.Transaction): TypedTransaction =
@@ -558,6 +606,47 @@ func asEngineExecutionPayload*(executionPayload: deneb.ExecutionPayload):
     blobGasUsed: Quantity(executionPayload.blob_gas_used),
     excessBlobGas: Quantity(executionPayload.excess_blob_gas))
 
+func asEngineExecutionPayload*(executionPayload: electra.ExecutionPayload):
+    ExecutionPayloadV4 =
+  template getTypedTransaction(tt: bellatrix.Transaction): TypedTransaction =
+    TypedTransaction(tt.distinctBase)
+
+  template getDepositReceipt(dr: DepositReceipt): DepositReceiptV1 =
+    DepositReceiptV1(
+      pubkey: FixedBytes[RawPubKeySize](dr.pubkey.blob),
+      withdrawalCredentials: FixedBytes[32](dr.withdrawal_credentials.data),
+      amount: dr.amount.Quantity,
+      signature: FixedBytes[RawSigSize](dr.signature.blob),
+      index: dr.index.Quantity)
+
+  template getExecutionLayerExit(ele: ExecutionLayerExit): ExitV1 =
+    ExitV1(
+      sourceAddress: Address(ele.source_address.data),
+      validatorPublicKey: FixedBytes[RawPubKeySize](ele.validator_pubkey.blob))
+
+  engine_api.ExecutionPayloadV4(
+    parentHash: executionPayload.parent_hash.asBlockHash,
+    feeRecipient: Address(executionPayload.fee_recipient.data),
+    stateRoot: executionPayload.state_root.asBlockHash,
+    receiptsRoot: executionPayload.receipts_root.asBlockHash,
+    logsBloom:
+      FixedBytes[BYTES_PER_LOGS_BLOOM](executionPayload.logs_bloom.data),
+    prevRandao: executionPayload.prev_randao.asBlockHash,
+    blockNumber: Quantity(executionPayload.block_number),
+    gasLimit: Quantity(executionPayload.gas_limit),
+    gasUsed: Quantity(executionPayload.gas_used),
+    timestamp: Quantity(executionPayload.timestamp),
+    extraData: DynamicBytes[0, MAX_EXTRA_DATA_BYTES](executionPayload.extra_data),
+    baseFeePerGas: executionPayload.base_fee_per_gas,
+    blockHash: executionPayload.block_hash.asBlockHash,
+    transactions: mapIt(executionPayload.transactions, it.getTypedTransaction),
+    withdrawals: mapIt(executionPayload.withdrawals, it.asEngineWithdrawal),
+    blobGasUsed: Quantity(executionPayload.blob_gas_used),
+    excessBlobGas: Quantity(executionPayload.excess_blob_gas),
+    depositReceipts: mapIt(
+      executionPayload.deposit_receipts, it.getDepositReceipt),
+    exits: mapIt(executionPayload.exits, it.getExecutionLayerExit))
+
 func isConnected(connection: ELConnection): bool =
   connection.web3.isSome
 

beacon_chain/rpc/rest_beacon_api.nim

@@ -129,6 +129,21 @@ proc toString*(kind: ValidatorFilterKind): string =
   of ValidatorFilterKind.WithdrawalDone:
     "withdrawal_done"
 
+func checkRestBlockBlobsValid(
+    forkyBlck: deneb.SignedBeaconBlock | electra.SignedBeaconBlock,
+    kzg_proofs: KzgProofs,
+    blobs: Blobs): Result[void, string] =
+  if kzg_proofs.len != blobs.len:
+    return err("Invalid block publish: " & $kzg_proofs.len & " KZG proofs and " &
+      $blobs.len & " blobs")
+
+  if kzg_proofs.len != forkyBlck.message.body.blob_kzg_commitments.len:
+    return err("Invalid block publish: " & $kzg_proofs.len &
+      " KZG proofs and " & $forkyBlck.message.body.blob_kzg_commitments.len &
+      " KZG commitments")
+
+  ok()
+
 proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
   # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-4881.md
   router.api2(MethodGet, "/eth/v1/beacon/deposit_snapshot") do (
@@ -920,6 +935,12 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
         of ConsensusFork.Deneb:
           var blck = restBlock.denebData.signed_block
           blck.root = hash_tree_root(blck.message)
+
+          let validity = checkRestBlockBlobsValid(
+            blck, restBlock.denebData.kzg_proofs, restBlock.denebData.blobs)
+          if validity.isErr:
+            return RestApiResponse.jsonError(Http400, validity.error)
+
           await node.router.routeSignedBeaconBlock(
             blck, Opt.some(blck.create_blob_sidecars(
               restBlock.denebData.kzg_proofs, restBlock.denebData.blobs)))
@@ -996,6 +1017,12 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
         of ConsensusFork.Deneb:
           var blck = restBlock.denebData.signed_block
           blck.root = hash_tree_root(blck.message)
+
+          let validity = checkRestBlockBlobsValid(
+            blck, restBlock.denebData.kzg_proofs, restBlock.denebData.blobs)
+          if validity.isErr:
+            return RestApiResponse.jsonError(Http400, validity.error)
+
           await node.router.routeSignedBeaconBlock(
             blck, Opt.some(blck.create_blob_sidecars(
               restBlock.denebData.kzg_proofs, restBlock.denebData.blobs)))

beacon_chain/spec/datatypes/electra.nim

@@ -30,8 +30,27 @@ from ./deneb import Blobs, BlobsBundle, KzgCommitments, KzgProofs
 
 export json_serialization, base, kzg4844
 
+const
+  # Keep these here for now, since things still in flux
+  # https://github.com/ethereum/consensus-specs/pull/3615
+  MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD* = 8192
+  MAX_EXECUTION_LAYER_EXITS_PER_PAYLOAD* = 16  # there's a discrepancy here, _PER_PAYLOAD or not
+
 type
-  # https://github.com/ethereum/consensus-specs/blob/v1.4.0/specs/deneb/beacon-chain.md#executionpayload
+  # https://github.com/ethereum/consensus-specs/pull/3615
+  DepositReceipt* = object
+    pubkey*: ValidatorPubKey
+    withdrawal_credentials*: Eth2Digest
+    amount*: Gwei
+    signature*: ValidatorSig
+    index*: uint64
+
+  # https://github.com/ethereum/consensus-specs/pull/3615
+  ExecutionLayerExit* = object
+    source_address*: ExecutionAddress
+    validator_pubkey*: ValidatorPubKey
+
+  # https://github.com/ethereum/consensus-specs/pull/3615
   ExecutionPayload* = object
     # Execution block header fields
     parent_hash*: Eth2Digest
@@ -54,8 +73,10 @@ type
     block_hash*: Eth2Digest # Hash of execution block
     transactions*: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD]
     withdrawals*: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD]
-    blob_gas_used*: uint64   # [New in Deneb]
+    blob_gas_used*: uint64
-    excess_blob_gas*: uint64 # [New in Deneb]
+    excess_blob_gas*: uint64
+    deposit_receipts*: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD]
+    exits*: List[ExecutionLayerExit, MAX_EXECUTION_LAYER_EXITS_PER_PAYLOAD]
 
   ExecutionPayloadForSigning* = object
     executionPayload*: ExecutionPayload

beacon_chain/version.nim

@@ -18,8 +18,8 @@ const
     "Copyright (c) 2019-" & compileYear & " Status Research & Development GmbH"
 
   versionMajor* = 24
-  versionMinor* = 2
+  versionMinor* = 3
-  versionBuild* = 2
+  versionBuild* = 0
 
   versionBlob* = "stateofus" # Single word - ends up in the default graffiti
 

ci/Jenkinsfile

@@ -1,5 +1,5 @@
 /* beacon_chain
- * Copyright (c) 2019-2023 Status Research & Development GmbH
+ * Copyright (c) 2019-2024 Status Research & Development GmbH
  * Licensed and distributed under either of
  *   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
  *   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -68,22 +68,15 @@ pipeline {
     stage('Build') {
       steps { timeout(50) {
         sh 'make LOG_LEVEL=TRACE'
-        /* Check documentation reflects `nimbus_beacon_node --help`. */
-        sh '''#!/usr/bin/env bash
-          if ! diff -u \\
-              <(sed -n '/Usage/,/^...$/ { /^...$/d; p; }' \\
-                docs/the_nimbus_book/src/options.md) \\
-              <(COLUMNS=200 build/nimbus_beacon_node --help | \\
-                sed -n '/Usage/,/Available sub-commands/ { /Available sub-commands/d; p; }' | \\
-                sed 's/\\x1B\\[[0-9;]*[mG]//g' | \\
-                sed 's/[[:space:]]*$//'); then \\
-            echo "Please update 'docs/the_nimbus_book/src/options.md' to match 'COLUMNS=200 nimbus_beacon_node --help'"; \\
-            false; \\
-          fi
-        '''
       } }
     }
 
+    stage('Check Docs') {
+      steps {
+        sh './scripts/check_docs_help_msg.sh'
+      }
+    }
+
     stage('Tests') {
       parallel {
         stage('General') {

scripts/check_docs_help_msg.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023-2024 Status Research & Development GmbH.
+# Licensed under either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -euo pipefail
+DOC_FILE='docs/the_nimbus_book/src/options.md'
+DOC_USAGE=$(sed -n '/Usage/,/^...$/ { /^...$/d; p; }' "${DOC_FILE}")
+BIN_USAGE=$(
+  COLUMNS=200 build/nimbus_beacon_node --help | \
+    sed 's/\x1b\[[0-9;]*m//g' | \
+    sed -n '/Usage/,/Available sub-commands/ { /Available sub-commands/d; p; }' | \
+    sed 's/\\x1B\\[[0-9;]*[mG]//g' | \
+    sed 's/[[:space:]]*$//'
+)
+if ! diff -u <(echo "${DOC_USAGE}") <(echo "${BIN_USAGE}"); then
+  echo "Please update '${DOC_FILE}' to match 'COLUMNS=200 nimbus_beacon_node --help'"
+  exit 1
+fi

vendor/nim-libbacktrace

@@ -1 +1 @@
-Subproject commit 9f4fd37153a84a7039b7e6a355a1c3374b0ba4ce
+Subproject commit 027570111c161d8378bca9e84b5f75500a8c38a3

vendor/nim-ssz-serialization

@@ -1 +1 @@
-Subproject commit 0fc5e49093fa8d3c07476738e3257d0d8e7999a3
+Subproject commit 248f2bdca2d65ff920920c72b764d0622d522596