Merge stable into unstable

CHANGELOG.md

@@ -1,4 +1,9 @@
-2023-09-12 v23.9.0
+2023-09-25 v23.9.1
+==================
+
+Nimbus `v23.9.1` is a `low-urgency` point release that corrects the [Holešky testnet](https://github.com/eth-clients/holesky) metadata after the [failed start](https://twitter.com/parithosh_j/status/1702816780542984504) on 15th of September. If you want to participate in the network, please update your client before the genesis event on 28th of September, 12:00 UTC.
+
+2023-09-08 v23.9.0
 ==================
 
 Nimbus `v23.9.0` is a `low-urgency` upgrade providing full support for the upcoming [Holešky testnet](https://github.com/eth-clients/holesky) and simplifying the required configuration for using [remote signers](https://nimbus.guide/web3signer.html).
@@ -10,7 +15,7 @@ We've been hard at work researching and developing a GossipSub protocol upgrade,
 * The GossipSub implementation of Nimbus now consumes less bandwidth and CPU cycles, while improving upon the existing DoS protections through better peer scoring:
   https://github.com/status-im/nimbus-eth2/pull/5229
 
-* The new `--web3-signer` command-line option can be used to connect Nimbus to one or more remote signers without requiring any remote keystore files to be created. The list of validators attached to each remote signer is obtained automatically through the [`/api/v1/eth2/publicKeys`](https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Public-Key/operation/ETH2_LIST) Web3Signer API endpoint:
+* The new `--web3-signer-url` command-line option can be used to connect Nimbus to one or more remote signers without requiring any remote keystore files to be created. The list of validators attached to each remote signer is obtained automatically through the [`/api/v1/eth2/publicKeys`](https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Public-Key/operation/ETH2_LIST) Web3Signer API endpoint:
   https://github.com/status-im/nimbus-eth2/pull/5366
   https://github.com/status-im/nimbus-eth2/pull/5385
   https://github.com/status-im/nimbus-eth2/pull/5389

beacon_chain/networking/network_metadata.nim

@@ -321,8 +321,8 @@ elif const_preset == "mainnet":
       vendorDir & "/holesky/custom_config_data",
       some holesky,
       downloadGenesisFrom = some DownloadInfo(
-        url: "https://github.com/status-im/nimbus-eth2/releases/download/v23.9.0/holesky-genesis.ssz.sz",
-        digest: Eth2Digest.fromHex "0x76631cd0b9ddc5b2c766b496e23f16759ce1181446a4efb40e5540cd15b78a07"))
+        url: "https://github.com/status-im/nimbus-eth2/releases/download/v23.9.1/holesky-genesis.ssz.sz",
+        digest: Eth2Digest.fromHex "0x0ea3f6f9515823b59c863454675fefcd1d8b4f2dbe454db166206a41fda060a0"))
 
     sepoliaMetadata = loadCompileTimeNetworkMetadata(
       vendorDir & "/sepolia/bepolia",

beacon_chain/networking/network_metadata_downloads.nim

@@ -8,7 +8,7 @@
 import
   std/uri,
   stew/io2, chronos, chronos/apps/http/httpclient, snappy,
-  ../spec/digest
+  ../spec/[digest, forks], ../spec/datatypes/base
 
 import network_metadata
 export network_metadata
@@ -29,15 +29,16 @@ proc downloadFile(url: Uri): Future[seq[byte]] {.async.} =
       msg: "Unexpected status code " & $response[0] & " when fetching " & $url,
       status: response[0])
 
-proc fetchBytes*(metadata: GenesisMetadata,
-                 genesisStateUrlOverride = none(Uri)): Future[seq[byte]] {.async.} =
-  case metadata.kind
+proc fetchGenesisBytes*(
+    metadata: Eth2NetworkMetadata,
+    genesisStateUrlOverride = none(Uri)): Future[seq[byte]] {.async.} =
+  case metadata.genesis.kind
   of NoGenesis:
-    raiseAssert "fetchBytes should be called only when metadata.hasGenesis is true"
+    raiseAssert "fetchGenesisBytes should be called only when metadata.hasGenesis is true"
   of BakedIn:
-    result = @(metadata.bakedBytes)
+    result = @(metadata.genesis.bakedBytes)
   of BakedInUrl:
-    result = await downloadFile(genesisStateUrlOverride.get(parseUri metadata.url))
+    result = await downloadFile(genesisStateUrlOverride.get(parseUri metadata.genesis.url))
     # Under the built-in default URL, we serve a snappy-encoded BeaconState in order
     # to reduce the size of the downloaded file with roughly 50% (this precise ratio
     # depends on the number of validator recors). The user is still free to provide
@@ -54,11 +55,13 @@ proc fetchBytes*(metadata: GenesisMetadata,
     #       HTTP servers.
     if result.isSnappyFramedStream:
       result = decodeFramed(result)
-    if eth2digest(result) != metadata.digest:
-      raise (ref DigestMismatchError)(
-        msg: "The downloaded genesis state cannot be verified (checksum mismatch)")
+    let state = newClone(readSszForkedHashedBeaconState(metadata.cfg, result))
+    withState(state[]):
+      if forkyState.root != metadata.genesis.digest:
+        raise (ref DigestMismatchError)(
+          msg: "The downloaded genesis state cannot be verified (checksum mismatch)")
   of UserSuppliedFile:
-    result = readAllBytes(metadata.path).tryGet()
+    result = readAllBytes(metadata.genesis.path).tryGet()
 
 proc sourceDesc*(metadata: GenesisMetadata): string =
   case metadata.kind
@@ -75,5 +78,5 @@ when isMainModule:
   let holeskyMetadata = getMetadataForNetwork("holesky")
   io2.writeFile(
     "holesky-genesis.ssz",
-    waitFor holeskyMetadata.genesis.fetchBytes()
+    waitFor holeskyMetadata.fetchGenesisBytes()
   ).expect("success")

beacon_chain/nimbus_beacon_node.nim

@@ -564,7 +564,7 @@ proc init*(T: type BeaconNode,
             if metadata.genesis.kind == BakedInUrl:
               info "Obtaining genesis state",
                     sourceUrl = $config.genesisStateUrl.get(parseUri metadata.genesis.url)
-            await metadata.genesis.fetchBytes(config.genesisStateUrl)
+            await metadata.fetchGenesisBytes(config.genesisStateUrl)
           except CatchableError as err:
             error "Failed to obtain genesis state",
                   source = metadata.genesis.sourceDesc,
@@ -2083,7 +2083,7 @@ proc handleStartUpCmd(config: var BeaconNodeConf) {.raises: [CatchableError].} =
             stateId: "finalized")
       genesis =
         if network.hasGenesis:
-          let genesisBytes = try: waitFor network.genesis.fetchBytes()
+          let genesisBytes = try: waitFor network.fetchGenesisBytes()
           except CatchableError as err:
             error "Failed to obtain genesis state",
                   source = network.genesis.sourceDesc,

beacon_chain/nimbus_light_client.nim

@@ -63,7 +63,7 @@ programMain:
   template cfg(): auto = metadata.cfg
 
   let
-    genesisBytes = try: waitFor metadata.genesis.fetchBytes()
+    genesisBytes = try: waitFor metadata.fetchGenesisBytes()
                    except CatchableError as err:
                      error "Failed to obtain genesis state",
                             source = metadata.genesis.sourceDesc,

beacon_chain/version.nim

@@ -18,7 +18,7 @@ const
 
   versionMajor* = 23
   versionMinor* = 9
-  versionBuild* = 0
+  versionBuild* = 1
 
   versionBlob* = "stateofus" # Single word - ends up in the default graffiti
 

vendor/holesky

@@ -1 +1 @@
-Subproject commit f1ad227a2511ea26f5d043fad15d9431fd681941
+Subproject commit ea39b9006210848e13f28d92e12a30548cecd41d