embed genesis states using incbin (#4905)

AllTests-mainnet.md

@@ -356,6 +356,13 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + Voluntary exit signatures                                                                  OK
 ```
 OK: 8/8 Fail: 0/8 Skip: 0/8
+## Network metadata
+```diff
++ goerli                                                                                     OK
++ mainnet                                                                                    OK
++ sepolia                                                                                    OK
+```
+OK: 3/3 Fail: 0/3 Skip: 0/3
 ## Nimbus remote signer/signing test (verifying-web3signer)
 ```diff
 + Signing BeaconBlock (getBlockSignature(altair))                                            OK
@@ -669,4 +676,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 382/387 Fail: 0/387 Skip: 5/387
+OK: 385/390 Fail: 0/390 Skip: 5/390

beacon_chain/conf.nim

@@ -1296,9 +1296,9 @@ proc loadEth2Network*(
     getMetadataForNetwork(eth2Network.get)
   else:
     when const_preset == "gnosis":
-      gnosisMetadata
+      getMetadataForNetwork("gnosis")
     elif const_preset == "mainnet":
-      mainnetMetadata
+      getMetadataForNetwork("mainnet")
     else:
       # Presumably other configurations can have other defaults, but for now
       # this simplifies the flow

beacon_chain/networking/network_metadata.nim

@@ -13,14 +13,8 @@ import
   web3/[ethtypes, conversions],
   chronicles,
   eth/common/eth_types_json_serialization,
-  ../spec/eth2_ssz_serialization,
+  ../spec/eth2_ssz_serialization
-  ../spec/datatypes/phase0
 
-# ATTENTION! This file will produce a large C file, because we are inlining
-# genesis states as C literals in the generated code (and blobs in the final
-# binary). It makes sense to keep the file small and separated from the rest
-# of the module in order go gain maximum efficiency in incremental compilation.
-#
 # TODO(zah):
 # We can compress the embedded states with snappy before embedding them here.
 
@@ -68,28 +62,14 @@ type
       depositContractBlock*: uint64
       depositContractBlockHash*: Eth2Digest
 
-      # Please note that we are using `string` here because SSZ.decode
-      # is not currently usable at compile time and we want to load the
-      # network metadata into a constant.
-      #
-      # We could have also used `seq[byte]`, but this results in a lot
-      # more generated code that slows down compilation. The impact on
-      # compilation times of embedding the genesis as a string is roughly
-      # 0.1s on my machine (you can test this by choosing an invalid name
-      # for the genesis file below).
-      #
       # `genesisData` will have `len == 0` for networks with a still
       # unknown genesis state.
-      genesisData*: string
+      genesisData*: seq[byte]
-      genesisDepositsSnapshot*: string
+      genesisDepositsSnapshot*: seq[byte]
     else:
       incompatibilityDesc*: string
 
-const
+proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError].} =
-  eth2NetworksDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/eth2-networks"
-  sepoliaDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/sepolia"
-
-proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect].} =
   # Read a list of ENR values from a YAML file containing a flat list of entries
   if fileExists(path):
     splitLines(readFile(path)).
@@ -98,7 +78,7 @@ proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect].
   else:
     @[]
 
-proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} =
+proc readBootEnr*(path: string): seq[string] {.raises: [IOError].} =
   # Read a list of ENR values from a YAML file containing a flat list of entries
   if fileExists(path):
     splitLines(readFile(path)).
@@ -107,8 +87,9 @@ proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} =
   else:
     @[]
 
-proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Eth2NetworkMetadata
+proc loadEth2NetworkMetadata*(
-                             {.raises: [CatchableError, Defect].} =
+    path: string, eth1Network = none(Eth1Network), loadGenesis = true):
+    Eth2NetworkMetadata {.raises: [CatchableError].} =
   # Load data in eth2-networks format
   # https://github.com/eth-clients/eth2-networks
 
@@ -175,15 +156,15 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
         readBootstrapNodes(bootstrapNodesPath) &
         readBootEnr(bootEnrPath))
 
-      genesisData = if fileExists(genesisPath):
+      genesisData = if loadGenesis and fileExists(genesisPath):
-        readFile(genesisPath)
+        toBytes(readFile(genesisPath))
       else:
-        ""
+        @[]
 
       genesisDepositsSnapshot = if fileExists(genesisDepositsSnapshotPath):
-        readFile(genesisDepositsSnapshotPath)
+        toBytes(readFile(genesisDepositsSnapshotPath))
       else:
-        ""
+        @[]
 
     Eth2NetworkMetadata(
       incompatible: false,
@@ -201,10 +182,11 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
 
 proc loadCompileTimeNetworkMetadata(
     path: string,
-    eth1Network = none(Eth1Network)): Eth2NetworkMetadata {.raises: [Defect].} =
+    eth1Network = none(Eth1Network),
+    loadGenesis = true): Eth2NetworkMetadata {.raises: [].} =
   if fileExists(path & "/config.yaml"):
     try:
-      result = loadEth2NetworkMetadata(path, eth1Network)
+      result = loadEth2NetworkMetadata(path, eth1Network, loadGenesis)
       if result.incompatible:
         macros.error "The current build is misconfigured. " &
                      "Attempt to load an incompatible network metadata: " &
@@ -214,26 +196,55 @@ proc loadCompileTimeNetworkMetadata(
   else:
     macros.error "config.yaml not found for network '" & path
 
-template eth2Network(path: string, eth1Network: Eth1Network): Eth2NetworkMetadata =
+const vendorDir =
-  loadCompileTimeNetworkMetadata(eth2NetworksDir & "/" & path,
+  currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor"
-                                 some eth1Network)
 
 when const_preset == "gnosis":
+  import stew/assign2
+
+  let
+    gnosisGenesis {.importc: "gnosis_mainnet_genesis".}: ptr UncheckedArray[byte]
+    gnosisGenesisSize {.importc: "gnosis_mainnet_genesis_size".}: int
+
+  {.compile: "network_metadata_gnosis.S".}
+
   const
-    gnosisMetadata* = loadCompileTimeNetworkMetadata(
+    gnosisMetadata = loadCompileTimeNetworkMetadata(
-      currentSourcePath.parentDir.replace('\\', '/') &
+      vendorDir & "/gnosis-chain-configs/mainnet")
-      "/../../vendor/gnosis-chain-configs/mainnet")
+
   static:
     checkForkConsistency(gnosisMetadata.cfg)
     doAssert gnosisMetadata.cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH
     doAssert gnosisMetadata.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH
 
 elif const_preset == "mainnet":
+  import stew/assign2
+
+  # Nim is very inefficent at loading large constants from binary files so we
+  # use this trick instead which saves significant amounts of compile time
+  let
+    mainnetGenesis {.importc: "eth2_mainnet_genesis".}: ptr UncheckedArray[byte]
+    mainnetGenesisSize {.importc: "eth2_mainnet_genesis_size".}: int
+
+    praterGenesis {.importc: "eth2_goerli_genesis".}: ptr UncheckedArray[byte]
+    praterGenesisSize {.importc: "eth2_goerli_genesis_size".}: int
+
+    sepoliaGenesis {.importc: "eth2_sepolia_genesis".}: ptr UncheckedArray[byte]
+    sepoliaGenesisSize {.importc: "eth2_sepolia_genesis_size".}: int
+
+  {.compile: "network_metadata_mainnet.S".}
+
   const
-    mainnetMetadata* = eth2Network("shared/mainnet", mainnet)
+    eth2NetworksDir = vendorDir & "/eth2-networks"
-    praterMetadata* = eth2Network("shared/prater", goerli)
+    sepoliaDir = vendorDir & "/sepolia"
-    sepoliaMetadata* =
+
-      loadCompileTimeNetworkMetadata(sepoliaDir & "/bepolia", some sepolia)
+    mainnetMetadata = loadCompileTimeNetworkMetadata(
+      vendorDir & "/eth2-networks/shared/mainnet", some mainnet, false)
+    praterMetadata = loadCompileTimeNetworkMetadata(
+      vendorDir & "/eth2-networks/shared/prater", some goerli, false)
+    sepoliaMetadata = loadCompileTimeNetworkMetadata(
+      vendorDir & "/sepolia/bepolia", some sepolia, false)
+
   static:
     for network in [mainnetMetadata, praterMetadata, sepoliaMetadata]:
       checkForkConsistency(network.cfg)
@@ -245,7 +256,7 @@ elif const_preset == "mainnet":
       doAssert network.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH
 
 proc getMetadataForNetwork*(
-    networkName: string): Eth2NetworkMetadata {.raises: [Defect, IOError].} =
+    networkName: string): Eth2NetworkMetadata {.raises: [IOError].} =
   template loadRuntimeMetadata(): auto =
     if fileExists(networkName / "config.yaml"):
       try:
@@ -260,26 +271,31 @@ proc getMetadataForNetwork*(
   if networkName == "ropsten":
     warn "Ropsten is unsupported; https://blog.ethereum.org/2022/11/30/ropsten-shutdown-announcement suggests migrating to Goerli or Sepolia"
 
+  template withGenesis(metadata, genesis: untyped): untyped =
+    var tmp = metadata
+    assign(tmp.genesisData, genesis.toOpenArray(0, `genesis Size` - 1))
+    tmp
+
   let metadata =
     when const_preset == "gnosis":
       case toLowerAscii(networkName)
       of "gnosis":
-        gnosisMetadata
+        withGenesis(gnosisMetadata, gnosisGenesis)
       of "gnosis-chain":
         warn "`--network:gnosis-chain` is deprecated, " &
           "use `--network:gnosis` instead"
-        gnosisMetadata
+        withGenesis(gnosisMetadata, gnosisGenesis)
       else:
         loadRuntimeMetadata()
 
     elif const_preset == "mainnet":
       case toLowerAscii(networkName)
       of "mainnet":
-        mainnetMetadata
+        withGenesis(mainnetMetadata, mainnetGenesis)
       of "prater", "goerli":
-        praterMetadata
+        withGenesis(praterMetadata, praterGenesis)
       of "sepolia":
-        sepoliaMetadata
+        withGenesis(sepoliaMetadata, sepoliaGenesis)
       else:
         loadRuntimeMetadata()
 
@@ -294,7 +310,7 @@ proc getMetadataForNetwork*(
   metadata
 
 proc getRuntimeConfig*(
-    eth2Network: Option[string]): RuntimeConfig {.raises: [Defect, IOError].} =
+    eth2Network: Option[string]): RuntimeConfig {.raises: [IOError].} =
   ## Returns the run-time config for a network specified on the command line
   ## If the network is not explicitly specified, the function will act as the
   ## regular Nimbus binary, returning the mainnet config.

beacon_chain/networking/network_metadata_gnosis.S

@@ -0,0 +1,43 @@
+# beacon_chain
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+#if defined(__APPLE__)
+# define cdecl(s) _##s
+#else
+# define cdecl(s) s
+#endif
+
+#if defined(__linux__)
+    .section .rodata,"a",@progbits
+#elif defined(__APPLE__)
+    .section __TEXT,__const
+#elif defined(__WIN32__)
+    .section .rdata,"dr"
+#else
+    .text
+#endif
+
+# name_data = start of data
+# name_end = end of data (without alignment)
+# name = 64-bit pointer to data
+# name_size = 64-bit length in bytes
+
+gnosis_mainnet_genesis_data:
+    .incbin "../../vendor/gnosis-chain-configs/mainnet/genesis.ssz"
+gnosis_mainnet_genesis_end:
+    .global cdecl(gnosis_mainnet_genesis_size)
+cdecl(gnosis_mainnet_genesis_size):
+    .quad    gnosis_mainnet_genesis_end - gnosis_mainnet_genesis_data
+
+#if defined(__APPLE__)
+    .section __DATA,__const
+#endif
+
+    .global cdecl(gnosis_mainnet_genesis)
+    .p2align 3
+cdecl(gnosis_mainnet_genesis):
+    .quad gnosis_mainnet_genesis_data

beacon_chain/networking/network_metadata_mainnet.S

@@ -0,0 +1,67 @@
+# beacon_chain
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+#if defined(__APPLE__)
+# define cdecl(s) _##s
+#else
+# define cdecl(s) s
+#endif
+
+#if defined(__linux__)
+    .section .rodata,"a",@progbits
+#elif defined(__APPLE__)
+    .section __TEXT,__const
+#elif defined(__WIN32__)
+    .section .rdata,"dr"
+#else
+    .text
+#endif
+
+# name_data = start of data
+# name_end = end of data (without alignment)
+# name = 64-bit pointer to data
+# name_size = 64-bit length in bytes
+
+eth2_mainnet_genesis_data:
+    .incbin "../../vendor/eth2-networks/shared/mainnet/genesis.ssz"
+eth2_mainnet_genesis_end:
+    .global cdecl(eth2_mainnet_genesis_size)
+cdecl(eth2_mainnet_genesis_size):
+    .quad    eth2_mainnet_genesis_end - eth2_mainnet_genesis_data
+
+eth2_goerli_genesis_data:
+    .incbin "../../vendor/eth2-networks/shared/prater/genesis.ssz"
+eth2_goerli_genesis_end:
+    .global cdecl(eth2_goerli_genesis_size)
+cdecl(eth2_goerli_genesis_size):
+    .quad    eth2_goerli_genesis_end - eth2_goerli_genesis_data
+
+eth2_sepolia_genesis_data:
+    .incbin "../../vendor/sepolia/bepolia/genesis.ssz"
+eth2_sepolia_genesis_end:
+    .global cdecl(eth2_sepolia_genesis_size)
+cdecl(eth2_sepolia_genesis_size):
+    .quad    eth2_sepolia_genesis_end - eth2_sepolia_genesis_data
+
+#if defined(__APPLE__)
+    .section __DATA,__const
+#endif
+
+    .global cdecl(eth2_mainnet_genesis)
+    .p2align 3
+cdecl(eth2_mainnet_genesis):
+    .quad eth2_mainnet_genesis_data
+
+    .global cdecl(eth2_goerli_genesis)
+    .p2align 3
+cdecl(eth2_goerli_genesis):
+    .quad eth2_goerli_genesis_data
+
+    .global cdecl(eth2_sepolia_genesis)
+    .p2align 3
+cdecl(eth2_sepolia_genesis):
+    .quad eth2_sepolia_genesis_data

beacon_chain/nimbus_beacon_node.nim

@@ -542,9 +542,7 @@ proc init*(T: type BeaconNode,
   var genesisState =
     if metadata.genesisData.len > 0:
       try:
-        newClone readSszForkedHashedBeaconState(
+        newClone readSszForkedHashedBeaconState(cfg, metadata.genesisData)
-          cfg,
-          metadata.genesisData.toOpenArrayByte(0, metadata.genesisData.high))
       except CatchableError as err:
         raiseAssert "Invalid baked-in state: " & err.msg
     else:
@@ -2057,9 +2055,7 @@ proc handleStartUpCmd(config: var BeaconNodeConf) {.raises: [Defect, CatchableEr
             stateId: "finalized")
       genesis =
         if network.genesisData.len > 0:
-          newClone(readSszForkedHashedBeaconState(
+          newClone(readSszForkedHashedBeaconState(cfg, network.genesisData))
-            cfg,
-            network.genesisData.toOpenArrayByte(0, network.genesisData.high())))
         else: nil
 
     if config.blockId.isSome():

beacon_chain/nimbus_light_client.nim

@@ -65,9 +65,7 @@ programMain:
   let
     genesisState =
       try:
-        template genesisData(): auto = metadata.genesisData
+        newClone(readSszForkedHashedBeaconState(cfg, metadata.genesisData))
-        newClone(readSszForkedHashedBeaconState(
-          cfg, genesisData.toOpenArrayByte(genesisData.low, genesisData.high)))
       except CatchableError as err:
         raiseAssert "Invalid baked-in state: " & err.msg
 

nfuzz/libnfuzz.nim

@@ -1,8 +1,15 @@
+# beacon_chain
+# Copyright (c) 2019-2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
 # Required for deserialisation of ValidatorSig in Attestation due to
 # https://github.com/nim-lang/Nim/issues/11225
 
 import
-  stew/ptrops, stew/ranges/ptr_arith, chronicles,
+  stew/ptrops, chronicles,
   ../beacon_chain/networking/network_metadata,
   ../beacon_chain/spec/datatypes/phase0,
   ../beacon_chain/spec/[
@@ -103,7 +110,7 @@ proc nfuzz_attestation(input: openArray[byte], xoutput: ptr byte,
 proc nfuzz_attester_slashing(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
   decodeAndProcess(AttesterSlashingInput):
-    process_attester_slashing(mainnetMetadata.cfg, data.state, data.attesterSlashing, flags, cache).isOk
+    process_attester_slashing(getRuntimeConfig(some "mainnet"), data.state, data.attesterSlashing, flags, cache).isOk
 
 proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
@@ -127,7 +134,7 @@ proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
 
   decodeAndProcess(BlockInput):
     state_transition(
-      mainnetMetadata.cfg, data, data.beaconBlock, flags, noRollback).isOk
+      getRuntimeConfig(some "mainnet"), data, data.beaconBlock, flags, noRollback).isOk
 
 proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
@@ -137,17 +144,17 @@ proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
 proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
   decodeAndProcess(DepositInput):
-    process_deposit(mainnetMetadata.cfg, data.state, data.deposit, flags).isOk
+    process_deposit(getRuntimeConfig(some "mainnet"), data.state, data.deposit, flags).isOk
 
 proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
   decodeAndProcess(ProposerSlashingInput):
-    process_proposer_slashing(mainnetMetadata.cfg, data.state, data.proposerSlashing, flags, cache).isOk
+    process_proposer_slashing(getRuntimeConfig(some "mainnet"), data.state, data.proposerSlashing, flags, cache).isOk
 
 proc nfuzz_voluntary_exit(input: openArray[byte], xoutput: ptr byte,
     xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
   decodeAndProcess(VoluntaryExitInput):
-    process_voluntary_exit(mainnetMetadata.cfg, data.state, data.exit, flags, cache).isOk
+    process_voluntary_exit(getRuntimeConfig(some "mainnet"), data.state, data.exit, flags, cache).isOk
 
 # Note: Could also accept raw input pointer and access list_size + seed here.
 # However, list_size needs to be known also outside this proc to allocate xoutput.

tests/all_tests.nim

@@ -36,6 +36,7 @@ import # Unit test
   ./test_light_client_processor,
   ./test_light_client,
   ./test_message_signatures,
+  ./test_network_metadata,
   ./test_peer_pool,
   ./test_remote_keystore,
   ./test_serialization,

tests/test_network_metadata.nim

@@ -0,0 +1,39 @@
+# beacon_chain
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  unittest2,
+  ../beacon_chain/networking/network_metadata,
+  ../beacon_chain/spec/forks
+
+{.used.}
+
+template checkRoot(name, root) =
+  let
+    metadata = getMetadataForNetwork(name)
+    cfg = metadata.cfg
+    state = newClone(readSszForkedHashedBeaconState(
+      metadata.cfg, metadata.genesisData))
+
+  check:
+    $getStateRoot(state[]) == root
+
+suite "Network metadata":
+  test "mainnet":
+    checkRoot(
+      "mainnet",
+      "7e76880eb67bbdc86250aa578958e9d0675e64e714337855204fb5abaaf82c2b")
+
+  test "goerli":
+    checkRoot(
+      "goerli",
+      "895390e92edc03df7096e9f51e51896e8dbe6e7e838180dadbfd869fdd77a659")
+
+  test "sepolia":
+    checkRoot(
+      "sepolia",
+      "fb9afe32150fa39f4b346be2519a67e2a4f5efcd50a1dc192c3f6b3d013d2798")