From b3c6320d56b8a929b846220c755a5b8b584c6b7f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 May 2023 13:11:00 +0200 Subject: [PATCH] embed genesis states using incbin (#4905) --- AllTests-mainnet.md | 9 +- beacon_chain/conf.nim | 4 +- beacon_chain/networking/network_metadata.nim | 118 ++++++++++-------- .../networking/network_metadata_gnosis.S | 43 +++++++ .../networking/network_metadata_mainnet.S | 67 ++++++++++ beacon_chain/nimbus_beacon_node.nim | 8 +- beacon_chain/nimbus_light_client.nim | 4 +- nfuzz/libnfuzz.nim | 19 ++- tests/all_tests.nim | 1 + tests/test_network_metadata.nim | 39 ++++++ 10 files changed, 243 insertions(+), 69 deletions(-) create mode 100644 beacon_chain/networking/network_metadata_gnosis.S create mode 100644 beacon_chain/networking/network_metadata_mainnet.S create mode 100644 tests/test_network_metadata.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 2dcd766d0..e79d1a145 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -356,6 +356,13 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + Voluntary exit signatures OK ``` OK: 8/8 Fail: 0/8 Skip: 0/8 +## Network metadata +```diff ++ goerli OK ++ mainnet OK ++ sepolia OK +``` +OK: 3/3 Fail: 0/3 Skip: 0/3 ## Nimbus remote signer/signing test (verifying-web3signer) ```diff + Signing BeaconBlock (getBlockSignature(altair)) OK @@ -669,4 +676,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 382/387 Fail: 0/387 Skip: 5/387 +OK: 385/390 Fail: 0/390 Skip: 5/390 diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 8d809e074..d34add521 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -1296,9 +1296,9 @@ proc loadEth2Network*( getMetadataForNetwork(eth2Network.get) else: when const_preset == "gnosis": - gnosisMetadata + getMetadataForNetwork("gnosis") elif const_preset == "mainnet": - mainnetMetadata + getMetadataForNetwork("mainnet") else: # Presumably other configurations can have other defaults, but for now # this simplifies the flow diff --git a/beacon_chain/networking/network_metadata.nim b/beacon_chain/networking/network_metadata.nim index 83225597c..b477cb86e 100644 --- a/beacon_chain/networking/network_metadata.nim +++ b/beacon_chain/networking/network_metadata.nim @@ -13,14 +13,8 @@ import web3/[ethtypes, conversions], chronicles, eth/common/eth_types_json_serialization, - ../spec/eth2_ssz_serialization, - ../spec/datatypes/phase0 + ../spec/eth2_ssz_serialization -# ATTENTION! This file will produce a large C file, because we are inlining -# genesis states as C literals in the generated code (and blobs in the final -# binary). It makes sense to keep the file small and separated from the rest -# of the module in order go gain maximum efficiency in incremental compilation. -# # TODO(zah): # We can compress the embedded states with snappy before embedding them here. @@ -68,28 +62,14 @@ type depositContractBlock*: uint64 depositContractBlockHash*: Eth2Digest - # Please note that we are using `string` here because SSZ.decode - # is not currently usable at compile time and we want to load the - # network metadata into a constant. - # - # We could have also used `seq[byte]`, but this results in a lot - # more generated code that slows down compilation. The impact on - # compilation times of embedding the genesis as a string is roughly - # 0.1s on my machine (you can test this by choosing an invalid name - # for the genesis file below). - # # `genesisData` will have `len == 0` for networks with a still # unknown genesis state. - genesisData*: string - genesisDepositsSnapshot*: string + genesisData*: seq[byte] + genesisDepositsSnapshot*: seq[byte] else: incompatibilityDesc*: string -const - eth2NetworksDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/eth2-networks" - sepoliaDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/sepolia" - -proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect].} = +proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError].} = # Read a list of ENR values from a YAML file containing a flat list of entries if fileExists(path): splitLines(readFile(path)). @@ -98,7 +78,7 @@ proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect]. else: @[] -proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} = +proc readBootEnr*(path: string): seq[string] {.raises: [IOError].} = # Read a list of ENR values from a YAML file containing a flat list of entries if fileExists(path): splitLines(readFile(path)). @@ -107,8 +87,9 @@ proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} = else: @[] -proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Eth2NetworkMetadata - {.raises: [CatchableError, Defect].} = +proc loadEth2NetworkMetadata*( + path: string, eth1Network = none(Eth1Network), loadGenesis = true): + Eth2NetworkMetadata {.raises: [CatchableError].} = # Load data in eth2-networks format # https://github.com/eth-clients/eth2-networks @@ -175,15 +156,15 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et readBootstrapNodes(bootstrapNodesPath) & readBootEnr(bootEnrPath)) - genesisData = if fileExists(genesisPath): - readFile(genesisPath) + genesisData = if loadGenesis and fileExists(genesisPath): + toBytes(readFile(genesisPath)) else: - "" + @[] genesisDepositsSnapshot = if fileExists(genesisDepositsSnapshotPath): - readFile(genesisDepositsSnapshotPath) + toBytes(readFile(genesisDepositsSnapshotPath)) else: - "" + @[] Eth2NetworkMetadata( incompatible: false, @@ -201,10 +182,11 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et proc loadCompileTimeNetworkMetadata( path: string, - eth1Network = none(Eth1Network)): Eth2NetworkMetadata {.raises: [Defect].} = + eth1Network = none(Eth1Network), + loadGenesis = true): Eth2NetworkMetadata {.raises: [].} = if fileExists(path & "/config.yaml"): try: - result = loadEth2NetworkMetadata(path, eth1Network) + result = loadEth2NetworkMetadata(path, eth1Network, loadGenesis) if result.incompatible: macros.error "The current build is misconfigured. " & "Attempt to load an incompatible network metadata: " & @@ -214,26 +196,55 @@ proc loadCompileTimeNetworkMetadata( else: macros.error "config.yaml not found for network '" & path -template eth2Network(path: string, eth1Network: Eth1Network): Eth2NetworkMetadata = - loadCompileTimeNetworkMetadata(eth2NetworksDir & "/" & path, - some eth1Network) +const vendorDir = + currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor" when const_preset == "gnosis": + import stew/assign2 + + let + gnosisGenesis {.importc: "gnosis_mainnet_genesis".}: ptr UncheckedArray[byte] + gnosisGenesisSize {.importc: "gnosis_mainnet_genesis_size".}: int + + {.compile: "network_metadata_gnosis.S".} + const - gnosisMetadata* = loadCompileTimeNetworkMetadata( - currentSourcePath.parentDir.replace('\\', '/') & - "/../../vendor/gnosis-chain-configs/mainnet") + gnosisMetadata = loadCompileTimeNetworkMetadata( + vendorDir & "/gnosis-chain-configs/mainnet") + static: checkForkConsistency(gnosisMetadata.cfg) doAssert gnosisMetadata.cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH doAssert gnosisMetadata.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH elif const_preset == "mainnet": + import stew/assign2 + + # Nim is very inefficent at loading large constants from binary files so we + # use this trick instead which saves significant amounts of compile time + let + mainnetGenesis {.importc: "eth2_mainnet_genesis".}: ptr UncheckedArray[byte] + mainnetGenesisSize {.importc: "eth2_mainnet_genesis_size".}: int + + praterGenesis {.importc: "eth2_goerli_genesis".}: ptr UncheckedArray[byte] + praterGenesisSize {.importc: "eth2_goerli_genesis_size".}: int + + sepoliaGenesis {.importc: "eth2_sepolia_genesis".}: ptr UncheckedArray[byte] + sepoliaGenesisSize {.importc: "eth2_sepolia_genesis_size".}: int + + {.compile: "network_metadata_mainnet.S".} + const - mainnetMetadata* = eth2Network("shared/mainnet", mainnet) - praterMetadata* = eth2Network("shared/prater", goerli) - sepoliaMetadata* = - loadCompileTimeNetworkMetadata(sepoliaDir & "/bepolia", some sepolia) + eth2NetworksDir = vendorDir & "/eth2-networks" + sepoliaDir = vendorDir & "/sepolia" + + mainnetMetadata = loadCompileTimeNetworkMetadata( + vendorDir & "/eth2-networks/shared/mainnet", some mainnet, false) + praterMetadata = loadCompileTimeNetworkMetadata( + vendorDir & "/eth2-networks/shared/prater", some goerli, false) + sepoliaMetadata = loadCompileTimeNetworkMetadata( + vendorDir & "/sepolia/bepolia", some sepolia, false) + static: for network in [mainnetMetadata, praterMetadata, sepoliaMetadata]: checkForkConsistency(network.cfg) @@ -245,7 +256,7 @@ elif const_preset == "mainnet": doAssert network.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH proc getMetadataForNetwork*( - networkName: string): Eth2NetworkMetadata {.raises: [Defect, IOError].} = + networkName: string): Eth2NetworkMetadata {.raises: [IOError].} = template loadRuntimeMetadata(): auto = if fileExists(networkName / "config.yaml"): try: @@ -260,26 +271,31 @@ proc getMetadataForNetwork*( if networkName == "ropsten": warn "Ropsten is unsupported; https://blog.ethereum.org/2022/11/30/ropsten-shutdown-announcement suggests migrating to Goerli or Sepolia" + template withGenesis(metadata, genesis: untyped): untyped = + var tmp = metadata + assign(tmp.genesisData, genesis.toOpenArray(0, `genesis Size` - 1)) + tmp + let metadata = when const_preset == "gnosis": case toLowerAscii(networkName) of "gnosis": - gnosisMetadata + withGenesis(gnosisMetadata, gnosisGenesis) of "gnosis-chain": warn "`--network:gnosis-chain` is deprecated, " & "use `--network:gnosis` instead" - gnosisMetadata + withGenesis(gnosisMetadata, gnosisGenesis) else: loadRuntimeMetadata() elif const_preset == "mainnet": case toLowerAscii(networkName) of "mainnet": - mainnetMetadata + withGenesis(mainnetMetadata, mainnetGenesis) of "prater", "goerli": - praterMetadata + withGenesis(praterMetadata, praterGenesis) of "sepolia": - sepoliaMetadata + withGenesis(sepoliaMetadata, sepoliaGenesis) else: loadRuntimeMetadata() @@ -294,7 +310,7 @@ proc getMetadataForNetwork*( metadata proc getRuntimeConfig*( - eth2Network: Option[string]): RuntimeConfig {.raises: [Defect, IOError].} = + eth2Network: Option[string]): RuntimeConfig {.raises: [IOError].} = ## Returns the run-time config for a network specified on the command line ## If the network is not explicitly specified, the function will act as the ## regular Nimbus binary, returning the mainnet config. diff --git a/beacon_chain/networking/network_metadata_gnosis.S b/beacon_chain/networking/network_metadata_gnosis.S new file mode 100644 index 000000000..ba29770ed --- /dev/null +++ b/beacon_chain/networking/network_metadata_gnosis.S @@ -0,0 +1,43 @@ +# beacon_chain +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +#if defined(__APPLE__) +# define cdecl(s) _##s +#else +# define cdecl(s) s +#endif + +#if defined(__linux__) + .section .rodata,"a",@progbits +#elif defined(__APPLE__) + .section __TEXT,__const +#elif defined(__WIN32__) + .section .rdata,"dr" +#else + .text +#endif + +# name_data = start of data +# name_end = end of data (without alignment) +# name = 64-bit pointer to data +# name_size = 64-bit length in bytes + +gnosis_mainnet_genesis_data: + .incbin "../../vendor/gnosis-chain-configs/mainnet/genesis.ssz" +gnosis_mainnet_genesis_end: + .global cdecl(gnosis_mainnet_genesis_size) +cdecl(gnosis_mainnet_genesis_size): + .quad gnosis_mainnet_genesis_end - gnosis_mainnet_genesis_data + +#if defined(__APPLE__) + .section __DATA,__const +#endif + + .global cdecl(gnosis_mainnet_genesis) + .p2align 3 +cdecl(gnosis_mainnet_genesis): + .quad gnosis_mainnet_genesis_data diff --git a/beacon_chain/networking/network_metadata_mainnet.S b/beacon_chain/networking/network_metadata_mainnet.S new file mode 100644 index 000000000..6deddee5c --- /dev/null +++ b/beacon_chain/networking/network_metadata_mainnet.S @@ -0,0 +1,67 @@ +# beacon_chain +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +#if defined(__APPLE__) +# define cdecl(s) _##s +#else +# define cdecl(s) s +#endif + +#if defined(__linux__) + .section .rodata,"a",@progbits +#elif defined(__APPLE__) + .section __TEXT,__const +#elif defined(__WIN32__) + .section .rdata,"dr" +#else + .text +#endif + +# name_data = start of data +# name_end = end of data (without alignment) +# name = 64-bit pointer to data +# name_size = 64-bit length in bytes + +eth2_mainnet_genesis_data: + .incbin "../../vendor/eth2-networks/shared/mainnet/genesis.ssz" +eth2_mainnet_genesis_end: + .global cdecl(eth2_mainnet_genesis_size) +cdecl(eth2_mainnet_genesis_size): + .quad eth2_mainnet_genesis_end - eth2_mainnet_genesis_data + +eth2_goerli_genesis_data: + .incbin "../../vendor/eth2-networks/shared/prater/genesis.ssz" +eth2_goerli_genesis_end: + .global cdecl(eth2_goerli_genesis_size) +cdecl(eth2_goerli_genesis_size): + .quad eth2_goerli_genesis_end - eth2_goerli_genesis_data + +eth2_sepolia_genesis_data: + .incbin "../../vendor/sepolia/bepolia/genesis.ssz" +eth2_sepolia_genesis_end: + .global cdecl(eth2_sepolia_genesis_size) +cdecl(eth2_sepolia_genesis_size): + .quad eth2_sepolia_genesis_end - eth2_sepolia_genesis_data + +#if defined(__APPLE__) + .section __DATA,__const +#endif + + .global cdecl(eth2_mainnet_genesis) + .p2align 3 +cdecl(eth2_mainnet_genesis): + .quad eth2_mainnet_genesis_data + + .global cdecl(eth2_goerli_genesis) + .p2align 3 +cdecl(eth2_goerli_genesis): + .quad eth2_goerli_genesis_data + + .global cdecl(eth2_sepolia_genesis) + .p2align 3 +cdecl(eth2_sepolia_genesis): + .quad eth2_sepolia_genesis_data diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 276d7f88f..0a196547c 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -542,9 +542,7 @@ proc init*(T: type BeaconNode, var genesisState = if metadata.genesisData.len > 0: try: - newClone readSszForkedHashedBeaconState( - cfg, - metadata.genesisData.toOpenArrayByte(0, metadata.genesisData.high)) + newClone readSszForkedHashedBeaconState(cfg, metadata.genesisData) except CatchableError as err: raiseAssert "Invalid baked-in state: " & err.msg else: @@ -2057,9 +2055,7 @@ proc handleStartUpCmd(config: var BeaconNodeConf) {.raises: [Defect, CatchableEr stateId: "finalized") genesis = if network.genesisData.len > 0: - newClone(readSszForkedHashedBeaconState( - cfg, - network.genesisData.toOpenArrayByte(0, network.genesisData.high()))) + newClone(readSszForkedHashedBeaconState(cfg, network.genesisData)) else: nil if config.blockId.isSome(): diff --git a/beacon_chain/nimbus_light_client.nim b/beacon_chain/nimbus_light_client.nim index 8b39c0fd8..6e1d9e24b 100644 --- a/beacon_chain/nimbus_light_client.nim +++ b/beacon_chain/nimbus_light_client.nim @@ -65,9 +65,7 @@ programMain: let genesisState = try: - template genesisData(): auto = metadata.genesisData - newClone(readSszForkedHashedBeaconState( - cfg, genesisData.toOpenArrayByte(genesisData.low, genesisData.high))) + newClone(readSszForkedHashedBeaconState(cfg, metadata.genesisData)) except CatchableError as err: raiseAssert "Invalid baked-in state: " & err.msg diff --git a/nfuzz/libnfuzz.nim b/nfuzz/libnfuzz.nim index dacaf30fb..8692cc9dc 100644 --- a/nfuzz/libnfuzz.nim +++ b/nfuzz/libnfuzz.nim @@ -1,8 +1,15 @@ +# beacon_chain +# Copyright (c) 2019-2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + # Required for deserialisation of ValidatorSig in Attestation due to # https://github.com/nim-lang/Nim/issues/11225 import - stew/ptrops, stew/ranges/ptr_arith, chronicles, + stew/ptrops, chronicles, ../beacon_chain/networking/network_metadata, ../beacon_chain/spec/datatypes/phase0, ../beacon_chain/spec/[ @@ -103,7 +110,7 @@ proc nfuzz_attestation(input: openArray[byte], xoutput: ptr byte, proc nfuzz_attester_slashing(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = decodeAndProcess(AttesterSlashingInput): - process_attester_slashing(mainnetMetadata.cfg, data.state, data.attesterSlashing, flags, cache).isOk + process_attester_slashing(getRuntimeConfig(some "mainnet"), data.state, data.attesterSlashing, flags, cache).isOk proc nfuzz_block(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = @@ -127,7 +134,7 @@ proc nfuzz_block(input: openArray[byte], xoutput: ptr byte, decodeAndProcess(BlockInput): state_transition( - mainnetMetadata.cfg, data, data.beaconBlock, flags, noRollback).isOk + getRuntimeConfig(some "mainnet"), data, data.beaconBlock, flags, noRollback).isOk proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = @@ -137,17 +144,17 @@ proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte, proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = decodeAndProcess(DepositInput): - process_deposit(mainnetMetadata.cfg, data.state, data.deposit, flags).isOk + process_deposit(getRuntimeConfig(some "mainnet"), data.state, data.deposit, flags).isOk proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = decodeAndProcess(ProposerSlashingInput): - process_proposer_slashing(mainnetMetadata.cfg, data.state, data.proposerSlashing, flags, cache).isOk + process_proposer_slashing(getRuntimeConfig(some "mainnet"), data.state, data.proposerSlashing, flags, cache).isOk proc nfuzz_voluntary_exit(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} = decodeAndProcess(VoluntaryExitInput): - process_voluntary_exit(mainnetMetadata.cfg, data.state, data.exit, flags, cache).isOk + process_voluntary_exit(getRuntimeConfig(some "mainnet"), data.state, data.exit, flags, cache).isOk # Note: Could also accept raw input pointer and access list_size + seed here. # However, list_size needs to be known also outside this proc to allocate xoutput. diff --git a/tests/all_tests.nim b/tests/all_tests.nim index b73eb8467..a967ae9b4 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -36,6 +36,7 @@ import # Unit test ./test_light_client_processor, ./test_light_client, ./test_message_signatures, + ./test_network_metadata, ./test_peer_pool, ./test_remote_keystore, ./test_serialization, diff --git a/tests/test_network_metadata.nim b/tests/test_network_metadata.nim new file mode 100644 index 000000000..3e59d4bdc --- /dev/null +++ b/tests/test_network_metadata.nim @@ -0,0 +1,39 @@ +# beacon_chain +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + unittest2, + ../beacon_chain/networking/network_metadata, + ../beacon_chain/spec/forks + +{.used.} + +template checkRoot(name, root) = + let + metadata = getMetadataForNetwork(name) + cfg = metadata.cfg + state = newClone(readSszForkedHashedBeaconState( + metadata.cfg, metadata.genesisData)) + + check: + $getStateRoot(state[]) == root + +suite "Network metadata": + test "mainnet": + checkRoot( + "mainnet", + "7e76880eb67bbdc86250aa578958e9d0675e64e714337855204fb5abaaf82c2b") + + test "goerli": + checkRoot( + "goerli", + "895390e92edc03df7096e9f51e51896e8dbe6e7e838180dadbfd869fdd77a659") + + test "sepolia": + checkRoot( + "sepolia", + "fb9afe32150fa39f4b346be2519a67e2a4f5efcd50a1dc192c3f6b3d013d2798")