status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

embed genesis states using incbin (#4905)

This commit is contained in:
Jacek Sieka 2023-05-11 13:11:00 +02:00 committed by GitHub
parent 2fcc01f516
commit b3c6320d56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 243 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
AllTests-mainnet.md
View File

@ -356,6 +356,13 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
+ Voluntary exit signatures OK
```
OK: 8/8 Fail: 0/8 Skip: 0/8
## Network metadata
```diff
+ goerli OK
+ mainnet OK
+ sepolia OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Nimbus remote signer/signing test (verifying-web3signer)
```diff
+ Signing BeaconBlock (getBlockSignature(altair)) OK
@ -669,4 +676,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
OK: 9/9 Fail: 0/9 Skip: 0/9
---TOTAL---
OK: 382/387 Fail: 0/387 Skip: 5/387
OK: 385/390 Fail: 0/390 Skip: 5/390

4
beacon_chain/conf.nim
View File

@ -1296,9 +1296,9 @@ proc loadEth2Network*(
getMetadataForNetwork(eth2Network.get)
else:
when const_preset == "gnosis":
gnosisMetadata
getMetadataForNetwork("gnosis")
elif const_preset == "mainnet":
mainnetMetadata
getMetadataForNetwork("mainnet")
else:
# Presumably other configurations can have other defaults, but for now
# this simplifies the flow

118
beacon_chain/networking/network_metadata.nim
View File

@ -13,14 +13,8 @@ import
web3/[ethtypes, conversions],
chronicles,
eth/common/eth_types_json_serialization,
../spec/eth2_ssz_serialization,
../spec/datatypes/phase0
../spec/eth2_ssz_serialization
# ATTENTION! This file will produce a large C file, because we are inlining
# genesis states as C literals in the generated code (and blobs in the final
# binary). It makes sense to keep the file small and separated from the rest
# of the module in order go gain maximum efficiency in incremental compilation.
#
# TODO(zah):
# We can compress the embedded states with snappy before embedding them here.
@ -68,28 +62,14 @@ type
depositContractBlock*: uint64
depositContractBlockHash*: Eth2Digest
# Please note that we are using `string` here because SSZ.decode
# is not currently usable at compile time and we want to load the
# network metadata into a constant.
#
# We could have also used `seq[byte]`, but this results in a lot
# more generated code that slows down compilation. The impact on
# compilation times of embedding the genesis as a string is roughly
# 0.1s on my machine (you can test this by choosing an invalid name
# for the genesis file below).
#
# `genesisData` will have `len == 0` for networks with a still
# unknown genesis state.
genesisData*: string
genesisDepositsSnapshot*: string
genesisData*: seq[byte]
genesisDepositsSnapshot*: seq[byte]
else:
incompatibilityDesc*: string
const
eth2NetworksDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/eth2-networks"
sepoliaDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor/sepolia"
proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect].} =
proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError].} =
# Read a list of ENR values from a YAML file containing a flat list of entries
if fileExists(path):
splitLines(readFile(path)).
@ -98,7 +78,7 @@ proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError, Defect].
else:
@[]
proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} =
proc readBootEnr*(path: string): seq[string] {.raises: [IOError].} =
# Read a list of ENR values from a YAML file containing a flat list of entries
if fileExists(path):
splitLines(readFile(path)).
@ -107,8 +87,9 @@ proc readBootEnr*(path: string): seq[string] {.raises: [IOError, Defect].} =
else:
@[]
proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Eth2NetworkMetadata
{.raises: [CatchableError, Defect].} =
proc loadEth2NetworkMetadata*(
path: string, eth1Network = none(Eth1Network), loadGenesis = true):
Eth2NetworkMetadata {.raises: [CatchableError].} =
# Load data in eth2-networks format
# https://github.com/eth-clients/eth2-networks
@ -175,15 +156,15 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
readBootstrapNodes(bootstrapNodesPath) &
readBootEnr(bootEnrPath))
genesisData = if fileExists(genesisPath):
readFile(genesisPath)
genesisData = if loadGenesis and fileExists(genesisPath):
toBytes(readFile(genesisPath))
else:
""
@[]
genesisDepositsSnapshot = if fileExists(genesisDepositsSnapshotPath):
readFile(genesisDepositsSnapshotPath)
toBytes(readFile(genesisDepositsSnapshotPath))
else:
""
@[]
Eth2NetworkMetadata(
incompatible: false,
@ -201,10 +182,11 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
proc loadCompileTimeNetworkMetadata(
path: string,
eth1Network = none(Eth1Network)): Eth2NetworkMetadata {.raises: [Defect].} =
eth1Network = none(Eth1Network),
loadGenesis = true): Eth2NetworkMetadata {.raises: [].} =
if fileExists(path & "/config.yaml"):
try:
result = loadEth2NetworkMetadata(path, eth1Network)
result = loadEth2NetworkMetadata(path, eth1Network, loadGenesis)
if result.incompatible:
macros.error "The current build is misconfigured. " &
"Attempt to load an incompatible network metadata: " &
@ -214,26 +196,55 @@ proc loadCompileTimeNetworkMetadata(
else:
macros.error "config.yaml not found for network '" & path
template eth2Network(path: string, eth1Network: Eth1Network): Eth2NetworkMetadata =
loadCompileTimeNetworkMetadata(eth2NetworksDir & "/" & path,
some eth1Network)
const vendorDir =
currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor"
when const_preset == "gnosis":
import stew/assign2
let
gnosisGenesis {.importc: "gnosis_mainnet_genesis".}: ptr UncheckedArray[byte]
gnosisGenesisSize {.importc: "gnosis_mainnet_genesis_size".}: int
{.compile: "network_metadata_gnosis.S".}
const
gnosisMetadata* = loadCompileTimeNetworkMetadata(
currentSourcePath.parentDir.replace('\\', '/') &
"/../../vendor/gnosis-chain-configs/mainnet")
gnosisMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/gnosis-chain-configs/mainnet")
static:
checkForkConsistency(gnosisMetadata.cfg)
doAssert gnosisMetadata.cfg.CAPELLA_FORK_EPOCH == FAR_FUTURE_EPOCH
doAssert gnosisMetadata.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH
elif const_preset == "mainnet":
import stew/assign2
# Nim is very inefficent at loading large constants from binary files so we
# use this trick instead which saves significant amounts of compile time
let
mainnetGenesis {.importc: "eth2_mainnet_genesis".}: ptr UncheckedArray[byte]
mainnetGenesisSize {.importc: "eth2_mainnet_genesis_size".}: int
praterGenesis {.importc: "eth2_goerli_genesis".}: ptr UncheckedArray[byte]
praterGenesisSize {.importc: "eth2_goerli_genesis_size".}: int
sepoliaGenesis {.importc: "eth2_sepolia_genesis".}: ptr UncheckedArray[byte]
sepoliaGenesisSize {.importc: "eth2_sepolia_genesis_size".}: int
{.compile: "network_metadata_mainnet.S".}
const
mainnetMetadata* = eth2Network("shared/mainnet", mainnet)
praterMetadata* = eth2Network("shared/prater", goerli)
sepoliaMetadata* =
loadCompileTimeNetworkMetadata(sepoliaDir & "/bepolia", some sepolia)
eth2NetworksDir = vendorDir & "/eth2-networks"
sepoliaDir = vendorDir & "/sepolia"
mainnetMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/eth2-networks/shared/mainnet", some mainnet, false)
praterMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/eth2-networks/shared/prater", some goerli, false)
sepoliaMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/sepolia/bepolia", some sepolia, false)
static:
for network in [mainnetMetadata, praterMetadata, sepoliaMetadata]:
checkForkConsistency(network.cfg)
@ -245,7 +256,7 @@ elif const_preset == "mainnet":
doAssert network.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH
proc getMetadataForNetwork*(
networkName: string): Eth2NetworkMetadata {.raises: [Defect, IOError].} =
networkName: string): Eth2NetworkMetadata {.raises: [IOError].} =
template loadRuntimeMetadata(): auto =
if fileExists(networkName / "config.yaml"):
try:
@ -260,26 +271,31 @@ proc getMetadataForNetwork*(
if networkName == "ropsten":
warn "Ropsten is unsupported; https://blog.ethereum.org/2022/11/30/ropsten-shutdown-announcement suggests migrating to Goerli or Sepolia"
template withGenesis(metadata, genesis: untyped): untyped =
var tmp = metadata
assign(tmp.genesisData, genesis.toOpenArray(0, `genesis Size` - 1))
tmp
let metadata =
when const_preset == "gnosis":
case toLowerAscii(networkName)
of "gnosis":
gnosisMetadata
withGenesis(gnosisMetadata, gnosisGenesis)
of "gnosis-chain":
warn "`--network:gnosis-chain` is deprecated, " &
"use `--network:gnosis` instead"
gnosisMetadata
withGenesis(gnosisMetadata, gnosisGenesis)
else:
loadRuntimeMetadata()
elif const_preset == "mainnet":
case toLowerAscii(networkName)
of "mainnet":
mainnetMetadata
withGenesis(mainnetMetadata, mainnetGenesis)
of "prater", "goerli":
praterMetadata
withGenesis(praterMetadata, praterGenesis)
of "sepolia":
sepoliaMetadata
withGenesis(sepoliaMetadata, sepoliaGenesis)
else:
loadRuntimeMetadata()
@ -294,7 +310,7 @@ proc getMetadataForNetwork*(
metadata
proc getRuntimeConfig*(
eth2Network: Option[string]): RuntimeConfig {.raises: [Defect, IOError].} =
eth2Network: Option[string]): RuntimeConfig {.raises: [IOError].} =
## Returns the run-time config for a network specified on the command line
## If the network is not explicitly specified, the function will act as the
## regular Nimbus binary, returning the mainnet config.

43
beacon_chain/networking/network_metadata_gnosis.S Normal file
View File

@ -0,0 +1,43 @@
# beacon_chain
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
#if defined(__APPLE__)
# define cdecl(s) _##s
#else
# define cdecl(s) s
#endif
#if defined(__linux__)
.section .rodata,"a",@progbits
#elif defined(__APPLE__)
.section __TEXT,__const
#elif defined(__WIN32__)
.section .rdata,"dr"
#else
.text
#endif
# name_data = start of data
# name_end = end of data (without alignment)
# name = 64-bit pointer to data
# name_size = 64-bit length in bytes
gnosis_mainnet_genesis_data:
.incbin "../../vendor/gnosis-chain-configs/mainnet/genesis.ssz"
gnosis_mainnet_genesis_end:
.global cdecl(gnosis_mainnet_genesis_size)
cdecl(gnosis_mainnet_genesis_size):
.quad gnosis_mainnet_genesis_end - gnosis_mainnet_genesis_data
#if defined(__APPLE__)
.section __DATA,__const
#endif
.global cdecl(gnosis_mainnet_genesis)
.p2align 3
cdecl(gnosis_mainnet_genesis):
.quad gnosis_mainnet_genesis_data

67
beacon_chain/networking/network_metadata_mainnet.S Normal file
View File

@ -0,0 +1,67 @@
# beacon_chain
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
#if defined(__APPLE__)
# define cdecl(s) _##s
#else
# define cdecl(s) s
#endif
#if defined(__linux__)
.section .rodata,"a",@progbits
#elif defined(__APPLE__)
.section __TEXT,__const
#elif defined(__WIN32__)
.section .rdata,"dr"
#else
.text
#endif
# name_data = start of data
# name_end = end of data (without alignment)
# name = 64-bit pointer to data
# name_size = 64-bit length in bytes
eth2_mainnet_genesis_data:
.incbin "../../vendor/eth2-networks/shared/mainnet/genesis.ssz"
eth2_mainnet_genesis_end:
.global cdecl(eth2_mainnet_genesis_size)
cdecl(eth2_mainnet_genesis_size):
.quad eth2_mainnet_genesis_end - eth2_mainnet_genesis_data
eth2_goerli_genesis_data:
.incbin "../../vendor/eth2-networks/shared/prater/genesis.ssz"
eth2_goerli_genesis_end:
.global cdecl(eth2_goerli_genesis_size)
cdecl(eth2_goerli_genesis_size):
.quad eth2_goerli_genesis_end - eth2_goerli_genesis_data
eth2_sepolia_genesis_data:
.incbin "../../vendor/sepolia/bepolia/genesis.ssz"
eth2_sepolia_genesis_end:
.global cdecl(eth2_sepolia_genesis_size)
cdecl(eth2_sepolia_genesis_size):
.quad eth2_sepolia_genesis_end - eth2_sepolia_genesis_data
#if defined(__APPLE__)
.section __DATA,__const
#endif
.global cdecl(eth2_mainnet_genesis)
.p2align 3
cdecl(eth2_mainnet_genesis):
.quad eth2_mainnet_genesis_data
.global cdecl(eth2_goerli_genesis)
.p2align 3
cdecl(eth2_goerli_genesis):
.quad eth2_goerli_genesis_data
.global cdecl(eth2_sepolia_genesis)
.p2align 3
cdecl(eth2_sepolia_genesis):
.quad eth2_sepolia_genesis_data

8
beacon_chain/nimbus_beacon_node.nim
View File

@ -542,9 +542,7 @@ proc init*(T: type BeaconNode,
var genesisState =
if metadata.genesisData.len > 0:
try:
newClone readSszForkedHashedBeaconState(
cfg,
metadata.genesisData.toOpenArrayByte(0, metadata.genesisData.high))
newClone readSszForkedHashedBeaconState(cfg, metadata.genesisData)
except CatchableError as err:
raiseAssert "Invalid baked-in state: " & err.msg
else:
@ -2057,9 +2055,7 @@ proc handleStartUpCmd(config: var BeaconNodeConf) {.raises: [Defect, CatchableEr
stateId: "finalized")
genesis =
if network.genesisData.len > 0:
newClone(readSszForkedHashedBeaconState(
cfg,
network.genesisData.toOpenArrayByte(0, network.genesisData.high())))
newClone(readSszForkedHashedBeaconState(cfg, network.genesisData))
else: nil
if config.blockId.isSome():

4
beacon_chain/nimbus_light_client.nim
View File

@ -65,9 +65,7 @@ programMain:
let
genesisState =
try:
template genesisData(): auto = metadata.genesisData
newClone(readSszForkedHashedBeaconState(
cfg, genesisData.toOpenArrayByte(genesisData.low, genesisData.high)))
newClone(readSszForkedHashedBeaconState(cfg, metadata.genesisData))
except CatchableError as err:
raiseAssert "Invalid baked-in state: " & err.msg

19
nfuzz/libnfuzz.nim
View File

@ -1,8 +1,15 @@
# beacon_chain
# Copyright (c) 2019-2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Required for deserialisation of ValidatorSig in Attestation due to
# https://github.com/nim-lang/Nim/issues/11225
import
stew/ptrops, stew/ranges/ptr_arith, chronicles,
stew/ptrops, chronicles,
../beacon_chain/networking/network_metadata,
../beacon_chain/spec/datatypes/phase0,
../beacon_chain/spec/[
@ -103,7 +110,7 @@ proc nfuzz_attestation(input: openArray[byte], xoutput: ptr byte,
proc nfuzz_attester_slashing(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(AttesterSlashingInput):
process_attester_slashing(mainnetMetadata.cfg, data.state, data.attesterSlashing, flags, cache).isOk
process_attester_slashing(getRuntimeConfig(some "mainnet"), data.state, data.attesterSlashing, flags, cache).isOk
proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
@ -127,7 +134,7 @@ proc nfuzz_block(input: openArray[byte], xoutput: ptr byte,
decodeAndProcess(BlockInput):
state_transition(
mainnetMetadata.cfg, data, data.beaconBlock, flags, noRollback).isOk
getRuntimeConfig(some "mainnet"), data, data.beaconBlock, flags, noRollback).isOk
proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
@ -137,17 +144,17 @@ proc nfuzz_block_header(input: openArray[byte], xoutput: ptr byte,
proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(DepositInput):
process_deposit(mainnetMetadata.cfg, data.state, data.deposit, flags).isOk
process_deposit(getRuntimeConfig(some "mainnet"), data.state, data.deposit, flags).isOk
proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(ProposerSlashingInput):
process_proposer_slashing(mainnetMetadata.cfg, data.state, data.proposerSlashing, flags, cache).isOk
process_proposer_slashing(getRuntimeConfig(some "mainnet"), data.state, data.proposerSlashing, flags, cache).isOk
proc nfuzz_voluntary_exit(input: openArray[byte], xoutput: ptr byte,
xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError, Defect].} =
decodeAndProcess(VoluntaryExitInput):
process_voluntary_exit(mainnetMetadata.cfg, data.state, data.exit, flags, cache).isOk
process_voluntary_exit(getRuntimeConfig(some "mainnet"), data.state, data.exit, flags, cache).isOk
# Note: Could also accept raw input pointer and access list_size + seed here.
# However, list_size needs to be known also outside this proc to allocate xoutput.

1
tests/all_tests.nim
View File

@ -36,6 +36,7 @@ import # Unit test
./test_light_client_processor,
./test_light_client,
./test_message_signatures,
./test_network_metadata,
./test_peer_pool,
./test_remote_keystore,
./test_serialization,

39
tests/test_network_metadata.nim Normal file
View File

@ -0,0 +1,39 @@
# beacon_chain
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
unittest2,
../beacon_chain/networking/network_metadata,
../beacon_chain/spec/forks
{.used.}
template checkRoot(name, root) =
let
metadata = getMetadataForNetwork(name)
cfg = metadata.cfg
state = newClone(readSszForkedHashedBeaconState(
metadata.cfg, metadata.genesisData))
check:
$getStateRoot(state[]) == root
suite "Network metadata":
test "mainnet":
checkRoot(
"mainnet",
"7e76880eb67bbdc86250aa578958e9d0675e64e714337855204fb5abaaf82c2b")
test "goerli":
checkRoot(
"goerli",
"895390e92edc03df7096e9f51e51896e8dbe6e7e838180dadbfd869fdd77a659")
test "sepolia":
checkRoot(
"sepolia",
"fb9afe32150fa39f4b346be2519a67e2a4f5efcd50a1dc192c3f6b3d013d2798")