nimbus-eth2/beacon_chain/spec/eth2_merkleization.nim

109 lines
4.0 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
disentangle eth2 types from the ssz library (#2785) * reorganize ssz dependencies This PR continues the work in https://github.com/status-im/nimbus-eth2/pull/2646, https://github.com/status-im/nimbus-eth2/pull/2779 as well as past issues with serialization and type, to disentangle SSZ from eth2 and at the same time simplify imports and exports with a structured approach. The principal idea here is that when a library wants to introduce SSZ support, they do so via 3 files: * `ssz_codecs` which imports and reexports `codecs` - this covers the basic byte conversions and ensures no overloads get lost * `xxx_merkleization` imports and exports `merkleization` to specialize and get access to `hash_tree_root` and friends * `xxx_ssz_serialization` imports and exports `ssz_serialization` to specialize ssz for a specific library Those that need to interact with SSZ always import the `xxx_` versions of the modules and never `ssz` itself so as to keep imports simple and safe. This is similar to how the REST / JSON-RPC serializers are structured in that someone wanting to serialize spec types to REST-JSON will import `eth2_rest_serialization` and nothing else. * split up ssz into a core library that is independendent of eth2 types * rename `bytes_reader` to `codec` to highlight that it contains coding and decoding of bytes and native ssz types * remove tricky List init overload that causes compile issues * get rid of top-level ssz import * reenable merkleization tests * move some "standard" json serializers to spec * remove `ValidatorIndex` serialization for now * remove test_ssz_merkleization * add tests for over/underlong byte sequences * fix broken seq[byte] test - seq[byte] is not an SSZ type There are a few things this PR doesn't solve: * like #2646 this PR is weak on how to handle root and other dontSerialize fields that "sometimes" should be computed - the same problem appears in REST / JSON-RPC etc * Fix a build problem on macOS * Another way to fix the macOS builds Co-authored-by: Zahary Karadjov <zahary@gmail.com>
2021-08-18 18:57:58 +00:00
# Import this module to get access to `hash_tree_root` for spec types
import
stew/endians2,
std/sets,
ssz_serialization/[merkleization, proofs],
2022-10-27 06:29:24 +00:00
./ssz_codec
from ./datatypes/base import HashedValidatorPubKeyItem
2022-10-27 06:29:24 +00:00
from ./datatypes/phase0 import HashedBeaconState, SignedBeaconBlock
from ./datatypes/altair import HashedBeaconState, SignedBeaconBlock
from ./datatypes/bellatrix import HashedBeaconState, SignedBeaconBlock
from ./datatypes/capella import HashedBeaconState, SignedBeaconBlock
from ./datatypes/deneb import HashedBeaconState, SignedBeaconBlock
from ./datatypes/electra import HashedBeaconState, SignedBeaconBlock
2019-11-18 12:48:41 +00:00
export ssz_codec, merkleization, proofs
type
DepositsMerkleizer* = SszMerkleizer2[DEPOSIT_CONTRACT_TREE_DEPTH + 1]
2022-10-27 06:29:24 +00:00
# Can't use `ForkyHashedBeaconState`/`ForkyHashedSignedBeaconBlock` without
# creating recursive module dependency through `forks`.
func hash_tree_root*(
x: phase0.HashedBeaconState | altair.HashedBeaconState |
bellatrix.HashedBeaconState | capella.HashedBeaconState |
deneb.HashedBeaconState | electra.SignedBeaconBlock) {.
error: "HashedBeaconState should not be hashed".}
2022-10-27 06:29:24 +00:00
func hash_tree_root*(
x: phase0.SignedBeaconBlock | altair.SignedBeaconBlock |
bellatrix.SignedBeaconBlock | capella.SignedBeaconBlock |
deneb.SignedBeaconBlock | electra.SignedBeaconBlock) {.
error: "SignedBeaconBlock should not be hashed".}
func depositCountBytes*(x: uint64): array[32, byte] =
doAssert(x <= 4294967295'u64)
var z = x
for i in 0..3:
result[31-i] = byte(int64(z) %% 256'i64)
z = z div 256
func depositCountU64*(xs: openArray[byte]): uint64 =
## depositCountU64 considers just the first 4 bytes as
## MAX_DEPOSIT_COUNT is defined as 2^32 - 1.
for i in 0 .. 27:
doAssert xs[i] == 0
return uint64.fromBytesBE(xs[24..31])
func init*(T: type DepositsMerkleizer, s: DepositContractState): DepositsMerkleizer =
let count = depositCountU64(s.deposit_count)
DepositsMerkleizer.init(s.branch, count)
func toDepositContractState*(merkleizer: DepositsMerkleizer): DepositContractState =
# TODO There is an off by one discrepancy in the size of the arrays here that
# need to be investigated. It shouldn't matter as long as the tree is
# not populated to its maximum size.
result.branch[0..31] = merkleizer.getCombinedChunks[0..31]
result.deposit_count[24..31] = merkleizer.getChunkCount().toBytesBE
func getDepositsRoot*(m: var DepositsMerkleizer): Eth2Digest =
mixInLength(m.getFinalHash, int m.totalChunks)
func hash*(v: ref HashedValidatorPubKeyItem): Hash =
if not isNil(v):
hash(v[].key)
else:
default(Hash)
func `==`*(a, b: ref HashedValidatorPubKeyItem): bool =
if isNil(a):
isNil(b)
elif isNil(b):
false
else:
a[].key == b[].key
func init*(T: type HashedValidatorPubKey, key: ValidatorPubKey): HashedValidatorPubKey =
{.noSideEffect.}:
var keys {.threadvar.}: HashSet[ref HashedValidatorPubKeyItem]
let
tmp = (ref HashedValidatorPubKeyItem)(
key: key,
root: hash_tree_root(key)
)
cached =
if keys.containsOrIncl(tmp):
try:
# The interface of HashSet is such that we must construct a full
# instance to check if it's in the set - then we can return that
# instace and discard the one we just created temporarily
avoid `--gc:arc` issue in validator key caching (#6203) The current implementation of the validator key cache as introduced in #5883 leads to issues when compiling with `--gc:arc`. Namely, the assert in `injectdestructors.nim` > `destructiveMoveVar` is triggered: ```nim assert n.kind != nkSym or not hasDestructor(c, n.sym.typ) ``` `cached == nkSym`, and `n.sym.typ == ref HashedValidatorPubKeyItem` with `hasDestructor(c, n.sym.typ) == true`. Inlining the `addr ...[]` avoids the problem and allows `--gc:arc` compilation as part of LC wasm demo project. Compilation command: ```sh nim c \ -d:disable_libbacktrace \ -d:disableMarchNative \ -d:disableLTO \ -d:emscripten \ -d:release \ -d:useGcAssert \ -d:useSysAssert \ --debuginfo:off \ --nimcache:nimcache \ --os:linux \ --cpu:wasm32 \ --cc:clang \ --clang.exe:emcc \ --clang.linkerexe:emcc \ --gc:arc \ --exceptions:goto \ --define:noSignalHandler \ --define:danger \ --panics:on \ --passC:-fpic \ --passL:-Os \ --passL:-fpic \ --passC:'-pthread' \ --passL:'-pthread' \ --passC:'-sASSERTIONS' \ --passL:'-sASSERTIONS' \ --passC:'-sINITIAL_MEMORY=256MB' \ --passL:'-sINITIAL_MEMORY=256MB' \ --passC:'-sSTACK_SIZE=128MB' \ --passL:'-sSTACK_SIZE=128MB' \ --passC:'-sUSE_PTHREADS=1' \ --passL:'-sUSE_PTHREADS=1' \ --passC:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sEXPORTED_FUNCTIONS="[_free, _malloc, _NimMain, _ETHRandomNumberCreate, _ETHConsensusConfigCreateFromYaml, _ETHConsensusConfigGetConsensusVersionAtEpoch, _ETHBeaconStateCreateFromSsz, _ETHBeaconStateDestroy, _ETHBeaconStateCopyGenesisValidatorsRoot, _ETHRootDestroy, _ETHForkDigestsCreateFromState, _ETHBeaconClockCreateFromState, _ETHBeaconClockGetSlot, _ETHLightClientStoreCreateFromBootstrap, _ETHLightClientStoreDestroy, _kETHLcSyncKind_UpdatesByRange, _kETHLcSyncKind_FinalityUpdate, _kETHLcSyncKind_OptimisticUpdate, _ETHLightClientStoreGetNextSyncTask, _ETHLightClientStoreGetMillisecondsToNextSyncTask, _ETHLightClientStoreProcessUpdatesByRange, _ETHLightClientStoreProcessFinalityUpdate, _ETHLightClientStoreProcessOptimisticUpdate, _ETHLightClientStoreGetFinalizedHeader, _ETHLightClientStoreIsNextSyncCommitteeKnown, _ETHLightClientStoreGetOptimisticHeader, _ETHLightClientStoreGetSafetyThreshold, _ETHLightClientHeaderCreateCopy, _ETHLightClientHeaderDestroy, _ETHLightClientHeaderCopyBeaconRoot, _ETHLightClientHeaderGetBeacon, _ETHBeaconBlockHeaderGetSlot, _ETHBeaconBlockHeaderGetProposerIndex, _ETHBeaconBlockHeaderGetParentRoot, _ETHBeaconBlockHeaderGetStateRoot, _ETHBeaconBlockHeaderGetBodyRoot, _ETHLightClientHeaderCopyExecutionHash, _ETHLightClientHeaderGetExecution, _ETHExecutionPayloadHeaderGetParentHash, _ETHExecutionPayloadHeaderGetFeeRecipient, _ETHExecutionPayloadHeaderGetStateRoot, _ETHExecutionPayloadHeaderGetReceiptsRoot, _ETHExecutionPayloadHeaderGetLogsBloom, _ETHExecutionPayloadHeaderGetPrevRandao, _ETHExecutionPayloadHeaderGetBlockNumber, _ETHExecutionPayloadHeaderGetGasLimit, _ETHExecutionPayloadHeaderGetGasUsed, _ETHExecutionPayloadHeaderGetTimestamp, _ETHExecutionPayloadHeaderGetExtraDataBytes, _ETHExecutionPayloadHeaderGetBaseFeePerGas, _ETHExecutionPayloadHeaderGetBlobGasUsed, _ETHExecutionPayloadHeaderGetExcessBlobGas, _ETHExecutionBlockHeaderCreateFromJson, _ETHExecutionBlockHeaderDestroy, _ETHExecutionBlockHeaderGetTransactionsRoot, _ETHExecutionBlockHeaderGetWithdrawalsRoot, _ETHTransactionsCreateFromJson, _ETHTransactionsDestroy, _ETHTransactionsGetCount, _ETHTransactionsGet, _ETHTransactionGetHash, _ETHTransactionGetFrom, _ETHTransactionGetNonce, _ETHTransactionGetMaxPriorityFeePerGas, _ETHTransactionGetMaxFeePerGas, _ETHTransactionGetGas, _ETHTransactionIsCreatingContract, _ETHTransactionGetTo, _ETHTransactionGetValue, _ETHTransactionGetInputBytes, _ETHTransactionGetBytes, _ETHTransactionGetEip6493Root, _ETHTransactionGetEip6493Bytes, _ETHTransactionGetNumEip6493SnappyBytes, _ETHReceiptsCreateFromJson, _ETHReceiptsDestroy, _ETHReceiptsGet, _ETHReceiptHasStatus, _ETHReceiptGetBytes, _ETHReceiptGetEip6493Bytes, _ETHReceiptGetNumEip6493SnappyBytes]"' \ --passL:'-sEXPORTED_RUNTIME_METHODS="[lengthBytesUTF8, stringToNewUTF8]"' \ --passL:'-Wl,--no-entry' \ --noMain:on \ --passL:'-o libnimbus_lc.js' \ nimbus-eth2/beacon_chain/libnimbus_lc/libnimbus_lc.nim ```
2024-04-16 09:47:18 +00:00
addr keys[tmp][]
except KeyError:
raiseAssert "just checked"
else:
avoid `--gc:arc` issue in validator key caching (#6203) The current implementation of the validator key cache as introduced in #5883 leads to issues when compiling with `--gc:arc`. Namely, the assert in `injectdestructors.nim` > `destructiveMoveVar` is triggered: ```nim assert n.kind != nkSym or not hasDestructor(c, n.sym.typ) ``` `cached == nkSym`, and `n.sym.typ == ref HashedValidatorPubKeyItem` with `hasDestructor(c, n.sym.typ) == true`. Inlining the `addr ...[]` avoids the problem and allows `--gc:arc` compilation as part of LC wasm demo project. Compilation command: ```sh nim c \ -d:disable_libbacktrace \ -d:disableMarchNative \ -d:disableLTO \ -d:emscripten \ -d:release \ -d:useGcAssert \ -d:useSysAssert \ --debuginfo:off \ --nimcache:nimcache \ --os:linux \ --cpu:wasm32 \ --cc:clang \ --clang.exe:emcc \ --clang.linkerexe:emcc \ --gc:arc \ --exceptions:goto \ --define:noSignalHandler \ --define:danger \ --panics:on \ --passC:-fpic \ --passL:-Os \ --passL:-fpic \ --passC:'-pthread' \ --passL:'-pthread' \ --passC:'-sASSERTIONS' \ --passL:'-sASSERTIONS' \ --passC:'-sINITIAL_MEMORY=256MB' \ --passL:'-sINITIAL_MEMORY=256MB' \ --passC:'-sSTACK_SIZE=128MB' \ --passL:'-sSTACK_SIZE=128MB' \ --passC:'-sUSE_PTHREADS=1' \ --passL:'-sUSE_PTHREADS=1' \ --passC:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sEXPORTED_FUNCTIONS="[_free, _malloc, _NimMain, _ETHRandomNumberCreate, _ETHConsensusConfigCreateFromYaml, _ETHConsensusConfigGetConsensusVersionAtEpoch, _ETHBeaconStateCreateFromSsz, _ETHBeaconStateDestroy, _ETHBeaconStateCopyGenesisValidatorsRoot, _ETHRootDestroy, _ETHForkDigestsCreateFromState, _ETHBeaconClockCreateFromState, _ETHBeaconClockGetSlot, _ETHLightClientStoreCreateFromBootstrap, _ETHLightClientStoreDestroy, _kETHLcSyncKind_UpdatesByRange, _kETHLcSyncKind_FinalityUpdate, _kETHLcSyncKind_OptimisticUpdate, _ETHLightClientStoreGetNextSyncTask, _ETHLightClientStoreGetMillisecondsToNextSyncTask, _ETHLightClientStoreProcessUpdatesByRange, _ETHLightClientStoreProcessFinalityUpdate, _ETHLightClientStoreProcessOptimisticUpdate, _ETHLightClientStoreGetFinalizedHeader, _ETHLightClientStoreIsNextSyncCommitteeKnown, _ETHLightClientStoreGetOptimisticHeader, _ETHLightClientStoreGetSafetyThreshold, _ETHLightClientHeaderCreateCopy, _ETHLightClientHeaderDestroy, _ETHLightClientHeaderCopyBeaconRoot, _ETHLightClientHeaderGetBeacon, _ETHBeaconBlockHeaderGetSlot, _ETHBeaconBlockHeaderGetProposerIndex, _ETHBeaconBlockHeaderGetParentRoot, _ETHBeaconBlockHeaderGetStateRoot, _ETHBeaconBlockHeaderGetBodyRoot, _ETHLightClientHeaderCopyExecutionHash, _ETHLightClientHeaderGetExecution, _ETHExecutionPayloadHeaderGetParentHash, _ETHExecutionPayloadHeaderGetFeeRecipient, _ETHExecutionPayloadHeaderGetStateRoot, _ETHExecutionPayloadHeaderGetReceiptsRoot, _ETHExecutionPayloadHeaderGetLogsBloom, _ETHExecutionPayloadHeaderGetPrevRandao, _ETHExecutionPayloadHeaderGetBlockNumber, _ETHExecutionPayloadHeaderGetGasLimit, _ETHExecutionPayloadHeaderGetGasUsed, _ETHExecutionPayloadHeaderGetTimestamp, _ETHExecutionPayloadHeaderGetExtraDataBytes, _ETHExecutionPayloadHeaderGetBaseFeePerGas, _ETHExecutionPayloadHeaderGetBlobGasUsed, _ETHExecutionPayloadHeaderGetExcessBlobGas, _ETHExecutionBlockHeaderCreateFromJson, _ETHExecutionBlockHeaderDestroy, _ETHExecutionBlockHeaderGetTransactionsRoot, _ETHExecutionBlockHeaderGetWithdrawalsRoot, _ETHTransactionsCreateFromJson, _ETHTransactionsDestroy, _ETHTransactionsGetCount, _ETHTransactionsGet, _ETHTransactionGetHash, _ETHTransactionGetFrom, _ETHTransactionGetNonce, _ETHTransactionGetMaxPriorityFeePerGas, _ETHTransactionGetMaxFeePerGas, _ETHTransactionGetGas, _ETHTransactionIsCreatingContract, _ETHTransactionGetTo, _ETHTransactionGetValue, _ETHTransactionGetInputBytes, _ETHTransactionGetBytes, _ETHTransactionGetEip6493Root, _ETHTransactionGetEip6493Bytes, _ETHTransactionGetNumEip6493SnappyBytes, _ETHReceiptsCreateFromJson, _ETHReceiptsDestroy, _ETHReceiptsGet, _ETHReceiptHasStatus, _ETHReceiptGetBytes, _ETHReceiptGetEip6493Bytes, _ETHReceiptGetNumEip6493SnappyBytes]"' \ --passL:'-sEXPORTED_RUNTIME_METHODS="[lengthBytesUTF8, stringToNewUTF8]"' \ --passL:'-Wl,--no-entry' \ --noMain:on \ --passL:'-o libnimbus_lc.js' \ nimbus-eth2/beacon_chain/libnimbus_lc/libnimbus_lc.nim ```
2024-04-16 09:47:18 +00:00
addr tmp[]
avoid `--gc:arc` issue in validator key caching (#6203) The current implementation of the validator key cache as introduced in #5883 leads to issues when compiling with `--gc:arc`. Namely, the assert in `injectdestructors.nim` > `destructiveMoveVar` is triggered: ```nim assert n.kind != nkSym or not hasDestructor(c, n.sym.typ) ``` `cached == nkSym`, and `n.sym.typ == ref HashedValidatorPubKeyItem` with `hasDestructor(c, n.sym.typ) == true`. Inlining the `addr ...[]` avoids the problem and allows `--gc:arc` compilation as part of LC wasm demo project. Compilation command: ```sh nim c \ -d:disable_libbacktrace \ -d:disableMarchNative \ -d:disableLTO \ -d:emscripten \ -d:release \ -d:useGcAssert \ -d:useSysAssert \ --debuginfo:off \ --nimcache:nimcache \ --os:linux \ --cpu:wasm32 \ --cc:clang \ --clang.exe:emcc \ --clang.linkerexe:emcc \ --gc:arc \ --exceptions:goto \ --define:noSignalHandler \ --define:danger \ --panics:on \ --passC:-fpic \ --passL:-Os \ --passL:-fpic \ --passC:'-pthread' \ --passL:'-pthread' \ --passC:'-sASSERTIONS' \ --passL:'-sASSERTIONS' \ --passC:'-sINITIAL_MEMORY=256MB' \ --passL:'-sINITIAL_MEMORY=256MB' \ --passC:'-sSTACK_SIZE=128MB' \ --passL:'-sSTACK_SIZE=128MB' \ --passC:'-sUSE_PTHREADS=1' \ --passL:'-sUSE_PTHREADS=1' \ --passC:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sPTHREAD_POOL_SIZE_STRICT=0' \ --passL:'-sEXPORTED_FUNCTIONS="[_free, _malloc, _NimMain, _ETHRandomNumberCreate, _ETHConsensusConfigCreateFromYaml, _ETHConsensusConfigGetConsensusVersionAtEpoch, _ETHBeaconStateCreateFromSsz, _ETHBeaconStateDestroy, _ETHBeaconStateCopyGenesisValidatorsRoot, _ETHRootDestroy, _ETHForkDigestsCreateFromState, _ETHBeaconClockCreateFromState, _ETHBeaconClockGetSlot, _ETHLightClientStoreCreateFromBootstrap, _ETHLightClientStoreDestroy, _kETHLcSyncKind_UpdatesByRange, _kETHLcSyncKind_FinalityUpdate, _kETHLcSyncKind_OptimisticUpdate, _ETHLightClientStoreGetNextSyncTask, _ETHLightClientStoreGetMillisecondsToNextSyncTask, _ETHLightClientStoreProcessUpdatesByRange, _ETHLightClientStoreProcessFinalityUpdate, _ETHLightClientStoreProcessOptimisticUpdate, _ETHLightClientStoreGetFinalizedHeader, _ETHLightClientStoreIsNextSyncCommitteeKnown, _ETHLightClientStoreGetOptimisticHeader, _ETHLightClientStoreGetSafetyThreshold, _ETHLightClientHeaderCreateCopy, _ETHLightClientHeaderDestroy, _ETHLightClientHeaderCopyBeaconRoot, _ETHLightClientHeaderGetBeacon, _ETHBeaconBlockHeaderGetSlot, _ETHBeaconBlockHeaderGetProposerIndex, _ETHBeaconBlockHeaderGetParentRoot, _ETHBeaconBlockHeaderGetStateRoot, _ETHBeaconBlockHeaderGetBodyRoot, _ETHLightClientHeaderCopyExecutionHash, _ETHLightClientHeaderGetExecution, _ETHExecutionPayloadHeaderGetParentHash, _ETHExecutionPayloadHeaderGetFeeRecipient, _ETHExecutionPayloadHeaderGetStateRoot, _ETHExecutionPayloadHeaderGetReceiptsRoot, _ETHExecutionPayloadHeaderGetLogsBloom, _ETHExecutionPayloadHeaderGetPrevRandao, _ETHExecutionPayloadHeaderGetBlockNumber, _ETHExecutionPayloadHeaderGetGasLimit, _ETHExecutionPayloadHeaderGetGasUsed, _ETHExecutionPayloadHeaderGetTimestamp, _ETHExecutionPayloadHeaderGetExtraDataBytes, _ETHExecutionPayloadHeaderGetBaseFeePerGas, _ETHExecutionPayloadHeaderGetBlobGasUsed, _ETHExecutionPayloadHeaderGetExcessBlobGas, _ETHExecutionBlockHeaderCreateFromJson, _ETHExecutionBlockHeaderDestroy, _ETHExecutionBlockHeaderGetTransactionsRoot, _ETHExecutionBlockHeaderGetWithdrawalsRoot, _ETHTransactionsCreateFromJson, _ETHTransactionsDestroy, _ETHTransactionsGetCount, _ETHTransactionsGet, _ETHTransactionGetHash, _ETHTransactionGetFrom, _ETHTransactionGetNonce, _ETHTransactionGetMaxPriorityFeePerGas, _ETHTransactionGetMaxFeePerGas, _ETHTransactionGetGas, _ETHTransactionIsCreatingContract, _ETHTransactionGetTo, _ETHTransactionGetValue, _ETHTransactionGetInputBytes, _ETHTransactionGetBytes, _ETHTransactionGetEip6493Root, _ETHTransactionGetEip6493Bytes, _ETHTransactionGetNumEip6493SnappyBytes, _ETHReceiptsCreateFromJson, _ETHReceiptsDestroy, _ETHReceiptsGet, _ETHReceiptHasStatus, _ETHReceiptGetBytes, _ETHReceiptGetEip6493Bytes, _ETHReceiptGetNumEip6493SnappyBytes]"' \ --passL:'-sEXPORTED_RUNTIME_METHODS="[lengthBytesUTF8, stringToNewUTF8]"' \ --passL:'-Wl,--no-entry' \ --noMain:on \ --passL:'-o libnimbus_lc.js' \ nimbus-eth2/beacon_chain/libnimbus_lc/libnimbus_lc.nim ```
2024-04-16 09:47:18 +00:00
HashedValidatorPubKey(value: cached) # https://github.com/nim-lang/Nim/issues/23505