From 20af24efdb31ff657f4e76bc50cc186a4e66b0e8 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 13 Jun 2024 15:11:04 +0200 Subject: [PATCH 01/13] align field name across `SomeSig` (#6348) `ValidatorSig` uses `blob` but `TrustedSig` uses `data`, aligning the names reduces code duplication and improves clarity. It also simplifies `StableContainer` compatibility checks. --- beacon_chain/spec/crypto.nim | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index 6dfb98f93..9529229f9 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -77,7 +77,7 @@ type BlsResult*[T] = Result[T, cstring] TrustedSig* = object - data* {.align: 16.}: array[RawSigSize, byte] + blob* {.align: 16.}: array[RawSigSize, byte] SomeSig* = TrustedSig | ValidatorSig @@ -390,12 +390,9 @@ func toRaw*(x: ValidatorPrivKey): array[32, byte] = # TODO: distinct type - see https://github.com/status-im/nim-blscurve/pull/67 static: doAssert BLS_BACKEND == BLST result = SecretKey(x).exportRaw() - -template toRaw*(x: ValidatorPubKey | ValidatorSig): auto = - x.blob -template toRaw*(x: TrustedSig): auto = - x.data +template toRaw*(x: ValidatorPubKey | SomeSig): auto = + x.blob func toHex*(x: BlsCurveType): string = toHex(toRaw(x)) @@ -507,7 +504,7 @@ template fromSszBytes*(T: type[ValidatorPubKey | ValidatorSig], bytes: openArray # Logging # ---------------------------------------------------------------------- -func shortLog*(x: ValidatorPubKey | ValidatorSig): string = +func shortLog*(x: ValidatorPubKey | SomeSig): string = ## Logging for wrapped BLS types ## that may contain valid or non-validated data byteutils.toHex(x.blob.toOpenArray(0, 3)) @@ -520,9 +517,6 @@ func shortLog*(x: ValidatorPrivKey): string = ## Logging for raw unwrapped BLS types "" -func shortLog*(x: TrustedSig): string = - byteutils.toHex(x.data.toOpenArray(0, 3)) - # Initialization # ---------------------------------------------------------------------- From fb0494e7399b62d7120a8c0b06a854a9a52b8eec Mon Sep 17 00:00:00 2001 From: andri lim Date: Thu, 13 Jun 2024 20:25:59 +0700 Subject: [PATCH 02/13] Bump nim-eth for revert GasInt back to int64 (#6347) * Bump nim-eth for revert GasInt back to int64 * Fix libnimbus_lc --- beacon_chain/libnimbus_lc/libnimbus_lc.nim | 11 ++++++----- beacon_chain/spec/helpers.nim | 6 +++--- tests/testblockutil.nim | 6 ++++-- vendor/nim-eth | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/beacon_chain/libnimbus_lc/libnimbus_lc.nim b/beacon_chain/libnimbus_lc/libnimbus_lc.nim index c3aa4eac9..cb0c7876b 100644 --- a/beacon_chain/libnimbus_lc/libnimbus_lc.nim +++ b/beacon_chain/libnimbus_lc/libnimbus_lc.nim @@ -9,6 +9,7 @@ import std/[json, sequtils, times], + stew/saturation_arith, eth/common/[eth_types_rlp, transaction], eth/keys, eth/p2p/discoveryv5/random2, @@ -1257,8 +1258,8 @@ proc ETHExecutionBlockHeaderCreateFromJson( logsBloom: distinctBase(data.logsBloom), difficulty: data.difficulty, number: distinctBase(data.number), - gasLimit: distinctBase(data.gasLimit), - gasUsed: distinctBase(data.gasUsed), + gasLimit: GasInt.saturate distinctBase(data.gasLimit), + gasUsed: GasInt.saturate distinctBase(data.gasUsed), timestamp: EthTime(distinctBase(data.timestamp)), extraData: distinctBase(data.extraData), mixHash: data.mixHash.asEth2Digest, @@ -1613,9 +1614,9 @@ proc ETHTransactionsCreateFromJson( chainId: distinctBase(tx.chainId).u256, `from`: ExecutionAddress(data: fromAddress), nonce: tx.nonce, - maxPriorityFeePerGas: tx.maxPriorityFeePerGas, - maxFeePerGas: tx.maxFeePerGas, - gas: tx.gasLimit, + maxPriorityFeePerGas: tx.maxPriorityFeePerGas.uint64, + maxFeePerGas: tx.maxFeePerGas.uint64, + gas: tx.gasLimit.uint64, destinationType: destinationType, to: ExecutionAddress(data: toAddress), value: tx.value, diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index ee376329d..f0d2d0091 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -513,9 +513,9 @@ proc blockToBlockHeader*(blck: ForkyBeaconBlock): ExecutionBlockHeader = logsBloom : payload.logs_bloom.data, difficulty : default(DifficultyInt), number : payload.block_number, - gasLimit : cast[GasInt](payload.gas_limit), - gasUsed : cast[GasInt](payload.gas_used), - timestamp : EthTime(int64.saturate payload.timestamp), + gasLimit : GasInt.saturate(payload.gas_limit), + gasUsed : GasInt.saturate(payload.gas_used), + timestamp : EthTime(payload.timestamp), extraData : payload.extra_data.asSeq, mixHash : payload.prev_randao, # EIP-4399 `mixHash` -> `prevRandao` nonce : default(BlockNonce), diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim index e3a1dba83..7820fb557 100644 --- a/tests/testblockutil.nim +++ b/tests/testblockutil.nim @@ -118,6 +118,8 @@ proc build_empty_merge_execution_payload(state: bellatrix.BeaconState): bellatrix.ExecutionPayloadForSigning(executionPayload: payload, blockValue: Wei.zero) +from stew/saturating_arith import saturate + proc build_empty_execution_payload( state: bellatrix.BeaconState, feeRecipient: Eth1Address): bellatrix.ExecutionPayloadForSigning = @@ -127,8 +129,8 @@ proc build_empty_execution_payload( latest = state.latest_execution_payload_header timestamp = compute_timestamp_at_slot(state, state.slot) randao_mix = get_randao_mix(state, get_current_epoch(state)) - base_fee = calcEip1599BaseFee(latest.gas_limit, - latest.gas_used, + base_fee = calcEip1599BaseFee(GasInt.saturate latest.gas_limit, + GasInt.saturate latest.gas_used, latest.base_fee_per_gas) var payload = bellatrix.ExecutionPayloadForSigning( diff --git a/vendor/nim-eth b/vendor/nim-eth index 9b6497ed8..f169068df 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit 9b6497ed8a05ba25ee47142f3fc1f61742b51a6c +Subproject commit f169068df6c11a2aeba27584c60e354e19c42e94 From 524c136d8cd1a35acaa7a078f378e0c5cb7a86e9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 13 Jun 2024 17:22:48 +0200 Subject: [PATCH 03/13] bump `nim-libbacktrace` to `aab85b6d242df38706664373f089675235953ab8` (#6350) - bump `libbacktrace` to `11427f31a64b11583fec94b4c2a265c7dafb1ab3` - bump `whereami` to `a08044b0f9d7a18b9ddd15bfcb490e422fa1f6c4` --- vendor/nim-libbacktrace | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index cefd3eec9..aab85b6d2 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit cefd3eec9c13330f6726383ed0c8a60ca8823603 +Subproject commit aab85b6d242df38706664373f089675235953ab8 From 81e6fe5045f844ca89515e84b7494351f46220e7 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 13 Jun 2024 17:22:54 +0200 Subject: [PATCH 04/13] bump 'nim-serialization' to `005ee90cb6aa563cdd690910455ea05f916ead3f` (#6349) - Work around Nim bug that erases type `{.pragma.}` when making alias --- vendor/nim-serialization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-serialization b/vendor/nim-serialization index afae13ada..005ee90cb 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit afae13adac25b6fa98bacf4b9f38458dc64317b0 +Subproject commit 005ee90cb6aa563cdd690910455ea05f916ead3f From b6d6247fa10d5322935af2b43a831da4c8cba2b7 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 14 Jun 2024 02:45:40 +0200 Subject: [PATCH 05/13] fix light client topic subscriptions (#6351) Using `let contextFork = consensusFork` no longer seems to work to avoid capturing the `var` loop variable; it ends up being `Electra` for all handlers. Use `closureScope` as a more sustainable fix. --- beacon_chain/light_client.nim | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/beacon_chain/light_client.nim b/beacon_chain/light_client.nim index 746c3cb9a..a8ffe0a23 100644 --- a/beacon_chain/light_client.nim +++ b/beacon_chain/light_client.nim @@ -342,25 +342,26 @@ proc installMessageValidators*( for consensusFork in ConsensusFork: withLcDataFork(lcDataForkAtConsensusFork(consensusFork)): when lcDataFork > LightClientDataFork.None: - let - contextFork = consensusFork # Avoid capturing `Deneb` (Nim 1.6) - digest = forkDigests[].atConsensusFork(contextFork) + closureScope: + let + contextFork = consensusFork + digest = forkDigests[].atConsensusFork(contextFork) - # light_client_optimistic_update - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.5/specs/altair/light-client/p2p-interface.md#light_client_finality_update - lightClient.network.addValidator( - getLightClientFinalityUpdateTopic(digest), proc ( - msg: lcDataFork.LightClientFinalityUpdate - ): ValidationResult = - validate(msg, contextFork, processLightClientFinalityUpdate)) + # light_client_optimistic_update + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.5/specs/altair/light-client/p2p-interface.md#light_client_finality_update + lightClient.network.addValidator( + getLightClientFinalityUpdateTopic(digest), proc ( + msg: lcDataFork.LightClientFinalityUpdate + ): ValidationResult = + validate(msg, contextFork, processLightClientFinalityUpdate)) - # light_client_optimistic_update - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.5/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update - lightClient.network.addValidator( - getLightClientOptimisticUpdateTopic(digest), proc ( - msg: lcDataFork.LightClientOptimisticUpdate - ): ValidationResult = - validate(msg, contextFork, processLightClientOptimisticUpdate)) + # light_client_optimistic_update + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.5/specs/altair/light-client/p2p-interface.md#light_client_optimistic_update + lightClient.network.addValidator( + getLightClientOptimisticUpdateTopic(digest), proc ( + msg: lcDataFork.LightClientOptimisticUpdate + ): ValidationResult = + validate(msg, contextFork, processLightClientOptimisticUpdate)) proc updateGossipStatus*( lightClient: LightClient, slot: Slot, dagIsBehind = default(Option[bool])) = From c5326619a4d094db6f9e36c18992e0fa62fdc3d1 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 14 Jun 2024 03:46:03 +0300 Subject: [PATCH 06/13] bump nim-kzg4844 for gcc-14 compatibility (#6342) * Bump nim-kzg4844. * Fix code to use proper KZG types. * Add missing coma. * Fix compilation issue. * Bump nim-kzg4844. --- beacon_chain/el/el_manager.nim | 12 ++++--- .../gossip_processing/block_processor.nim | 4 +-- .../gossip_processing/gossip_validation.nim | 2 +- beacon_chain/spec/datatypes/deneb.nim | 2 +- .../eth2_apis/eth2_rest_serialization.nim | 4 +-- beacon_chain/spec/state_transition_block.nim | 2 +- beacon_chain/validators/message_router.nim | 6 ++-- .../validators/message_router_mev.nim | 2 +- .../test_fixture_fork_choice.nim | 5 +-- tests/consensus_spec/test_fixture_kzg.nim | 32 ++++++++++++------- tests/test_rest_json_serialization.nim | 4 +-- vendor/nim-kzg4844 | 2 +- 12 files changed, 47 insertions(+), 30 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index d2e754c87..03645eca7 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -491,9 +491,11 @@ func asConsensusType*(payload: engine_api.GetPayloadV3Response): # Both are defined as `array[N, byte]` under the hood. blobsBundle: deneb.BlobsBundle( commitments: KzgCommitments.init( - payload.blobsBundle.commitments.mapIt(it.bytes)), + payload.blobsBundle.commitments.mapIt( + kzg_abi.KzgCommitment(bytes: it.bytes))), proofs: KzgProofs.init( - payload.blobsBundle.proofs.mapIt(it.bytes)), + payload.blobsBundle.proofs.mapIt( + kzg_abi.KzgProof(bytes: it.bytes))), blobs: Blobs.init( payload.blobsBundle.blobs.mapIt(it.bytes)))) @@ -560,9 +562,11 @@ func asConsensusType*(payload: engine_api.GetPayloadV4Response): # Both are defined as `array[N, byte]` under the hood. blobsBundle: deneb.BlobsBundle( commitments: KzgCommitments.init( - payload.blobsBundle.commitments.mapIt(it.bytes)), + payload.blobsBundle.commitments.mapIt( + kzg_abi.KzgCommitment(bytes: it.bytes))), proofs: KzgProofs.init( - payload.blobsBundle.proofs.mapIt(it.bytes)), + payload.blobsBundle.proofs.mapIt( + kzg_abi.KzgProof(bytes: it.bytes))), blobs: Blobs.init( payload.blobsBundle.blobs.mapIt(it.bytes)))) diff --git a/beacon_chain/gossip_processing/block_processor.nim b/beacon_chain/gossip_processing/block_processor.nim index bfef52d31..a4db1924b 100644 --- a/beacon_chain/gossip_processing/block_processor.nim +++ b/beacon_chain/gossip_processing/block_processor.nim @@ -186,7 +186,7 @@ proc storeBackfillBlock( let blobs = blobsOpt.get() let kzgCommits = signedBlock.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let r = validate_blobs(kzgCommits, blobs.mapIt(it.blob), + let r = validate_blobs(kzgCommits, blobs.mapIt(KzgBlob(bytes: it.blob)), blobs.mapIt(it.kzg_proof)) if r.isErr(): debug "backfill blob validation failed", @@ -575,7 +575,7 @@ proc storeBlock( let blobs = blobsOpt.get() let kzgCommits = signedBlock.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let r = validate_blobs(kzgCommits, blobs.mapIt(it.blob), + let r = validate_blobs(kzgCommits, blobs.mapIt(KzgBlob(bytes: it.blob)), blobs.mapIt(it.kzg_proof)) if r.isErr(): debug "blob validation failed", diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 3f3a15c4c..22dbcfb5d 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -458,7 +458,7 @@ proc validateBlobSidecar*( # blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. block: let ok = verifyProof( - blob_sidecar.blob, + KzgBlob(bytes: blob_sidecar.blob), blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof).valueOr: return dag.checkedReject("BlobSidecar: blob verify failed") diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index 248ae5b99..bba69cc88 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -528,7 +528,7 @@ func initHashedBeaconState*(s: BeaconState): HashedBeaconState = HashedBeaconState(data: s) func shortLog*(v: KzgCommitment | KzgProof): auto = - to0xHex(v) + to0xHex(v.bytes) func shortLog*(v: Blob): auto = to0xHex(v.toOpenArray(0, 31)) diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index e9d5edac5..3bf5d117c 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -1358,7 +1358,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var (KzgCommitment|KzgProof)) {. raises: [IOError, SerializationError].} = try: - hexToByteArray(reader.readValue(string), distinctBase(value)) + hexToByteArray(reader.readValue(string), distinctBase(value.bytes)) except ValueError: raiseUnexpectedValue(reader, "KzgCommitment value should be a valid hex string") @@ -1366,7 +1366,7 @@ proc readValue*(reader: var JsonReader[RestJson], proc writeValue*( writer: var JsonWriter[RestJson], value: KzgCommitment | KzgProof ) {.raises: [IOError].} = - writeValue(writer, hexOriginal(distinctBase(value))) + writeValue(writer, hexOriginal(distinctBase(value.bytes))) ## GraffitiBytes proc writeValue*( diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 91947856f..61dbcd231 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -1076,7 +1076,7 @@ func kzg_commitment_to_versioned_hash*( var res: VersionedHash res[0] = VERSIONED_HASH_VERSION_KZG - res[1 .. 31] = eth2digest(kzg_commitment).data.toOpenArray(1, 31) + res[1 .. 31] = eth2digest(kzg_commitment.bytes).data.toOpenArray(1, 31) res proc validate_blobs*( diff --git a/beacon_chain/validators/message_router.nim b/beacon_chain/validators/message_router.nim index c70adb743..2ebd2e65a 100644 --- a/beacon_chain/validators/message_router.nim +++ b/beacon_chain/validators/message_router.nim @@ -117,8 +117,10 @@ proc routeSignedBeaconBlock*( let blobs = blobsOpt.get() let kzgCommits = blck.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let res = validate_blobs(kzgCommits, blobs.mapIt(it.blob), - blobs.mapIt(it.kzg_proof)) + let res = validate_blobs( + kzgCommits, + blobs.mapIt(KzgBlob(bytes: it.blob)), + blobs.mapIt(it.kzg_proof)) if res.isErr(): warn "blobs failed validation", blockRoot = shortLog(blck.root), diff --git a/beacon_chain/validators/message_router_mev.nim b/beacon_chain/validators/message_router_mev.nim index 76a2c44e3..8f24bfecd 100644 --- a/beacon_chain/validators/message_router_mev.nim +++ b/beacon_chain/validators/message_router_mev.nim @@ -128,7 +128,7 @@ proc unblindAndRouteBlockMEV*( bundle.data.blobs_bundle.commitments: return err("unblinded blobs bundle has unexpected commitments") let ok = verifyProofs( - asSeq blobs_bundle.blobs, + blobs_bundle.blobs.mapIt(KzgBlob(bytes: it)), asSeq blobs_bundle.commitments, asSeq blobs_bundle.proofs).valueOr: return err("unblinded blobs bundle fails verification") diff --git a/tests/consensus_spec/test_fixture_fork_choice.nim b/tests/consensus_spec/test_fixture_fork_choice.nim index 8334ef825..163996b8f 100644 --- a/tests/consensus_spec/test_fixture_fork_choice.nim +++ b/tests/consensus_spec/test_fixture_fork_choice.nim @@ -136,7 +136,8 @@ proc loadOps( blobs: distinctBase(parseTest( path/(step["blobs"].getStr()) & ".ssz_snappy", SSZ, List[KzgBlob, Limit MAX_BLOBS_PER_BLOCK])), - proofs: step["proofs"].mapIt(KzgProof.fromHex(it.getStr()))) + proofs: step["proofs"].mapIt( + KzgProof(bytes: fromHex(array[48, byte], it.getStr())))) else: Opt.none(BlobData) else: @@ -407,4 +408,4 @@ from ../../beacon_chain/conf import loadKzgTrustedSetup discard loadKzgTrustedSetup() # Required for Deneb tests fcSuite("ForkChoice", "fork_choice") -fcSuite("Sync", "sync") \ No newline at end of file +fcSuite("Sync", "sync") diff --git a/tests/consensus_spec/test_fixture_kzg.nim b/tests/consensus_spec/test_fixture_kzg.nim index 94a0610d9..8ce41b25c 100644 --- a/tests/consensus_spec/test_fixture_kzg.nim +++ b/tests/consensus_spec/test_fixture_kzg.nim @@ -50,12 +50,12 @@ proc runBlobToKzgCommitmentTest(suiteName, suitePath, path: string) = if blob.isNone: check output.kind == JNull else: - let commitment = blobToKzgCommitment(blob.get) + let commitment = blobToKzgCommitment(KzgBlob(bytes: blob.get)) check: if commitment.isErr: output.kind == JNull else: - commitment.get == fromHex[48](output.getStr).get + commitment.get().bytes == fromHex[48](output.getStr).get proc runVerifyKzgProofTest(suiteName, suitePath, path: string) = let relativePathComponent = path.relativeTestPathComponent(suitePath) @@ -75,7 +75,10 @@ proc runVerifyKzgProofTest(suiteName, suitePath, path: string) = if commitment.isNone or z.isNone or y.isNone or proof.isNone: check output.kind == JNull else: - let v = verifyProof(commitment.get, z.get, y.get, proof.get) + let v = verifyProof( + KzgCommitment(bytes: commitment.get), + KzgBytes32(bytes: z.get), KzgBytes32(bytes: y.get), + KzgBytes48(bytes: proof.get)) check: if v.isErr: output.kind == JNull @@ -100,7 +103,10 @@ proc runVerifyBlobKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or commitment.isNone or proof.isNone: check output.kind == JNull else: - let v = verifyBlobKzgProof(blob.get, commitment.get, proof.get) + let v = verifyBlobKzgProof( + KzgBlob(bytes: blob.get), + KzgBytes48(bytes: commitment.get), + KzgBytes48(bytes: proof.get)) check: if v.isErr: output.kind == JNull @@ -127,7 +133,9 @@ proc runVerifyBlobKzgProofBatchTest(suiteName, suitePath, path: string) = check output.kind == JNull else: let v = verifyBlobKzgProofBatch( - blobs.mapIt(it.get), commitments.mapIt(it.get), proofs.mapIt(it.get)) + blobs.mapIt(KzgBlob(bytes: it.get)), + commitments.mapIt(KzgCommitment(bytes: it.get)), + proofs.mapIt(KzgProof(bytes: it.get))) check: if v.isErr: output.kind == JNull @@ -150,7 +158,8 @@ proc runComputeKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or z.isNone: check output.kind == JNull else: - let p = computeKzgProof(blob.get, z.get) + let p = computeKzgProof( + KzgBlob(bytes: blob.get), KzgBytes32(bytes: z.get)) if p.isErr: check output.kind == JNull else: @@ -158,8 +167,8 @@ proc runComputeKzgProofTest(suiteName, suitePath, path: string) = proof = fromHex[48](output[0].getStr) y = fromHex[32](output[1].getStr) check: - p.get.proof == proof.get - p.get.y == y.get + p.get.proof.bytes == proof.get + p.get.y.bytes == y.get proc runComputeBlobKzgProofTest(suiteName, suitePath, path: string) = let relativePathComponent = path.relativeTestPathComponent(suitePath) @@ -177,11 +186,12 @@ proc runComputeBlobKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or commitment.isNone: check output.kind == JNull else: - let p = computeBlobKzgProof(blob.get, commitment.get) + let p = computeBlobKzgProof( + KzgBlob(bytes: blob.get), KzgBytes48(bytes: commitment.get)) if p.isErr: check output.kind == JNull else: - check p.get == fromHex[48](output.getStr).get + check p.get.bytes == fromHex[48](output.getStr).get from std/algorithm import sorted @@ -227,4 +237,4 @@ suite suiteName: for kind, path in walkDir(testsDir, relative = true, checkDir = true): runComputeBlobKzgProofTest(suiteName, testsDir, testsDir / path) -doAssert Kzg.freeTrustedSetup().isOk \ No newline at end of file +doAssert Kzg.freeTrustedSetup().isOk diff --git a/tests/test_rest_json_serialization.nim b/tests/test_rest_json_serialization.nim index ba0404f96..e3d2a7fe9 100644 --- a/tests/test_rest_json_serialization.nim +++ b/tests/test_rest_json_serialization.nim @@ -213,7 +213,7 @@ from stew/byteutils import hexToByteArray func fromHex(T: typedesc[KzgCommitment], s: string): T {. raises: [ValueError].} = var res: T - hexToByteArray(s, res) + hexToByteArray(s, res.bytes) res suite "REST JSON encoding and decoding": @@ -353,4 +353,4 @@ suite "REST JSON encoding and decoding": check: validator.pubkey == ValidatorPubKey.fromHex( "0x933ad9491b62059dd065b560d256d8957a8c402cc6e8d8ee7290ae11e8f7329267a8811c397529dac52ae1342ba58c95")[] - validator.exit_epoch == FAR_FUTURE_EPOCH \ No newline at end of file + validator.exit_epoch == FAR_FUTURE_EPOCH diff --git a/vendor/nim-kzg4844 b/vendor/nim-kzg4844 index f12616d06..2f5cee7be 160000 --- a/vendor/nim-kzg4844 +++ b/vendor/nim-kzg4844 @@ -1 +1 @@ -Subproject commit f12616d0675d9f6346141ca95f0840ab227eb213 +Subproject commit 2f5cee7bea0d62e2b502ff668f752bda7f3eb0c4 From 9fc870777c3c40481b320b1c1bf5af2efb1921dc Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 14 Jun 2024 03:23:17 +0200 Subject: [PATCH 07/13] remove Bellatrix EL syncing support from light client (#6352) Bellatrix light client data does not contain the EL block hash, so we had to follow blocks gossip to learn the EL `block_hash` of such blocks. Now that Bellatrix is obsolete, we can simplify EL syncing logic under light client scenarios. Bellatrix light client data can still be used to advance the light client sync itself, but will no longer result in `engine_forkchoiceUpdated` calls until the sync reaches Capella. This also frees up some memory as we no longer have to retain blocks. --- beacon_chain/beacon_node.nim | 2 + beacon_chain/beacon_node_light_client.nim | 93 +++++++++---------- .../optimistic_processor.nim | 93 +++---------------- beacon_chain/nimbus_beacon_node.nim | 8 +- beacon_chain/nimbus_light_client.nim | 54 +++++++---- docs/the_nimbus_book/src/el-light-client.md | 2 - 6 files changed, 100 insertions(+), 152 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 71c3ab6d4..a7a4f0f76 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -67,6 +67,8 @@ type config*: BeaconNodeConf attachedValidators*: ref ValidatorPool optimisticProcessor*: OptimisticProcessor + optimisticFcuFut*: Future[(PayloadExecutionStatus, Opt[BlockHash])] + .Raising([CancelledError]) lightClient*: LightClient dag*: ChainDAGRef quarantine*: ref Quarantine diff --git a/beacon_chain/beacon_node_light_client.nim b/beacon_chain/beacon_node_light_client.nim index b6e15b06b..6b2c47743 100644 --- a/beacon_chain/beacon_node_light_client.nim +++ b/beacon_chain/beacon_node_light_client.nim @@ -38,61 +38,17 @@ proc initLightClient*( # for broadcasting light client data as a server. let - optimisticHandler = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock): - Future[void] {.async: (raises: [CancelledError]).} = - debug "New LC optimistic block", - opt = signedBlock.toBlockId(), - dag = node.dag.head.bid, - wallSlot = node.currentSlot + optimisticHandler = proc( + signedBlock: ForkedSignedBeaconBlock + ): Future[void] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): when consensusFork >= ConsensusFork.Bellatrix: if forkyBlck.message.is_execution_block: - template blckPayload(): auto = - forkyBlck.message.body.execution_payload - - if not blckPayload.block_hash.isZero: - # engine_newPayloadV1 + template payload(): auto = forkyBlck.message.body.execution_payload + if not payload.block_hash.isZero: discard await node.elManager.newExecutionPayload( forkyBlck.message) - - # Retain optimistic head for other `forkchoiceUpdated` callers. - # May temporarily block `forkchoiceUpdatedV1` calls, e.g., Geth: - # - Refuses `newPayload`: "Ignoring payload while snap syncing" - # - Refuses `fcU`: "Forkchoice requested unknown head" - # Once DAG sync catches up or as new optimistic heads are fetched - # the situation recovers - node.consensusManager[].setOptimisticHead( - forkyBlck.toBlockId(), blckPayload.block_hash) - - # engine_forkchoiceUpdatedV1 or engine_forkchoiceUpdatedV2, - # depending on pre or post-Shapella - let beaconHead = node.attestationPool[].getBeaconHead(nil) - - template callForkchoiceUpdated(attributes: untyped) = - discard await node.elManager.forkchoiceUpdated( - headBlockHash = blckPayload.block_hash, - safeBlockHash = beaconHead.safeExecutionBlockHash, - finalizedBlockHash = beaconHead.finalizedExecutionBlockHash, - payloadAttributes = Opt.none attributes) - - case node.dag.cfg.consensusForkAtEpoch( - forkyBlck.message.slot.epoch) - of ConsensusFork.Deneb, ConsensusFork.Electra: - callForkchoiceUpdated(PayloadAttributesV3) - of ConsensusFork.Capella: - # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.3/src/engine/shanghai.md#specification-1 - # Consensus layer client MUST call this method instead of - # `engine_forkchoiceUpdatedV1` under any of the following - # conditions: - # `headBlockHash` references a block which `timestamp` is - # greater or equal to the Shanghai timestamp - callForkchoiceUpdated(PayloadAttributesV2) - of ConsensusFork.Bellatrix: - callForkchoiceUpdated(PayloadAttributesV1) - of ConsensusFork.Phase0, ConsensusFork.Altair: - discard else: discard - optimisticProcessor = initOptimisticProcessor( getBeaconTime, optimisticHandler) @@ -104,9 +60,46 @@ proc initLightClient*( proc onOptimisticHeader( lightClient: LightClient, optimisticHeader: ForkedLightClientHeader) = + if node.optimisticFcuFut != nil: + return withForkyHeader(optimisticHeader): when lcDataFork > LightClientDataFork.None: - optimisticProcessor.setOptimisticHeader(forkyHeader.beacon) + let bid = forkyHeader.beacon.toBlockId() + logScope: + opt = bid + dag = node.dag.head.bid + wallSlot = node.currentSlot + when lcDataFork >= LightClientDataFork.Capella: + let + consensusFork = node.dag.cfg.consensusForkAtEpoch(bid.slot.epoch) + blockHash = forkyHeader.execution.block_hash + + # Retain optimistic head for other `forkchoiceUpdated` callers. + # May temporarily block `forkchoiceUpdated` calls, e.g., Geth: + # - Refuses `newPayload`: "Ignoring payload while snap syncing" + # - Refuses `fcU`: "Forkchoice requested unknown head" + # Once DAG sync catches up or as new optimistic heads are fetched + # the situation recovers + debug "New LC optimistic header" + node.consensusManager[].setOptimisticHead(bid, blockHash) + if not node.consensusManager[] + .shouldSyncOptimistically(node.currentSlot): + return + + # engine_forkchoiceUpdated + let beaconHead = node.attestationPool[].getBeaconHead(nil) + withConsensusFork(consensusFork): + when lcDataForkAtConsensusFork(consensusFork) == lcDataFork: + node.optimisticFcuFut = node.elManager.forkchoiceUpdated( + headBlockHash = blockHash, + safeBlockHash = beaconHead.safeExecutionBlockHash, + finalizedBlockHash = beaconHead.finalizedExecutionBlockHash, + payloadAttributes = Opt.none consensusFork.PayloadAttributes) + node.optimisticFcuFut.addCallback do (future: pointer): + node.optimisticFcuFut = nil + else: + # The execution block hash is only available from Capella onward + info "Ignoring new LC optimistic header until Capella" lightClient.onOptimisticHeader = onOptimisticHeader lightClient.trustedBlockRoot = config.trustedBlockRoot diff --git a/beacon_chain/gossip_processing/optimistic_processor.nim b/beacon_chain/gossip_processing/optimistic_processor.nim index 684171651..15fd12725 100644 --- a/beacon_chain/gossip_processing/optimistic_processor.nim +++ b/beacon_chain/gossip_processing/optimistic_processor.nim @@ -20,29 +20,19 @@ export gossip_validation logScope: topics = "gossip_opt" -const - # Maximum `blocks` to cache (not validated; deleted on new optimistic header) - maxBlocks = 16 # <= `GOSSIP_MAX_SIZE_BELLATRIX` (10 MB) each - - # Minimum interval at which spam is logged - minLogInterval = chronos.seconds(5) - type - MsgTrustedBlockProcessor* = - proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock): Future[void] {. - async: (raises: [CancelledError]).} + OptimisticBlockVerifier* = proc( + signedBlock: ForkedSignedBeaconBlock + ): Future[void] {.async: (raises: [CancelledError]).} OptimisticProcessor* = ref object getBeaconTime: GetBeaconTimeFn - optimisticVerifier: MsgTrustedBlockProcessor - blocks: Table[Eth2Digest, ref ForkedSignedBeaconBlock] - latestOptimisticSlot: Slot + optimisticVerifier: OptimisticBlockVerifier processFut: Future[void].Raising([CancelledError]) - logMoment: Moment proc initOptimisticProcessor*( getBeaconTime: GetBeaconTimeFn, - optimisticVerifier: MsgTrustedBlockProcessor): OptimisticProcessor = + optimisticVerifier: OptimisticBlockVerifier): OptimisticProcessor = OptimisticProcessor( getBeaconTime: getBeaconTime, optimisticVerifier: optimisticVerifier) @@ -56,9 +46,6 @@ proc validateBeaconBlock( (wallTime + MAXIMUM_GOSSIP_CLOCK_DISPARITY).slotOrZero): return errIgnore("BeaconBlock: slot too high") - if signed_beacon_block.message.slot <= self.latestOptimisticSlot: - return errIgnore("BeaconBlock: no significant progress") - if not signed_beacon_block.message.is_execution_block(): return errIgnore("BeaconBlock: no execution block") @@ -93,32 +80,16 @@ proc processSignedBeaconBlock*( debug "Dropping optimistic block", error = v.error return err(v.error) - # Note that validation of blocks is delayed by ~4/3 slots because we have to - # wait for the sync committee to sign the correct block and for that signature - # to be included in the next block. Therefore, we skip block validation here - # and cache the block in memory. Because there is no validation, we have to - # mitigate against bogus blocks, mostly by bounding the caches. Assuming that - # any denial-of-service attacks eventually subside, care is taken to recover. - template logWithSpamProtection(body: untyped): untyped = - block: - let now = Moment.now() - if self.logMoment + minLogInterval <= now: - logScope: minLogInterval - body - self.logMoment = now + # Only process one block at a time (backpressure) + trace "Optimistic block validated" + if self.processFut == nil: + self.processFut = self.optimisticVerifier( + ForkedSignedBeaconBlock.init(signedBlock)) - # Store block for later verification - if not self.blocks.hasKey(signedBlock.root): - # If `blocks` is full, we got spammed with multiple blocks for a slot, - # of the optimistic header advancements have been all withheld from us. - # Whenever the optimistic header advances, old blocks are cleared, - # so we can simply ignore additional spam blocks until that happens. - if self.blocks.len >= maxBlocks: - logWithSpamProtection: - error "`blocks` full - ignoring", maxBlocks - else: - self.blocks[signedBlock.root] = - newClone(ForkedSignedBeaconBlock.init(signedBlock)) + proc handleFinishedProcess(future: pointer) = + self.processFut = nil + + self.processFut.addCallback(handleFinishedProcess) # Block validation is delegated to the sync committee and is done with delay. # If we forward invalid spam blocks, we may be disconnected + IP banned, @@ -127,40 +98,4 @@ proc processSignedBeaconBlock*( # However, we are actively contributing to other topics, so some of the # negative peer score may be offset through those different topics. # The practical impact depends on the actually deployed scoring heuristics. - trace "Optimistic block cached" return errIgnore("Validation delegated to sync committee") - -proc setOptimisticHeader*( - self: OptimisticProcessor, optimisticHeader: BeaconBlockHeader) = - # If irrelevant, skip processing - if optimisticHeader.slot <= self.latestOptimisticSlot: - return - self.latestOptimisticSlot = optimisticHeader.slot - - # Delete blocks that are no longer of interest - let blockRoot = optimisticHeader.hash_tree_root() - var - rootsToDelete: seq[Eth2Digest] - signedBlock: ref ForkedMsgTrustedSignedBeaconBlock - for root, blck in self.blocks: - if root == blockRoot: - signedBlock = blck.asMsgTrusted() - if blck[].slot <= optimisticHeader.slot: - rootsToDelete.add root - for root in rootsToDelete: - self.blocks.del root - - # Block must be known - if signedBlock == nil: - return - - # If a block is already being processed, skip (backpressure) - if self.processFut != nil: - return - - self.processFut = self.optimisticVerifier(signedBlock[]) - - proc handleFinishedProcess(future: pointer) = - self.processFut = nil - - self.processFut.addCallback(handleFinishedProcess) diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 0079690dc..69d089833 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1619,7 +1619,7 @@ func syncStatus(node: BeaconNode, wallSlot: Slot): string = node.syncManager.syncStatus & optimisticSuffix & lightClientSuffix elif node.backfiller.inProgress: "backfill: " & node.backfiller.syncStatus - elif optimistic_head: + elif optimisticHead: "synced/opt" else: "synced" @@ -1768,7 +1768,8 @@ proc installMessageValidators(node: BeaconNode) = node.network.addAsyncValidator( getAttestationTopic(digest, subnet_id), proc ( attestation: electra.Attestation - ): Future[ValidationResult] {.async: (raises: [CancelledError]).} = + ): Future[ValidationResult] {. + async: (raises: [CancelledError]).} = return toValidationResult( await node.processor.processAttestation( MsgSource.gossip, attestation, subnet_id, @@ -1780,7 +1781,8 @@ proc installMessageValidators(node: BeaconNode) = node.network.addAsyncValidator( getAttestationTopic(digest, subnet_id), proc ( attestation: phase0.Attestation - ): Future[ValidationResult] {.async: (raises: [CancelledError]).} = + ): Future[ValidationResult] {. + async: (raises: [CancelledError]).} = return toValidationResult( await node.processor.processAttestation( MsgSource.gossip, attestation, subnet_id, diff --git a/beacon_chain/nimbus_light_client.nim b/beacon_chain/nimbus_light_client.nim index 2f78df875..519b1e89d 100644 --- a/beacon_chain/nimbus_light_client.nim +++ b/beacon_chain/nimbus_light_client.nim @@ -107,23 +107,15 @@ programMain: else: nil - optimisticHandler = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock): - Future[void] {.async: (raises: [CancelledError]).} = - notice "New LC optimistic block", - opt = signedBlock.toBlockId(), - wallSlot = getBeaconTime().slotOrZero + optimisticHandler = proc( + signedBlock: ForkedSignedBeaconBlock + ): Future[void] {.async: (raises: [CancelledError]).} = withBlck(signedBlock): when consensusFork >= ConsensusFork.Bellatrix: if forkyBlck.message.is_execution_block: template payload(): auto = forkyBlck.message.body.execution_payload - if elManager != nil and not payload.block_hash.isZero: discard await elManager.newExecutionPayload(forkyBlck.message) - discard await elManager.forkchoiceUpdated( - headBlockHash = payload.block_hash, - safeBlockHash = payload.block_hash, # stub value - finalizedBlockHash = ZERO_HASH, - payloadAttributes = Opt.none(consensusFork.PayloadAttributes)) else: discard optimisticProcessor = initOptimisticProcessor( getBeaconTime, optimisticHandler) @@ -153,26 +145,54 @@ programMain: waitFor network.startListening() waitFor network.start() + func isSynced(optimisticSlot: Slot, wallSlot: Slot): bool = + # Check whether light client has synced sufficiently close to wall slot + const maxAge = 2 * SLOTS_PER_EPOCH + optimisticSlot >= max(wallSlot, maxAge.Slot) - maxAge + proc onFinalizedHeader( lightClient: LightClient, finalizedHeader: ForkedLightClientHeader) = withForkyHeader(finalizedHeader): when lcDataFork > LightClientDataFork.None: info "New LC finalized header", finalized_header = shortLog(forkyHeader) - let period = forkyHeader.beacon.slot.sync_committee_period syncCommittee = lightClient.finalizedSyncCommittee.expect("Init OK") db.putSyncCommittee(period, syncCommittee) db.putLatestFinalizedHeader(finalizedHeader) + var optimisticFcuFut: Future[(PayloadExecutionStatus, Opt[BlockHash])] + .Raising([CancelledError]) proc onOptimisticHeader( lightClient: LightClient, optimisticHeader: ForkedLightClientHeader) = + if optimisticFcuFut != nil: + return withForkyHeader(optimisticHeader): when lcDataFork > LightClientDataFork.None: - info "New LC optimistic header", - optimistic_header = shortLog(forkyHeader) - optimisticProcessor.setOptimisticHeader(forkyHeader.beacon) + logScope: optimistic_header = shortLog(forkyHeader) + when lcDataFork >= LightClientDataFork.Capella: + let + bid = forkyHeader.beacon.toBlockId() + consensusFork = cfg.consensusForkAtEpoch(bid.slot.epoch) + blockHash = forkyHeader.execution.block_hash + + info "New LC optimistic header" + if elManager == nil or blockHash.isZero or + not isSynced(bid.slot, getBeaconTime().slotOrZero()): + return + + withConsensusFork(consensusFork): + when lcDataForkAtConsensusFork(consensusFork) == lcDataFork: + optimisticFcuFut = elManager.forkchoiceUpdated( + headBlockHash = blockHash, + safeBlockHash = blockHash, # stub value + finalizedBlockHash = ZERO_HASH, + payloadAttributes = Opt.none(consensusFork.PayloadAttributes)) + optimisticFcuFut.addCallback do (future: pointer): + optimisticFcuFut = nil + else: + info "Ignoring new LC optimistic header until Capella" lightClient.onFinalizedHeader = onFinalizedHeader lightClient.onOptimisticHeader = onOptimisticHeader @@ -204,9 +224,7 @@ programMain: let optimisticHeader = lightClient.optimisticHeader withForkyHeader(optimisticHeader): when lcDataFork > LightClientDataFork.None: - # Check whether light client has synced sufficiently close to wall slot - const maxAge = 2 * SLOTS_PER_EPOCH - forkyHeader.beacon.slot >= max(wallSlot, maxAge.Slot) - maxAge + isSynced(forkyHeader.beacon.slot, wallSlot) else: false diff --git a/docs/the_nimbus_book/src/el-light-client.md b/docs/the_nimbus_book/src/el-light-client.md index 31569c045..eb29ebede 100644 --- a/docs/the_nimbus_book/src/el-light-client.md +++ b/docs/the_nimbus_book/src/el-light-client.md @@ -178,13 +178,11 @@ INF 2022-11-21 18:03:27.984+01:00 New LC optimistic header opt WRN 2022-11-21 18:03:31.419+01:00 Peer count low, no new peers discovered topics="networking" discovered_nodes=0 new_peers=@[] current_peers=7 wanted_peers=160 INF 2022-11-21 18:03:36.001+01:00 Slot start slot=1109718 epoch=34678 sync=synced peers=7 head=c5464508:1109716 finalized=c092a1d1:1109216 delay=1ms98us INF 2022-11-21 18:03:40.012+01:00 New LC optimistic header optimistic_header="(beacon: (slot: 1109717, proposer_index: 835, parent_root: \"c5464508\", state_root: \"13f823f8\"))" -NTC 2022-11-21 18:03:40.012+01:00 New LC optimistic block opt=99ab28aa:1109717 wallSlot=1109718 WRN 2022-11-21 18:03:40.422+01:00 Peer count low, no new peers discovered topics="networking" discovered_nodes=1 new_peers=@[] current_peers=7 wanted_peers=160 INF 2022-11-21 18:03:48.001+01:00 Slot start slot=1109719 epoch=34678 sync=synced peers=7 head=99ab28aa:1109717 finalized=c092a1d1:1109216 delay=1ms53us WRN 2022-11-21 18:03:50.205+01:00 Peer count low, no new peers discovered topics="networking" discovered_nodes=0 new_peers=@[] current_peers=7 wanted_peers=160 INF 2022-11-21 18:04:00.001+01:00 Slot start slot=1109720 epoch=34678 sync=synced peers=7 head=99ab28aa:1109717 finalized=c092a1d1:1109216 delay=1ms145us INF 2022-11-21 18:04:03.982+01:00 New LC optimistic header optimistic_header="(beacon: (slot: 1109718, proposer_index: 1202, parent_root: \"99ab28aa\", state_root: \"7f7f88d2\"))" -NTC 2022-11-21 18:04:03.982+01:00 New LC optimistic block opt=ab007266:1109718 wallSlot=1109720 ``` !!! note From 50894ad0d5f462789308f1a54fd80cdcd5ebb584 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 14 Jun 2024 04:25:42 +0000 Subject: [PATCH 08/13] add Grandine to known libp2p agents --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 06a2d6974..76f5fb949 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ DEPOSITS_DELAY := 0 #- "--define:release" cannot be added to "config.nims" #- disable Nim's default parallelisation because it starts too many processes for too little gain #- https://github.com/status-im/nim-libp2p#use-identify-metrics -NIM_PARAMS := -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku $(NIM_PARAMS) +NIM_PARAMS := -d:release --parallelBuild:1 -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku,grandine $(NIM_PARAMS) ifeq ($(USE_LIBBACKTRACE), 0) NIM_PARAMS += -d:disable_libbacktrace From 05d2271a9fdcfaf690e065597b676316748c64f3 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 14 Jun 2024 17:06:58 +0000 Subject: [PATCH 09/13] Revert "Revert "bump nim-bearssl for gcc-14 compatibility (#6337)" (#6346)" (#6359) This reverts commit 3d65bdcb09e4ad6b5039dfc7beecae20f6924c0d. --- tests/test_sync_manager.nim | 2 +- vendor/nim-bearssl | 2 +- vendor/nim-chronos | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_sync_manager.nim b/tests/test_sync_manager.nim index 65fa75cd0..5d7fa83cf 100644 --- a/tests/test_sync_manager.nim +++ b/tests/test_sync_manager.nim @@ -884,7 +884,7 @@ suite "SyncManager test suite": # Push a single request that will fail with all blocks being unviable var f11 = queue.push(r11, chain.getSlice(startSlot, r11), Opt.none(seq[BlobSidecars])) - discard await f11.withTimeout(100.milliseconds) + discard await f11.withTimeout(1.seconds) check: f11.finished == true diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl index d81b37dc2..a806cbfab 160000 --- a/vendor/nim-bearssl +++ b/vendor/nim-bearssl @@ -1 +1 @@ -Subproject commit d81b37dc2011bf3a2bd93500489877c2ce8e6ac3 +Subproject commit a806cbfab5fe8de49c76139f8705fff79daf99ee diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 8a306763c..1b9d9253e 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 8a306763cec8105fa83574b56734b0f66823f844 +Subproject commit 1b9d9253e89445d585d0fff39cc0d19254fdfd0d From a93ab7cf3bfd8195a6b26bec2e2166fe2d668c3c Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 14 Jun 2024 23:23:19 +0000 Subject: [PATCH 10/13] Revert "bump nim-kzg4844 for gcc-14 compatibility (#6342)" (#6360) This reverts commit c5326619a4d094db6f9e36c18992e0fa62fdc3d1. --- beacon_chain/el/el_manager.nim | 12 +++---- .../gossip_processing/block_processor.nim | 4 +-- .../gossip_processing/gossip_validation.nim | 2 +- beacon_chain/spec/datatypes/deneb.nim | 2 +- .../eth2_apis/eth2_rest_serialization.nim | 4 +-- beacon_chain/spec/state_transition_block.nim | 2 +- beacon_chain/validators/message_router.nim | 6 ++-- .../validators/message_router_mev.nim | 2 +- .../test_fixture_fork_choice.nim | 5 ++- tests/consensus_spec/test_fixture_kzg.nim | 32 +++++++------------ tests/test_rest_json_serialization.nim | 4 +-- vendor/nim-kzg4844 | 2 +- 12 files changed, 30 insertions(+), 47 deletions(-) diff --git a/beacon_chain/el/el_manager.nim b/beacon_chain/el/el_manager.nim index 03645eca7..d2e754c87 100644 --- a/beacon_chain/el/el_manager.nim +++ b/beacon_chain/el/el_manager.nim @@ -491,11 +491,9 @@ func asConsensusType*(payload: engine_api.GetPayloadV3Response): # Both are defined as `array[N, byte]` under the hood. blobsBundle: deneb.BlobsBundle( commitments: KzgCommitments.init( - payload.blobsBundle.commitments.mapIt( - kzg_abi.KzgCommitment(bytes: it.bytes))), + payload.blobsBundle.commitments.mapIt(it.bytes)), proofs: KzgProofs.init( - payload.blobsBundle.proofs.mapIt( - kzg_abi.KzgProof(bytes: it.bytes))), + payload.blobsBundle.proofs.mapIt(it.bytes)), blobs: Blobs.init( payload.blobsBundle.blobs.mapIt(it.bytes)))) @@ -562,11 +560,9 @@ func asConsensusType*(payload: engine_api.GetPayloadV4Response): # Both are defined as `array[N, byte]` under the hood. blobsBundle: deneb.BlobsBundle( commitments: KzgCommitments.init( - payload.blobsBundle.commitments.mapIt( - kzg_abi.KzgCommitment(bytes: it.bytes))), + payload.blobsBundle.commitments.mapIt(it.bytes)), proofs: KzgProofs.init( - payload.blobsBundle.proofs.mapIt( - kzg_abi.KzgProof(bytes: it.bytes))), + payload.blobsBundle.proofs.mapIt(it.bytes)), blobs: Blobs.init( payload.blobsBundle.blobs.mapIt(it.bytes)))) diff --git a/beacon_chain/gossip_processing/block_processor.nim b/beacon_chain/gossip_processing/block_processor.nim index a4db1924b..bfef52d31 100644 --- a/beacon_chain/gossip_processing/block_processor.nim +++ b/beacon_chain/gossip_processing/block_processor.nim @@ -186,7 +186,7 @@ proc storeBackfillBlock( let blobs = blobsOpt.get() let kzgCommits = signedBlock.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let r = validate_blobs(kzgCommits, blobs.mapIt(KzgBlob(bytes: it.blob)), + let r = validate_blobs(kzgCommits, blobs.mapIt(it.blob), blobs.mapIt(it.kzg_proof)) if r.isErr(): debug "backfill blob validation failed", @@ -575,7 +575,7 @@ proc storeBlock( let blobs = blobsOpt.get() let kzgCommits = signedBlock.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let r = validate_blobs(kzgCommits, blobs.mapIt(KzgBlob(bytes: it.blob)), + let r = validate_blobs(kzgCommits, blobs.mapIt(it.blob), blobs.mapIt(it.kzg_proof)) if r.isErr(): debug "blob validation failed", diff --git a/beacon_chain/gossip_processing/gossip_validation.nim b/beacon_chain/gossip_processing/gossip_validation.nim index 22dbcfb5d..3f3a15c4c 100644 --- a/beacon_chain/gossip_processing/gossip_validation.nim +++ b/beacon_chain/gossip_processing/gossip_validation.nim @@ -458,7 +458,7 @@ proc validateBlobSidecar*( # blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`. block: let ok = verifyProof( - KzgBlob(bytes: blob_sidecar.blob), + blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof).valueOr: return dag.checkedReject("BlobSidecar: blob verify failed") diff --git a/beacon_chain/spec/datatypes/deneb.nim b/beacon_chain/spec/datatypes/deneb.nim index bba69cc88..248ae5b99 100644 --- a/beacon_chain/spec/datatypes/deneb.nim +++ b/beacon_chain/spec/datatypes/deneb.nim @@ -528,7 +528,7 @@ func initHashedBeaconState*(s: BeaconState): HashedBeaconState = HashedBeaconState(data: s) func shortLog*(v: KzgCommitment | KzgProof): auto = - to0xHex(v.bytes) + to0xHex(v) func shortLog*(v: Blob): auto = to0xHex(v.toOpenArray(0, 31)) diff --git a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim index 3bf5d117c..e9d5edac5 100644 --- a/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim @@ -1358,7 +1358,7 @@ proc readValue*(reader: var JsonReader[RestJson], value: var (KzgCommitment|KzgProof)) {. raises: [IOError, SerializationError].} = try: - hexToByteArray(reader.readValue(string), distinctBase(value.bytes)) + hexToByteArray(reader.readValue(string), distinctBase(value)) except ValueError: raiseUnexpectedValue(reader, "KzgCommitment value should be a valid hex string") @@ -1366,7 +1366,7 @@ proc readValue*(reader: var JsonReader[RestJson], proc writeValue*( writer: var JsonWriter[RestJson], value: KzgCommitment | KzgProof ) {.raises: [IOError].} = - writeValue(writer, hexOriginal(distinctBase(value.bytes))) + writeValue(writer, hexOriginal(distinctBase(value))) ## GraffitiBytes proc writeValue*( diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 61dbcd231..91947856f 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -1076,7 +1076,7 @@ func kzg_commitment_to_versioned_hash*( var res: VersionedHash res[0] = VERSIONED_HASH_VERSION_KZG - res[1 .. 31] = eth2digest(kzg_commitment.bytes).data.toOpenArray(1, 31) + res[1 .. 31] = eth2digest(kzg_commitment).data.toOpenArray(1, 31) res proc validate_blobs*( diff --git a/beacon_chain/validators/message_router.nim b/beacon_chain/validators/message_router.nim index 2ebd2e65a..c70adb743 100644 --- a/beacon_chain/validators/message_router.nim +++ b/beacon_chain/validators/message_router.nim @@ -117,10 +117,8 @@ proc routeSignedBeaconBlock*( let blobs = blobsOpt.get() let kzgCommits = blck.message.body.blob_kzg_commitments.asSeq if blobs.len > 0 or kzgCommits.len > 0: - let res = validate_blobs( - kzgCommits, - blobs.mapIt(KzgBlob(bytes: it.blob)), - blobs.mapIt(it.kzg_proof)) + let res = validate_blobs(kzgCommits, blobs.mapIt(it.blob), + blobs.mapIt(it.kzg_proof)) if res.isErr(): warn "blobs failed validation", blockRoot = shortLog(blck.root), diff --git a/beacon_chain/validators/message_router_mev.nim b/beacon_chain/validators/message_router_mev.nim index 8f24bfecd..76a2c44e3 100644 --- a/beacon_chain/validators/message_router_mev.nim +++ b/beacon_chain/validators/message_router_mev.nim @@ -128,7 +128,7 @@ proc unblindAndRouteBlockMEV*( bundle.data.blobs_bundle.commitments: return err("unblinded blobs bundle has unexpected commitments") let ok = verifyProofs( - blobs_bundle.blobs.mapIt(KzgBlob(bytes: it)), + asSeq blobs_bundle.blobs, asSeq blobs_bundle.commitments, asSeq blobs_bundle.proofs).valueOr: return err("unblinded blobs bundle fails verification") diff --git a/tests/consensus_spec/test_fixture_fork_choice.nim b/tests/consensus_spec/test_fixture_fork_choice.nim index 163996b8f..8334ef825 100644 --- a/tests/consensus_spec/test_fixture_fork_choice.nim +++ b/tests/consensus_spec/test_fixture_fork_choice.nim @@ -136,8 +136,7 @@ proc loadOps( blobs: distinctBase(parseTest( path/(step["blobs"].getStr()) & ".ssz_snappy", SSZ, List[KzgBlob, Limit MAX_BLOBS_PER_BLOCK])), - proofs: step["proofs"].mapIt( - KzgProof(bytes: fromHex(array[48, byte], it.getStr())))) + proofs: step["proofs"].mapIt(KzgProof.fromHex(it.getStr()))) else: Opt.none(BlobData) else: @@ -408,4 +407,4 @@ from ../../beacon_chain/conf import loadKzgTrustedSetup discard loadKzgTrustedSetup() # Required for Deneb tests fcSuite("ForkChoice", "fork_choice") -fcSuite("Sync", "sync") +fcSuite("Sync", "sync") \ No newline at end of file diff --git a/tests/consensus_spec/test_fixture_kzg.nim b/tests/consensus_spec/test_fixture_kzg.nim index 8ce41b25c..94a0610d9 100644 --- a/tests/consensus_spec/test_fixture_kzg.nim +++ b/tests/consensus_spec/test_fixture_kzg.nim @@ -50,12 +50,12 @@ proc runBlobToKzgCommitmentTest(suiteName, suitePath, path: string) = if blob.isNone: check output.kind == JNull else: - let commitment = blobToKzgCommitment(KzgBlob(bytes: blob.get)) + let commitment = blobToKzgCommitment(blob.get) check: if commitment.isErr: output.kind == JNull else: - commitment.get().bytes == fromHex[48](output.getStr).get + commitment.get == fromHex[48](output.getStr).get proc runVerifyKzgProofTest(suiteName, suitePath, path: string) = let relativePathComponent = path.relativeTestPathComponent(suitePath) @@ -75,10 +75,7 @@ proc runVerifyKzgProofTest(suiteName, suitePath, path: string) = if commitment.isNone or z.isNone or y.isNone or proof.isNone: check output.kind == JNull else: - let v = verifyProof( - KzgCommitment(bytes: commitment.get), - KzgBytes32(bytes: z.get), KzgBytes32(bytes: y.get), - KzgBytes48(bytes: proof.get)) + let v = verifyProof(commitment.get, z.get, y.get, proof.get) check: if v.isErr: output.kind == JNull @@ -103,10 +100,7 @@ proc runVerifyBlobKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or commitment.isNone or proof.isNone: check output.kind == JNull else: - let v = verifyBlobKzgProof( - KzgBlob(bytes: blob.get), - KzgBytes48(bytes: commitment.get), - KzgBytes48(bytes: proof.get)) + let v = verifyBlobKzgProof(blob.get, commitment.get, proof.get) check: if v.isErr: output.kind == JNull @@ -133,9 +127,7 @@ proc runVerifyBlobKzgProofBatchTest(suiteName, suitePath, path: string) = check output.kind == JNull else: let v = verifyBlobKzgProofBatch( - blobs.mapIt(KzgBlob(bytes: it.get)), - commitments.mapIt(KzgCommitment(bytes: it.get)), - proofs.mapIt(KzgProof(bytes: it.get))) + blobs.mapIt(it.get), commitments.mapIt(it.get), proofs.mapIt(it.get)) check: if v.isErr: output.kind == JNull @@ -158,8 +150,7 @@ proc runComputeKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or z.isNone: check output.kind == JNull else: - let p = computeKzgProof( - KzgBlob(bytes: blob.get), KzgBytes32(bytes: z.get)) + let p = computeKzgProof(blob.get, z.get) if p.isErr: check output.kind == JNull else: @@ -167,8 +158,8 @@ proc runComputeKzgProofTest(suiteName, suitePath, path: string) = proof = fromHex[48](output[0].getStr) y = fromHex[32](output[1].getStr) check: - p.get.proof.bytes == proof.get - p.get.y.bytes == y.get + p.get.proof == proof.get + p.get.y == y.get proc runComputeBlobKzgProofTest(suiteName, suitePath, path: string) = let relativePathComponent = path.relativeTestPathComponent(suitePath) @@ -186,12 +177,11 @@ proc runComputeBlobKzgProofTest(suiteName, suitePath, path: string) = if blob.isNone or commitment.isNone: check output.kind == JNull else: - let p = computeBlobKzgProof( - KzgBlob(bytes: blob.get), KzgBytes48(bytes: commitment.get)) + let p = computeBlobKzgProof(blob.get, commitment.get) if p.isErr: check output.kind == JNull else: - check p.get.bytes == fromHex[48](output.getStr).get + check p.get == fromHex[48](output.getStr).get from std/algorithm import sorted @@ -237,4 +227,4 @@ suite suiteName: for kind, path in walkDir(testsDir, relative = true, checkDir = true): runComputeBlobKzgProofTest(suiteName, testsDir, testsDir / path) -doAssert Kzg.freeTrustedSetup().isOk +doAssert Kzg.freeTrustedSetup().isOk \ No newline at end of file diff --git a/tests/test_rest_json_serialization.nim b/tests/test_rest_json_serialization.nim index e3d2a7fe9..ba0404f96 100644 --- a/tests/test_rest_json_serialization.nim +++ b/tests/test_rest_json_serialization.nim @@ -213,7 +213,7 @@ from stew/byteutils import hexToByteArray func fromHex(T: typedesc[KzgCommitment], s: string): T {. raises: [ValueError].} = var res: T - hexToByteArray(s, res.bytes) + hexToByteArray(s, res) res suite "REST JSON encoding and decoding": @@ -353,4 +353,4 @@ suite "REST JSON encoding and decoding": check: validator.pubkey == ValidatorPubKey.fromHex( "0x933ad9491b62059dd065b560d256d8957a8c402cc6e8d8ee7290ae11e8f7329267a8811c397529dac52ae1342ba58c95")[] - validator.exit_epoch == FAR_FUTURE_EPOCH + validator.exit_epoch == FAR_FUTURE_EPOCH \ No newline at end of file diff --git a/vendor/nim-kzg4844 b/vendor/nim-kzg4844 index 2f5cee7be..f12616d06 160000 --- a/vendor/nim-kzg4844 +++ b/vendor/nim-kzg4844 @@ -1 +1 @@ -Subproject commit 2f5cee7bea0d62e2b502ff668f752bda7f3eb0c4 +Subproject commit f12616d0675d9f6346141ca95f0840ab227eb213 From 20923a2465fad188401d8c1cb5baaa518889f070 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 15 Jun 2024 04:37:44 +0200 Subject: [PATCH 11/13] bump nimbus-build-system to `8cdaec502b5a48f2514e11209f0d81a001d2a2b1` (#6357) - bump rocksdb --- vendor/nimbus-build-system | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index a15dc546a..8cdaec502 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit a15dc546a053dd94c610a4178887bbf7c908aadb +Subproject commit 8cdaec502b5a48f2514e11209f0d81a001d2a2b1 From 7920dd0bb9e03b75345dce5d04c51943eb877253 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Sat, 15 Jun 2024 04:39:18 +0200 Subject: [PATCH 12/13] bump nim-metrics to `2e29df095059a7a787b234f040612b742567b2bc` (#6353) - use monotonic clock for update checks - Adopt asyncraises and add metrics middleware implementation --- vendor/nim-metrics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-metrics b/vendor/nim-metrics index 0e768ca7f..2e29df095 160000 --- a/vendor/nim-metrics +++ b/vendor/nim-metrics @@ -1 +1 @@ -Subproject commit 0e768ca7fb7df4798649145a403e7db65cae1f8b +Subproject commit 2e29df095059a7a787b234f040612b742567b2bc From 7d00786d5ed2024f309b7f4d638272d212ea94dc Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 24 Jun 2024 07:24:14 +0000 Subject: [PATCH 13/13] version v24.6.0 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ beacon_chain/version.nim | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad2db282..a9664ff4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +2024-06-24 v24.6.0 +================== + +Nimbus `v24.6.0` is a `low-urgency` release with performance and safety improvements. + +### Improvements + +* Improve SHA256 protocol object hashing speed by 30%: + https://github.com/status-im/nimbus-eth2/pull/6292 + +* Ensure that when a paired Nimbus beacon node and validator client improperly share validator keys, they avoid slashing: + https://github.com/status-im/nimbus-eth2/pull/6329 + +* Add block scoring to validator client to pick the best block from multiple beacon nodes: + https://github.com/status-im/nimbus-eth2/pull/6303 + +* Enable block monitoring in validator client by default to attest earlier and more reliably: + https://github.com/status-im/nimbus-eth2/pull/6331 + +### Fixes + +* Fix light client libp2p gossip topic subscriptions: + https://github.com/status-im/nimbus-eth2/pull/6351 + 2024-05-23 v24.5.1 ================== diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 8cae95c1f..7d199a5e7 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -18,8 +18,8 @@ const "Copyright (c) 2019-" & compileYear & " Status Research & Development GmbH" versionMajor* = 24 - versionMinor* = 5 - versionBuild* = 1 + versionMinor* = 6 + versionBuild* = 0 versionBlob* = "stateofus" # Single word - ends up in the default graffiti