From 242e1b73bb16e2cbad5b3947f63a7569c833c732 Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 20 Oct 2022 10:30:49 -0400 Subject: [PATCH 01/49] Rebase Capella on EIP-4844 This also introduces an `ENABLE_WITHDRAWALS` feature-flag to allow implementers test EIP-4844 without including Capella-specific state changes. --- specs/eip4844/beacon-chain.md | 64 ++++++++++++++++++++++++++++++++-- specs/eip4844/fork.md | 13 ++++--- specs/eip4844/p2p-interface.md | 2 +- specs/eip4844/validator.md | 59 +++++++++++++++++++++++++++++-- 4 files changed, 128 insertions(+), 10 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 4cf953593..877c3332a 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -15,6 +15,7 @@ - [Domain types](#domain-types) - [Preset](#preset) - [Execution](#execution) + - [Test Parameters](#test-parameters) - [Configuration](#configuration) - [Containers](#containers) - [Extended containers](#extended-containers) @@ -27,9 +28,11 @@ - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Epoch processing](#epoch-processing) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) + - [Modified `process_operations`](#modified-process_operations) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -38,7 +41,7 @@ ## Introduction -This upgrade adds blobs to the beacon chain as part of EIP-4844. +This upgrade adds blobs to the beacon chain as part of EIP-4844. This is an extension of the Capella upgrade. We introduce a new feature flag, `ENABLE_WITHDRAWALS`, to disable Capella-specific updates to the state transition function. This is done to minimize Capella specific issues that may arise during testing. `ENABLE_WITHDRAWALS` will be removed in the final upgrade specification. ## Custom types @@ -72,6 +75,10 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. | - | - | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | +### Test Parameters +| Name | Value | +| `ENABLE_WITHDRAWALS` | `uint64(0)` | + ## Configuration @@ -97,6 +104,7 @@ class BeaconBlockBody(Container): sync_aggregate: SyncAggregate # Execution execution_payload: ExecutionPayload + bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in EIP-4844] ``` @@ -121,6 +129,7 @@ class ExecutionPayload(Container): # Extra payload fields block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] ``` #### `ExecutionPayloadHeader` @@ -144,6 +153,7 @@ class ExecutionPayloadHeader(Container): # Extra payload fields block_hash: Hash32 # Hash of execution block transactions_root: Root + withdrawals_root: Root ``` ## Helper functions @@ -192,13 +202,37 @@ def verify_kzg_commitments_against_transactions(transactions: Sequence[Transacti ## Beacon chain state transition function +### Epoch processing + +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_inactivity_updates(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) + process_participation_flag_updates(state) + process_sync_committee_updates(state) + if ENABLE_WITHDRAWALS: + process_full_withdrawals(state) + process_partial_withdrawals(state) +``` + + ### Block processing ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) + if ENABLE_WITHDRAWALS: # [New in EIP-4844] + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP-4844] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) @@ -221,6 +255,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid assert execution_engine.notify_new_payload(payload) + # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, @@ -238,9 +273,32 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe excess_blobs=payload.excess_blobs, # [New in EIP-4844] block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), - ) + withdrawals_root=hash_tree_root(payload.withdrawals) if ENABLE_WITHDRAWALS else Bytes32(), # [New in EIP-4844] ``` +#### Modified `process_operations` + +*Note*: The function `process_operations` is modified to feature flag Withdrawals. + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) + if ENABLE_WITHDRAWALS: # [New in EIP-4844] + for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) +``` + + #### Blob KZG commitments ```python diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index eaabba916..bce6ffe70 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -44,6 +44,8 @@ def compute_fork_version(epoch: Epoch) -> Version: """ if epoch >= EIP4844_FORK_EPOCH: return EIP4844_FORK_VERSION + if epoch >= CAPELLA_FORK_EPOCH: + return CAPELLA_FORK_VERSION if epoch >= BELLATRIX_FORK_EPOCH: return BELLATRIX_FORK_VERSION if epoch >= ALTAIR_FORK_EPOCH: @@ -62,12 +64,11 @@ Note that for the pure EIP-4844 networks, we don't apply `upgrade_to_eip4844` si ### Upgrading the state -Since the `eip4844.BeaconState` format is equal to the `bellatrix.BeaconState` format, we only have to update `BeaconState.fork`. +Since the `eip4844.BeaconState` format is equal to the `Capella.BeaconState` format, we only have to update `BeaconState.fork`. ```python -def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: - # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState - epoch = bellatrix.get_current_epoch(pre) +def upgrade_to_eip4844(pre: Capella.BeaconState) -> BeaconState: + epoch = capella.get_current_epoch(pre) post = BeaconState( # Versioning genesis_time=pre.genesis_time, @@ -109,6 +110,10 @@ def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: next_sync_committee=pre.next_sync_committee, # Execution-layer latest_execution_payload_header=pre.latest_execution_payload_header, + # Withdrawals + withdrawal_queue=[], + next_withdrawal_index=WithdrawalIndex(0), + next_partial_withdrawal_validator_index=ValidatorIndex(0), ) return post diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index eab8815a4..60ccb0739 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -67,7 +67,7 @@ Some gossip meshes are upgraded in the fork of EIP4844 to support upgraded types Topics follow the same specification as in prior upgrades. All topics remain stable except the beacon block topic which is updated with the modified type. -The specification around the creation, validation, and dissemination of messages has not changed from the Bellatrix document unless explicitly noted here. +The specification around the creation, validation, and dissemination of messages has not changed from the Capella document unless explicitly noted here. The derivation of the `message-id` remains stable. diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index e1bcf9564..c998aeb6e 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -15,6 +15,9 @@ - [`BlobsAndCommitments`](#blobsandcommitments) - [`PolynomialAndCommitment`](#polynomialandcommitment) - [Helpers](#helpers) +- [Protocols](#protocols) + - [`ExecutionEngine`](#executionengine) + - [`get_payload`](#get_payload) - [`is_data_available`](#is_data_available) - [`hash_to_bls_field`](#hash_to_bls_field) - [`compute_powers`](#compute_powers) @@ -27,6 +30,7 @@ - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [Blob KZG commitments](#blob-kzg-commitments) - [Beacon Block publishing time](#beacon-block-publishing-time) + - [ExecutionPayload](#executionpayload) @@ -37,7 +41,7 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This document is an extension of the [Bellatrix -- Honest Validator](../bellatrix/validator.md) guide. +This document is an extension of the [Capella -- Honest Validator](../capella/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of EIP4844](./beacon-chain.md) are requisite for this document and used throughout. @@ -70,6 +74,14 @@ class PolynomialAndCommitment(Container): ## Helpers +## Protocols + +### `ExecutionEngine` + +#### `get_payload` + +`get_payload` returns the upgraded EIP-4844 `ExecutionPayload` type. + ### `is_data_available` The implementation of `is_data_available` is meant to change with later sharding upgrades. @@ -200,7 +212,7 @@ Namely, the blob handling and the addition of `BlobsSidecar`. ##### Blob KZG commitments -1. After retrieving the execution payload from the execution engine as specified in Bellatrix, +1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blobs_and_kzg_commitments(payload_id)`. 2. Validate `blobs` and `blob_kzg_commitments`: @@ -252,3 +264,46 @@ The validator MUST hold on to blobs for `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` to ensure the data-availability of these blobs throughout the network. After `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` nodes MAY prune the blobs and/or stop serving them. + +##### ExecutionPayload + +`ExecutionPayload`s are constructed as they were in Capella, except that we allow withdrawals to be disabled for testing. + +```python +def prepare_execution_payload(state: BeaconState, + pow_chain: Dict[Hash32, PowBlock], + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + suggested_fee_recipient: ExecutionAddress, + execution_engine: ExecutionEngine) -> Optional[PayloadId]: + if not is_merge_transition_complete(state): + is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() + is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH + if is_terminal_block_hash_set and not is_activation_epoch_reached: + # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed + return None + + terminal_pow_block = get_terminal_pow_block(pow_chain) + if terminal_pow_block is None: + # Pre-merge, no prepare payload call is needed + return None + # Signify merge via producing on top of the terminal PoW block + parent_hash = terminal_pow_block.block_hash + else: + # Post-merge, normal payload + parent_hash = state.latest_execution_payload_header.block_hash + + # Set the forkchoice head and initiate the payload build process + payload_attributes = PayloadAttributes( + timestamp=compute_timestamp_at_slot(state, state.slot), + prev_randao=get_randao_mix(state, get_current_epoch(state)), + suggested_fee_recipient=suggested_fee_recipient, + withdrawals=get_expected_withdrawals(state) if ENABLE_WITHDRAWALS else None, # [New in EIP-4844] + ) + return execution_engine.notify_forkchoice_updated( + head_block_hash=parent_hash, + safe_block_hash=safe_block_hash, + finalized_block_hash=finalized_block_hash, + payload_attributes=payload_attributes, + ) +``` From f6f2474c87edb910ec22710f7340a9dffc570e06 Mon Sep 17 00:00:00 2001 From: Inphi Date: Thu, 20 Oct 2022 14:39:18 -0400 Subject: [PATCH 02/49] Update specs/eip4844/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/eip4844/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 877c3332a..7e16bca2f 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -274,6 +274,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals) if ENABLE_WITHDRAWALS else Bytes32(), # [New in EIP-4844] + ) ``` #### Modified `process_operations` From 2ac57c7e7b203cce45df6d284ca01b29a0b60f4b Mon Sep 17 00:00:00 2001 From: inphi Date: Mon, 24 Oct 2022 12:44:11 -0400 Subject: [PATCH 03/49] Fix py setup And remove the ENABLE_WITHDRAWALS feature-flag. The Testing section in the spec has been updated to specify how withdrawals is to be disabled --- setup.py | 6 ++--- specs/eip4844/beacon-chain.md | 30 +++++++++++++----------- specs/eip4844/fork.md | 2 +- specs/eip4844/validator.md | 44 ----------------------------------- 4 files changed, 20 insertions(+), 62 deletions(-) diff --git a/setup.py b/setup.py index 6db4aa870..6a58e6ff1 100644 --- a/setup.py +++ b/setup.py @@ -582,14 +582,14 @@ from eth2spec.bellatrix import {preset_name} as bellatrix # # EIP4844SpecBuilder # -class EIP4844SpecBuilder(BellatrixSpecBuilder): +class EIP4844SpecBuilder(CapellaSpecBuilder): fork: str = EIP4844 @classmethod def imports(cls, preset_name: str): return super().imports(preset_name) + f''' from eth2spec.utils import kzg -from eth2spec.bellatrix import {preset_name} as bellatrix +from eth2spec.capella import {preset_name} as capella from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize ''' @@ -967,7 +967,7 @@ class PySpecCommand(Command): specs/bellatrix/p2p-interface.md sync/optimistic.md """ - if self.spec_fork == CAPELLA: + if self.spec_fork in (CAPELLA, EIP4844): self.md_doc_paths += """ specs/capella/beacon-chain.md specs/capella/fork.md diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 7e16bca2f..e92f932d0 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -15,7 +15,6 @@ - [Domain types](#domain-types) - [Preset](#preset) - [Execution](#execution) - - [Test Parameters](#test-parameters) - [Configuration](#configuration) - [Containers](#containers) - [Extended containers](#extended-containers) @@ -35,13 +34,14 @@ - [Modified `process_operations`](#modified-process_operations) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) + - [Disabling Withdrawals](#disabling-withdrawals) ## Introduction -This upgrade adds blobs to the beacon chain as part of EIP-4844. This is an extension of the Capella upgrade. We introduce a new feature flag, `ENABLE_WITHDRAWALS`, to disable Capella-specific updates to the state transition function. This is done to minimize Capella specific issues that may arise during testing. `ENABLE_WITHDRAWALS` will be removed in the final upgrade specification. +This upgrade adds blobs to the beacon chain as part of EIP-4844. This is an extension of the Capella upgrade. ## Custom types @@ -75,10 +75,6 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. This is an exte | - | - | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | -### Test Parameters -| Name | Value | -| `ENABLE_WITHDRAWALS` | `uint64(0)` | - ## Configuration @@ -218,9 +214,8 @@ def process_epoch(state: BeaconState) -> None: process_historical_roots_update(state) process_participation_flag_updates(state) process_sync_committee_updates(state) - if ENABLE_WITHDRAWALS: - process_full_withdrawals(state) - process_partial_withdrawals(state) + process_full_withdrawals(state) + process_partial_withdrawals(state) ``` @@ -230,8 +225,7 @@ def process_epoch(state: BeaconState) -> None: def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): - if ENABLE_WITHDRAWALS: # [New in EIP-4844] - process_withdrawals(state, block.body.execution_payload) + process_withdrawals(state, block.body.execution_payload) process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP-4844] process_randao(state, block.body) process_eth1_data(state, block.body) @@ -273,7 +267,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe excess_blobs=payload.excess_blobs, # [New in EIP-4844] block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), - withdrawals_root=hash_tree_root(payload.withdrawals) if ENABLE_WITHDRAWALS else Bytes32(), # [New in EIP-4844] + withdrawals_root=hash_tree_root(payload.withdrawals), ) ``` @@ -295,8 +289,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.attestations, process_attestation) for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) - if ENABLE_WITHDRAWALS: # [New in EIP-4844] - for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) + for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) ``` @@ -362,3 +355,12 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, return state ``` + +### Disabling Withdrawals +During testing we avoid Capella-specific updates the state transition. We do this by replacing the following functions with a no-op implementation: +- `process_full_withdrawals` +- `process_partial_withdrawals` +- `process_withdrawals` +- `process_bls_to_execution_change` + +The `get_expected_withdrawals` function is also modified to return an empty withdrawals list. As such, the PayloadAttributes used to update forkchoice does not contain withdrawals. diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index bce6ffe70..e40426557 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -67,7 +67,7 @@ Note that for the pure EIP-4844 networks, we don't apply `upgrade_to_eip4844` si Since the `eip4844.BeaconState` format is equal to the `Capella.BeaconState` format, we only have to update `BeaconState.fork`. ```python -def upgrade_to_eip4844(pre: Capella.BeaconState) -> BeaconState: +def upgrade_to_eip4844(pre: capella.BeaconState) -> BeaconState: epoch = capella.get_current_epoch(pre) post = BeaconState( # Versioning diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index c998aeb6e..41f5aab70 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -30,7 +30,6 @@ - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [Blob KZG commitments](#blob-kzg-commitments) - [Beacon Block publishing time](#beacon-block-publishing-time) - - [ExecutionPayload](#executionpayload) @@ -264,46 +263,3 @@ The validator MUST hold on to blobs for `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` to ensure the data-availability of these blobs throughout the network. After `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` nodes MAY prune the blobs and/or stop serving them. - -##### ExecutionPayload - -`ExecutionPayload`s are constructed as they were in Capella, except that we allow withdrawals to be disabled for testing. - -```python -def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], - safe_block_hash: Hash32, - finalized_block_hash: Hash32, - suggested_fee_recipient: ExecutionAddress, - execution_engine: ExecutionEngine) -> Optional[PayloadId]: - if not is_merge_transition_complete(state): - is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() - is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - if is_terminal_block_hash_set and not is_activation_epoch_reached: - # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed - return None - - terminal_pow_block = get_terminal_pow_block(pow_chain) - if terminal_pow_block is None: - # Pre-merge, no prepare payload call is needed - return None - # Signify merge via producing on top of the terminal PoW block - parent_hash = terminal_pow_block.block_hash - else: - # Post-merge, normal payload - parent_hash = state.latest_execution_payload_header.block_hash - - # Set the forkchoice head and initiate the payload build process - payload_attributes = PayloadAttributes( - timestamp=compute_timestamp_at_slot(state, state.slot), - prev_randao=get_randao_mix(state, get_current_epoch(state)), - suggested_fee_recipient=suggested_fee_recipient, - withdrawals=get_expected_withdrawals(state) if ENABLE_WITHDRAWALS else None, # [New in EIP-4844] - ) - return execution_engine.notify_forkchoice_updated( - head_block_hash=parent_hash, - safe_block_hash=safe_block_hash, - finalized_block_hash=finalized_block_hash, - payload_attributes=payload_attributes, - ) -``` From 0488c0c26e541db04a04f4bbc3b3d138a6b62ee7 Mon Sep 17 00:00:00 2001 From: inphi Date: Mon, 24 Oct 2022 12:51:54 -0400 Subject: [PATCH 04/49] remove unchanged epoch processing section --- specs/eip4844/beacon-chain.md | 45 ----------------------------------- specs/eip4844/fork.md | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index e92f932d0..78953bb36 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -27,11 +27,9 @@ - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [Epoch processing](#epoch-processing) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - - [Modified `process_operations`](#modified-process_operations) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) - [Disabling Withdrawals](#disabling-withdrawals) @@ -198,27 +196,6 @@ def verify_kzg_commitments_against_transactions(transactions: Sequence[Transacti ## Beacon chain state transition function -### Epoch processing - -```python -def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) - process_inactivity_updates(state) - process_rewards_and_penalties(state) - process_registry_updates(state) - process_slashings(state) - process_eth1_data_reset(state) - process_effective_balance_updates(state) - process_slashings_reset(state) - process_randao_mixes_reset(state) - process_historical_roots_update(state) - process_participation_flag_updates(state) - process_sync_committee_updates(state) - process_full_withdrawals(state) - process_partial_withdrawals(state) -``` - - ### Block processing ```python @@ -271,28 +248,6 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ) ``` -#### Modified `process_operations` - -*Note*: The function `process_operations` is modified to feature flag Withdrawals. - -```python -def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: - # Verify that outstanding deposits are processed up to the maximum number of deposits - assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - - def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: - for operation in operations: - fn(state, operation) - - for_ops(body.proposer_slashings, process_proposer_slashing) - for_ops(body.attester_slashings, process_attester_slashing) - for_ops(body.attestations, process_attestation) - for_ops(body.deposits, process_deposit) - for_ops(body.voluntary_exits, process_voluntary_exit) - for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) -``` - - #### Blob KZG commitments ```python diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index e40426557..62908112a 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -64,7 +64,7 @@ Note that for the pure EIP-4844 networks, we don't apply `upgrade_to_eip4844` si ### Upgrading the state -Since the `eip4844.BeaconState` format is equal to the `Capella.BeaconState` format, we only have to update `BeaconState.fork`. +Since the `eip4844.BeaconState` format is equal to the `capella.BeaconState` format, we only have to update `BeaconState.fork`. ```python def upgrade_to_eip4844(pre: capella.BeaconState) -> BeaconState: From 459310f744e35a5ce5f2ef0188127413535cb49b Mon Sep 17 00:00:00 2001 From: inphi Date: Mon, 24 Oct 2022 16:25:50 -0400 Subject: [PATCH 05/49] Fix test_process_execution_payload --- specs/eip4844/fork.md | 6 +++--- tests/core/pyspec/eth2spec/test/context.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index 62908112a..9d42a2302 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -111,9 +111,9 @@ def upgrade_to_eip4844(pre: capella.BeaconState) -> BeaconState: # Execution-layer latest_execution_payload_header=pre.latest_execution_payload_header, # Withdrawals - withdrawal_queue=[], - next_withdrawal_index=WithdrawalIndex(0), - next_partial_withdrawal_validator_index=ValidatorIndex(0), + withdrawal_queue=pre.withdrawal_queue, + next_withdrawal_index=pre.next_withdrawal_index, + next_partial_withdrawal_validator_index=pre.next_partial_withdrawal_validator_index, ) return post diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index bc04c05f2..242a0f891 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -14,7 +14,7 @@ from .helpers.constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844, SHARDING, MINIMAL, MAINNET, ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_BELLATRIX, - ALL_FORK_UPGRADES, + FORKS_BEFORE_CAPELLA, ALL_FORK_UPGRADES, ) from .helpers.typing import SpecForkName, PresetBaseName from .helpers.genesis import create_genesis_state @@ -596,7 +596,7 @@ def is_post_bellatrix(spec): def is_post_capella(spec): - return spec.fork == CAPELLA + return spec.fork not in FORKS_BEFORE_CAPELLA def is_post_eip4844(spec): From 6d270cdc52a5985b1d3da784a846b2f0768978a5 Mon Sep 17 00:00:00 2001 From: inphi Date: Mon, 24 Oct 2022 17:21:36 -0400 Subject: [PATCH 06/49] Add CAPELLA_FORK_EPOCH overrides --- tests/core/pyspec/eth2spec/test/context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 242a0f891..1172b73a4 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -305,6 +305,7 @@ def config_fork_epoch_overrides(spec, state): elif _check_current_version(spec, state, EIP4844): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['BELLATRIX_FORK_EPOCH'] = spec.GENESIS_EPOCH + overrides['CAPELLA_FORK_EPOCH'] = spec.GENESIS_EPOCH overrides['EIP4844_FORK_EPOCH'] = spec.GENESIS_EPOCH elif _check_current_version(spec, state, SHARDING): overrides['ALTAIR_FORK_EPOCH'] = spec.GENESIS_EPOCH From ca538f52a05d0738cbee0e5aaa075e703724f99b Mon Sep 17 00:00:00 2001 From: Inphi Date: Tue, 25 Oct 2022 12:06:07 -0400 Subject: [PATCH 07/49] Update specs/eip4844/beacon-chain.md Co-authored-by: Alex Stokes --- specs/eip4844/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 78953bb36..1df899b81 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -312,7 +312,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Hash32, ``` ### Disabling Withdrawals -During testing we avoid Capella-specific updates the state transition. We do this by replacing the following functions with a no-op implementation: +During testing we avoid Capella-specific updates to the state transition. We do this by replacing the following functions with a no-op implementation: - `process_full_withdrawals` - `process_partial_withdrawals` - `process_withdrawals` From 317209591deba05c7270260d4bb983ec410f593a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 26 Oct 2022 10:53:04 -0500 Subject: [PATCH 08/49] Make pyspec disable withdrawal-functions in EIP4844 --- setup.py | 28 +++++++++++++ .../test_process_bls_to_execution_change.py | 23 ++++++----- .../block_processing/test_process_deposit.py | 5 ++- .../test_process_withdrawals.py | 31 +++++++------- .../test_process_full_withdrawals.py | 33 +++++++-------- .../test_process_partial_withdrawals.py | 41 ++++++++++--------- .../test/capella/sanity/test_blocks.py | 14 +++---- tests/core/pyspec/eth2spec/test/context.py | 2 +- .../pyspec/eth2spec/test/helpers/constants.py | 1 + 9 files changed, 106 insertions(+), 72 deletions(-) diff --git a/setup.py b/setup.py index 6a58e6ff1..728f7d7ba 100644 --- a/setup.py +++ b/setup.py @@ -617,6 +617,34 @@ KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB) +# +# Temporarily disable Withdrawals functions for EIP4844 testnets +# + + +def no_op(fn): # type: ignore + def wrapper(*args, **kw): # type: ignore + return None + return wrapper + + +def get_empty_list_result(fn): # type: ignore + def wrapper(*args, **kw): # type: ignore + return [] + return wrapper + + +process_full_withdrawals = no_op(process_full_withdrawals) +process_partial_withdrawals = no_op(process_partial_withdrawals) +process_withdrawals = no_op(process_withdrawals) +process_bls_to_execution_change = no_op(process_bls_to_execution_change) +get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals) + + +# +# End +# + def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> BlobsSidecar: pass''' diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py index 8ff02489c..79d79fa6e 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_bls_to_execution_change.py @@ -1,7 +1,8 @@ +from eth2spec.test.helpers.constants import CAPELLA from eth2spec.test.helpers.keys import pubkeys from eth2spec.test.helpers.bls_to_execution_changes import get_signed_address_change -from eth2spec.test.context import spec_state_test, expect_assertion_error, with_capella_and_later, always_bls +from eth2spec.test.context import spec_state_test, expect_assertion_error, with_phases, always_bls def run_bls_to_execution_change_processing(spec, state, signed_address_change, valid=True): @@ -37,14 +38,14 @@ def run_bls_to_execution_change_processing(spec, state, signed_address_change, v yield 'post', state -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success(spec, state): signed_address_change = get_signed_address_change(spec, state) yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_not_activated(spec, state): validator_index = 3 @@ -62,7 +63,7 @@ def test_success_not_activated(spec, state): assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_in_activation_queue(spec, state): validator_index = 3 @@ -80,7 +81,7 @@ def test_success_in_activation_queue(spec, state): assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_in_exit_queue(spec, state): validator_index = 3 @@ -93,7 +94,7 @@ def test_success_in_exit_queue(spec, state): yield from run_bls_to_execution_change_processing(spec, state, signed_address_change) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_exited(spec, state): validator_index = 4 @@ -110,7 +111,7 @@ def test_success_exited(spec, state): assert not spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_withdrawable(spec, state): validator_index = 4 @@ -128,7 +129,7 @@ def test_success_withdrawable(spec, state): assert spec.is_fully_withdrawable_validator(validator, balance, spec.get_current_epoch(state)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_val_index_out_of_range(spec, state): # Create for one validator beyond the validator list length @@ -137,7 +138,7 @@ def test_fail_val_index_out_of_range(spec, state): yield from run_bls_to_execution_change_processing(spec, state, signed_address_change, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_already_0x01(spec, state): # Create for one validator beyond the validator list length @@ -149,7 +150,7 @@ def test_fail_already_0x01(spec, state): yield from run_bls_to_execution_change_processing(spec, state, signed_address_change, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_incorrect_from_bls_pubkey(spec, state): # Create for one validator beyond the validator list length @@ -163,7 +164,7 @@ def test_fail_incorrect_from_bls_pubkey(spec, state): yield from run_bls_to_execution_change_processing(spec, state, signed_address_change, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test @always_bls def test_fail_bad_signature(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py index e0603d301..685d17651 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_deposit.py @@ -1,7 +1,8 @@ from eth2spec.test.context import ( spec_state_test, - with_capella_and_later, + with_phases, ) +from eth2spec.test.helpers.constants import CAPELLA from eth2spec.test.helpers.state import next_epoch_via_block from eth2spec.test.helpers.deposits import ( prepare_state_and_deposit, @@ -10,7 +11,7 @@ from eth2spec.test.helpers.deposits import ( from eth2spec.test.helpers.withdrawals import set_validator_fully_withdrawable -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_top_up_to_withdrawn_validator(spec, state): validator_index = 0 diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py index 4932bc60d..a765aa835 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py @@ -1,8 +1,9 @@ +from eth2spec.test.helpers.constants import CAPELLA from eth2spec.test.helpers.execution_payload import ( build_empty_execution_payload, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error, with_capella_and_later +from eth2spec.test.context import spec_state_test, expect_assertion_error, with_phases from eth2spec.test.helpers.state import next_slot @@ -53,7 +54,7 @@ def run_withdrawals_processing(spec, state, execution_payload, valid=True): assert state.withdrawal_queue == pre_withdrawal_queue[num_withdrawals:] -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_empty_queue(spec, state): assert len(state.withdrawal_queue) == 0 @@ -64,7 +65,7 @@ def test_success_empty_queue(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_in_queue(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -75,7 +76,7 @@ def test_success_one_in_queue(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_max_per_slot_in_queue(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) @@ -86,7 +87,7 @@ def test_success_max_per_slot_in_queue(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_a_lot_in_queue(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) @@ -101,7 +102,7 @@ def test_success_a_lot_in_queue(spec, state): # Failure cases in which the number of withdrawals in the execution_payload is incorrect # -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_empty_queue_non_empty_withdrawals(spec, state): assert len(state.withdrawal_queue) == 0 @@ -118,7 +119,7 @@ def test_fail_empty_queue_non_empty_withdrawals(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_one_in_queue_none_in_withdrawals(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -130,7 +131,7 @@ def test_fail_one_in_queue_none_in_withdrawals(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_one_in_queue_two_in_withdrawals(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -142,7 +143,7 @@ def test_fail_one_in_queue_two_in_withdrawals(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_max_per_slot_in_queue_one_less_in_withdrawals(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD) @@ -154,7 +155,7 @@ def test_fail_max_per_slot_in_queue_one_less_in_withdrawals(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_a_lot_in_queue_too_few_in_withdrawals(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) @@ -170,7 +171,7 @@ def test_fail_a_lot_in_queue_too_few_in_withdrawals(spec, state): # Failure cases in which the withdrawals in the execution_payload are incorrect # -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_incorrect_dequeue_index(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -182,7 +183,7 @@ def test_fail_incorrect_dequeue_index(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_incorrect_dequeue_address(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -194,7 +195,7 @@ def test_fail_incorrect_dequeue_address(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_incorrect_dequeue_amount(spec, state): prepare_withdrawal_queue(spec, state, 1) @@ -206,7 +207,7 @@ def test_fail_incorrect_dequeue_amount(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_one_of_many_dequeued_incorrectly(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) @@ -224,7 +225,7 @@ def test_fail_one_of_many_dequeued_incorrectly(spec, state): yield from run_withdrawals_processing(spec, state, execution_payload, valid=False) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_fail_many_dequeued_incorrectly(spec, state): prepare_withdrawal_queue(spec, state, spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4) diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py index 35d2968cb..1e1c9d60a 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_full_withdrawals.py @@ -1,15 +1,16 @@ from random import Random from eth2spec.test.context import ( - with_capella_and_later, + with_phases, spec_state_test, ) +from eth2spec.test.helpers.constants import CAPELLA +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_to, +) from eth2spec.test.helpers.random import ( randomize_state, ) -from eth2spec.test.helpers.epoch_processing import ( - run_epoch_processing_to, -) from eth2spec.test.helpers.withdrawals import ( set_validator_fully_withdrawable, ) @@ -41,7 +42,7 @@ def run_process_full_withdrawals(spec, state, num_expected_withdrawals=None): assert state.next_withdrawal_index == pre_next_withdrawal_index + num_expected_withdrawals -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_no_withdrawable_validators(spec, state): pre_validators = state.validators.copy() @@ -50,7 +51,7 @@ def test_no_withdrawable_validators(spec, state): assert pre_validators == state.validators -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_withdrawable_epoch_but_0_balance(spec, state): current_epoch = spec.get_current_epoch(state) @@ -62,7 +63,7 @@ def test_withdrawable_epoch_but_0_balance(spec, state): yield from run_process_full_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_withdrawable_epoch_but_0_effective_balance_0_balance(spec, state): current_epoch = spec.get_current_epoch(state) @@ -74,7 +75,7 @@ def test_withdrawable_epoch_but_0_effective_balance_0_balance(spec, state): yield from run_process_full_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_withdrawable_epoch_but_0_effective_balance_nonzero_balance(spec, state): current_epoch = spec.get_current_epoch(state) @@ -86,7 +87,7 @@ def test_withdrawable_epoch_but_0_effective_balance_nonzero_balance(spec, state) yield from run_process_full_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_no_withdrawals_but_some_next_epoch(spec, state): current_epoch = spec.get_current_epoch(state) @@ -98,7 +99,7 @@ def test_no_withdrawals_but_some_next_epoch(spec, state): yield from run_process_full_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_single_withdrawal(spec, state): # Make one validator withdrawable @@ -110,7 +111,7 @@ def test_single_withdrawal(spec, state): assert state.next_withdrawal_index == 1 -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_multi_withdrawal(spec, state): # Make a few validators withdrawable @@ -120,7 +121,7 @@ def test_multi_withdrawal(spec, state): yield from run_process_full_withdrawals(spec, state, 3) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_all_withdrawal(spec, state): # Make all validators withdrawable @@ -150,25 +151,25 @@ def run_random_full_withdrawals_test(spec, state, rng): yield from run_process_full_withdrawals(spec, state, None) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_withdrawals_0(spec, state): yield from run_random_full_withdrawals_test(spec, state, Random(444)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_withdrawals_1(spec, state): yield from run_random_full_withdrawals_test(spec, state, Random(420)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_withdrawals_2(spec, state): yield from run_random_full_withdrawals_test(spec, state, Random(200)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_withdrawals_3(spec, state): yield from run_random_full_withdrawals_test(spec, state, Random(2000000)) diff --git a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py index 7569d2862..91411b9d3 100644 --- a/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_partial_withdrawals.py @@ -1,10 +1,11 @@ import random from eth2spec.test.helpers.constants import MINIMAL from eth2spec.test.context import ( - with_capella_and_later, + with_phases, spec_state_test, with_presets, ) +from eth2spec.test.helpers.constants import CAPELLA from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.random import randomize_state @@ -48,7 +49,7 @@ def run_process_partial_withdrawals(spec, state, num_expected_withdrawals=None): assert state.next_withdrawal_index == pre_next_withdrawal_index + num_expected_withdrawals -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_no_withdrawable(spec, state): pre_validators = state.validators.copy() @@ -57,7 +58,7 @@ def test_success_no_withdrawable(spec, state): assert pre_validators == state.validators -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_no_max_effective_balance(spec, state): validator_index = len(state.validators) // 2 @@ -71,7 +72,7 @@ def test_success_no_max_effective_balance(spec, state): yield from run_process_partial_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_no_excess_balance(spec, state): validator_index = len(state.validators) // 2 @@ -85,7 +86,7 @@ def test_success_no_excess_balance(spec, state): yield from run_process_partial_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_excess_balance_but_no_max_effective_balance(spec, state): validator_index = len(state.validators) // 2 @@ -100,7 +101,7 @@ def test_success_excess_balance_but_no_max_effective_balance(spec, state): yield from run_process_partial_withdrawals(spec, state, 0) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable(spec, state): validator_index = len(state.validators) // 2 @@ -109,7 +110,7 @@ def test_success_one_partial_withdrawable(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable_not_yet_active(spec, state): validator_index = len(state.validators) // 2 @@ -121,7 +122,7 @@ def test_success_one_partial_withdrawable_not_yet_active(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable_in_exit_queue(spec, state): validator_index = len(state.validators) // 2 @@ -134,7 +135,7 @@ def test_success_one_partial_withdrawable_in_exit_queue(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable_exited(spec, state): validator_index = len(state.validators) // 2 @@ -146,7 +147,7 @@ def test_success_one_partial_withdrawable_exited(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable_active_and_slashed(spec, state): validator_index = len(state.validators) // 2 @@ -158,7 +159,7 @@ def test_success_one_partial_withdrawable_active_and_slashed(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_one_partial_withdrawable_exited_and_slashed(spec, state): validator_index = len(state.validators) // 2 @@ -171,7 +172,7 @@ def test_success_one_partial_withdrawable_exited_and_slashed(spec, state): yield from run_process_partial_withdrawals(spec, state, 1) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_two_partial_withdrawable(spec, state): set_validator_partially_withdrawable(spec, state, 0) @@ -180,7 +181,7 @@ def test_success_two_partial_withdrawable(spec, state): yield from run_process_partial_withdrawals(spec, state, 2) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_success_max_partial_withdrawable(spec, state): # Sanity check that this test works for this state @@ -192,7 +193,7 @@ def test_success_max_partial_withdrawable(spec, state): yield from run_process_partial_withdrawals(spec, state, spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) -@with_capella_and_later +@with_phases([CAPELLA]) @with_presets([MINIMAL], reason="not enough validators with mainnet config") @spec_state_test def test_success_max_plus_one_withdrawable(spec, state): @@ -226,37 +227,37 @@ def run_random_partial_withdrawals_test(spec, state, rng): yield from run_process_partial_withdrawals(spec, state) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_0(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(0)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_1(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(1)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_2(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(2)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_3(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(3)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_4(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(4)) -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_random_5(spec, state): yield from run_random_partial_withdrawals_test(spec, state, random.Random(5)) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index 28c20a2cd..c76c66e8d 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -1,7 +1,7 @@ from eth2spec.test.context import ( - with_capella_and_later, spec_state_test + with_phases, spec_state_test ) - +from eth2spec.test.helpers.constants import CAPELLA from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) @@ -16,7 +16,7 @@ from eth2spec.test.helpers.withdrawals import ( from eth2spec.test.helpers.voluntary_exits import prepare_signed_exits -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_successful_bls_change(spec, state): index = 0 @@ -39,7 +39,7 @@ def test_successful_bls_change(spec, state): assert post_credentials[12:] == signed_address_change.message.to_execution_address -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_full_withdrawal_in_epoch_transition(spec, state): index = 0 @@ -57,7 +57,7 @@ def test_full_withdrawal_in_epoch_transition(spec, state): assert state.balances[index] == 0 -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_partial_withdrawal_in_epoch_transition(spec, state): index = state.next_withdrawal_index @@ -81,7 +81,7 @@ def test_partial_withdrawal_in_epoch_transition(spec, state): assert len(state.withdrawal_queue) == pre_withdrawal_queue_len -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_many_partial_withdrawals_in_epoch_transition(spec, state): assert len(state.validators) > spec.MAX_WITHDRAWALS_PER_PAYLOAD @@ -106,7 +106,7 @@ def test_many_partial_withdrawals_in_epoch_transition(spec, state): assert len(state.withdrawal_queue) == pre_withdrawal_queue_len + 1 -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_exit_and_bls_change(spec, state): # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1172b73a4..11ed21b10 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -606,7 +606,7 @@ def is_post_eip4844(spec): with_altair_and_later = with_all_phases_except([PHASE0]) with_bellatrix_and_later = with_all_phases_except([PHASE0, ALTAIR]) -with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, EIP4844]) +with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX]) with_eip4844_and_later = with_all_phases_except([PHASE0, ALTAIR, BELLATRIX, CAPELLA]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index b1463b97b..bfb8ffea0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -34,6 +34,7 @@ ALL_FORK_UPGRADES = { PHASE0: ALTAIR, ALTAIR: BELLATRIX, BELLATRIX: CAPELLA, + CAPELLA: EIP4844, } ALL_PRE_POST_FORKS = ALL_FORK_UPGRADES.items() AFTER_BELLATRIX_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() if key not in FORKS_BEFORE_ALTAIR} From e460005aff5f8c6b4778a0c5f8090f82ed729551 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 26 Oct 2022 15:06:59 -0500 Subject: [PATCH 09/49] Add tests for no-op functions --- .../test/eip4844/block_processing/__init__.py | 0 .../test_process_bls_to_execution_change.py | 40 +++++++++++++++++ .../test/eip4844/epoch_processing/__init__.py | 0 .../test_process_full_withdrawals.py | 43 +++++++++++++++++++ .../test_process_partial_withdrawals.py | 43 +++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_bls_to_execution_change.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_bls_to_execution_change.py b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_bls_to_execution_change.py new file mode 100644 index 000000000..d9b93394f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_bls_to_execution_change.py @@ -0,0 +1,40 @@ +from eth2spec.test.helpers.bls_to_execution_changes import get_signed_address_change +from eth2spec.test.context import spec_state_test, expect_assertion_error, with_eip4844_and_later + + +def run_bls_to_execution_change_processing_no_op(spec, state, signed_address_change, valid=True): + """ + Run ``process_bls_to_execution_change``, yielding: + - pre-state ('pre') + - address-change ('address_change') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + pre_state = state.copy() + + # yield pre-state + yield 'pre', state + + yield 'address_change', signed_address_change + + # If the address_change is invalid, processing is aborted, and there is no post-state. + if not valid: + expect_assertion_error(lambda: spec.process_bls_to_execution_change(state, signed_address_change)) + yield 'post', None + return + + # process address change + spec.process_bls_to_execution_change(state, signed_address_change) + + # yield post-state + yield 'post', state + + # Make sure state has NOT been changed + assert state == pre_state + + +@with_eip4844_and_later +@spec_state_test +def test_no_op(spec, state): + signed_address_change = get_signed_address_change(spec, state) + yield from run_bls_to_execution_change_processing_no_op(spec, state, signed_address_change) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py new file mode 100644 index 000000000..1a7cda91d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py @@ -0,0 +1,43 @@ +from eth2spec.test.context import ( + with_eip4844_and_later, + spec_state_test, +) +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_to, +) +from eth2spec.test.helpers.withdrawals import ( + set_validator_fully_withdrawable, +) + + +def run_process_full_withdrawals_no_op(spec, state, num_expected_withdrawals=None): + run_epoch_processing_to(spec, state, 'process_full_withdrawals') + + state.next_withdrawal_index = 0 + to_be_withdrawn_indices = [ + index for index, validator in enumerate(state.validators) + if spec.is_fully_withdrawable_validator(validator, state.balances[index], spec.get_current_epoch(state)) + ] + + if num_expected_withdrawals is not None: + assert len(to_be_withdrawn_indices) == num_expected_withdrawals + else: + num_expected_withdrawals = len(to_be_withdrawn_indices) + + pre_state = state.copy() + + yield 'pre', state + spec.process_full_withdrawals(state) + yield 'post', state + + # Make sure state has NOT been changed + assert state == pre_state + + +@with_eip4844_and_later +@spec_state_test +def test_no_op(spec, state): + # Make one validator withdrawable + set_validator_fully_withdrawable(spec, state, 0) + + yield from run_process_full_withdrawals_no_op(spec, state, 1) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py new file mode 100644 index 000000000..992133420 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py @@ -0,0 +1,43 @@ +from eth2spec.test.context import ( + spec_state_test, + with_eip4844_and_later, +) +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to +from eth2spec.test.helpers.withdrawals import ( + set_validator_partially_withdrawable, +) + + +def run_process_partial_withdrawals_no_op(spec, state, num_expected_withdrawals=None): + # Run rest of epoch processing before predicting partial withdrawals as + # balance changes can affect withdrawability + run_epoch_processing_to(spec, state, 'process_partial_withdrawals') + + partially_withdrawable_indices = [ + index for index, validator in enumerate(state.validators) + if spec.is_partially_withdrawable_validator(validator, state.balances[index]) + ] + num_partial_withdrawals = min(len(partially_withdrawable_indices), spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) + + if num_expected_withdrawals is not None: + assert num_partial_withdrawals == num_expected_withdrawals + else: + num_expected_withdrawals = num_partial_withdrawals + + pre_state = state.copy() + + yield 'pre', state + spec.process_partial_withdrawals(state) + yield 'post', state + + # Make sure state has NOT been changed + assert state == pre_state + + +@with_eip4844_and_later +@spec_state_test +def test_success_one_partial_withdrawable(spec, state): + validator_index = len(state.validators) // 2 + set_validator_partially_withdrawable(spec, state, validator_index) + + yield from run_process_partial_withdrawals_no_op(spec, state, 1) From 60187e5dc4d12137e3441933a63903344750704c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 26 Oct 2022 15:19:15 -0500 Subject: [PATCH 10/49] Add `test_process_withdrawals` no-op test --- .../test_process_withdrawals.py | 57 +++++++++++++++++++ .../test_process_partial_withdrawals.py | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py new file mode 100644 index 000000000..86ac7e4ae --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py @@ -0,0 +1,57 @@ + +from eth2spec.test.context import spec_state_test, expect_assertion_error, with_eip4844_and_later +from eth2spec.test.helpers.execution_payload import ( + build_empty_execution_payload, +) +from eth2spec.test.helpers.state import next_slot + + +def prepare_withdrawal_queue(spec, state, num_withdrawals): + pre_queue_len = len(state.withdrawal_queue) + + for i in range(num_withdrawals): + withdrawal = spec.Withdrawal( + index=i + 5, + address=b'\x42' * 20, + amount=200000 + i, + ) + state.withdrawal_queue.append(withdrawal) + + assert len(state.withdrawal_queue) == num_withdrawals + pre_queue_len + + +def run_withdrawals_processing(spec, state, execution_payload, valid=True): + """ + Run ``process_execution_payload``, yielding: + - pre-state ('pre') + - execution payload ('execution_payload') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + pre_state = state.copy() + + yield 'pre', state + yield 'execution_payload', execution_payload + + if not valid: + expect_assertion_error(lambda: spec.process_withdrawals(state, execution_payload)) + yield 'post', None + return + + spec.process_withdrawals(state, execution_payload) + + yield 'post', state + + # Make sure state has NOT been changed + assert state == pre_state + + +@with_eip4844_and_later +@spec_state_test +def test_no_op(spec, state): + prepare_withdrawal_queue(spec, state, 1) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + + yield from run_withdrawals_processing(spec, state, execution_payload) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py index 992133420..78be3eb6c 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py @@ -36,7 +36,7 @@ def run_process_partial_withdrawals_no_op(spec, state, num_expected_withdrawals= @with_eip4844_and_later @spec_state_test -def test_success_one_partial_withdrawable(spec, state): +def test_no_op(spec, state): validator_index = len(state.validators) // 2 set_validator_partially_withdrawable(spec, state, validator_index) From cdaf7e84dd8e4af2400fabd5505f6c8d46ea4c87 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 27 Oct 2022 21:59:17 +0200 Subject: [PATCH 11/49] Allow extending LC merkle proof tests Currently, `test_single_merkle_proof` only supports `BeaconState` tests. For future tests, different object classes are desirable. Update format to allow testing other objects as well. --- .../light_client/test_single_merkle_proof.py | 18 ++++++++++++------ .../light_client/single_merkle_proof.md | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py index 064760bf0..5628cb70d 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py @@ -7,8 +7,10 @@ from eth2spec.test.context import ( @with_altair_and_later @spec_state_test def test_current_sync_committee_merkle_proof(spec, state): - yield "state", state - current_sync_committee_branch = spec.compute_merkle_proof_for_state(state, spec.CURRENT_SYNC_COMMITTEE_INDEX) + yield "object_class", "meta", "BeaconState" + yield "object", state + current_sync_committee_branch = \ + spec.compute_merkle_proof_for_state(state, spec.CURRENT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.current_sync_committee.hash_tree_root().hex(), "leaf_index": spec.CURRENT_SYNC_COMMITTEE_INDEX, @@ -26,8 +28,10 @@ def test_current_sync_committee_merkle_proof(spec, state): @with_altair_and_later @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): - yield "state", state - next_sync_committee_branch = spec.compute_merkle_proof_for_state(state, spec.NEXT_SYNC_COMMITTEE_INDEX) + yield "object_class", "meta", "BeaconState" + yield "object", state + next_sync_committee_branch = \ + spec.compute_merkle_proof_for_state(state, spec.NEXT_SYNC_COMMITTEE_INDEX) yield "proof", { "leaf": "0x" + state.next_sync_committee.hash_tree_root().hex(), "leaf_index": spec.NEXT_SYNC_COMMITTEE_INDEX, @@ -45,8 +49,10 @@ def test_next_sync_committee_merkle_proof(spec, state): @with_altair_and_later @spec_state_test def test_finality_root_merkle_proof(spec, state): - yield "state", state - finality_branch = spec.compute_merkle_proof_for_state(state, spec.FINALIZED_ROOT_INDEX) + yield "object_class", "meta", "BeaconState" + yield "object", state + finality_branch = \ + spec.compute_merkle_proof_for_state(state, spec.FINALIZED_ROOT_INDEX) yield "proof", { "leaf": "0x" + state.finalized_checkpoint.root.hex(), "leaf_index": spec.FINALIZED_ROOT_INDEX, diff --git a/tests/formats/light_client/single_merkle_proof.md b/tests/formats/light_client/single_merkle_proof.md index 65fe7c988..c8b2dea6e 100644 --- a/tests/formats/light_client/single_merkle_proof.md +++ b/tests/formats/light_client/single_merkle_proof.md @@ -5,24 +5,30 @@ generation and verification of merkle proofs based on static data. ## Test case format -### `state.ssz_snappy` +### `meta.yaml` -An SSZ-snappy encoded `BeaconState` object from which other data is generated. +```yaml +object_class: string -- 'BeaconState' +``` + +### `object.yaml` + +A SSZ-snappy encoded object of type `object_class` from which other data is generated. ### `proof.yaml` -A proof of the leaf value (a merkle root) at generalized-index `leaf_index` in the given `state`. +A proof of the leaf value (a merkle root) at generalized-index `leaf_index` in the given `object`. ```yaml leaf: Bytes32 # string, hex encoded, with 0x prefix leaf_index: int # integer, decimal -branch: list of Bytes32 # list, each element is a string, hex encoded, with 0x prefix +branch: list of Bytes32 # list, each element is a string, hex encoded, with 0x prefix ``` ## Condition A test-runner can implement the following assertions: - Check that `is_valid_merkle_branch` confirms `leaf` at `leaf_index` to verify - against `has_tree_root(state)` and `proof`. + against `hash_tree_root(object)` and `branch`. - If the implementation supports generating merkle proofs, check that the - self-generated proof matches the `proof` provided with the test. + self-generated proof matches the `branch` provided with the test. From 5c663001711ae6507862731801cb161adc122802 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 28 Oct 2022 20:32:33 +0200 Subject: [PATCH 12/49] Use `suite_name` instead of `meta.yaml` for SSZ type --- .../pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 2 +- .../test/altair/light_client/test_single_merkle_proof.py | 7 ++++--- tests/core/pyspec/eth2spec/test/context.py | 7 +++++++ tests/formats/light_client/single_merkle_proof.md | 8 ++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 328b7edf1..b951a6a85 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -49,7 +49,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, preset_name=preset_name, runner_name=runner_name, handler_name=handler_name, - suite_name='pyspec_tests', + suite_name=getattr(tfn, 'suite_name', 'pyspec_tests'), case_name=case_name, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. case_fn=lambda: tfn(generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active) diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py index 5628cb70d..465fa629f 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_single_merkle_proof.py @@ -1,13 +1,14 @@ from eth2spec.test.context import ( spec_state_test, with_altair_and_later, + with_test_suite_name, ) +@with_test_suite_name("BeaconState") @with_altair_and_later @spec_state_test def test_current_sync_committee_merkle_proof(spec, state): - yield "object_class", "meta", "BeaconState" yield "object", state current_sync_committee_branch = \ spec.compute_merkle_proof_for_state(state, spec.CURRENT_SYNC_COMMITTEE_INDEX) @@ -25,10 +26,10 @@ def test_current_sync_committee_merkle_proof(spec, state): ) +@with_test_suite_name("BeaconState") @with_altair_and_later @spec_state_test def test_next_sync_committee_merkle_proof(spec, state): - yield "object_class", "meta", "BeaconState" yield "object", state next_sync_committee_branch = \ spec.compute_merkle_proof_for_state(state, spec.NEXT_SYNC_COMMITTEE_INDEX) @@ -46,10 +47,10 @@ def test_next_sync_committee_merkle_proof(spec, state): ) +@with_test_suite_name("BeaconState") @with_altair_and_later @spec_state_test def test_finality_root_merkle_proof(spec, state): - yield "object_class", "meta", "BeaconState" yield "object", state finality_branch = \ spec.compute_merkle_proof_for_state(state, spec.FINALIZED_ROOT_INDEX) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index bc04c05f2..bcc77398c 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -620,6 +620,13 @@ def only_generator(reason): return _decorator +def with_test_suite_name(suite_name: str): + def _decorator(inner): + inner.suite_name = suite_name + return inner + return _decorator + + # # Fork transition state tests # diff --git a/tests/formats/light_client/single_merkle_proof.md b/tests/formats/light_client/single_merkle_proof.md index c8b2dea6e..d2137605e 100644 --- a/tests/formats/light_client/single_merkle_proof.md +++ b/tests/formats/light_client/single_merkle_proof.md @@ -5,15 +5,11 @@ generation and verification of merkle proofs based on static data. ## Test case format -### `meta.yaml` - -```yaml -object_class: string -- 'BeaconState' -``` +Tests for each individual SSZ type are grouped into a `suite` indicating the SSZ type name. ### `object.yaml` -A SSZ-snappy encoded object of type `object_class` from which other data is generated. +A SSZ-snappy encoded object from which other data is generated. The SSZ type can be determined from the test `suite` name. ### `proof.yaml` From e3e73a8f5484a35a9e491b64134bc6275be06996 Mon Sep 17 00:00:00 2001 From: inphi Date: Tue, 8 Nov 2022 09:06:38 -0500 Subject: [PATCH 13/49] Add Capella fork version --- specs/eip4844/p2p-interface.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index f861ed28b..2a8786969 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -123,6 +123,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | | `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | | `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | +| `CAPELLA_FORK_VERSION` | `capella.SignedBeaconBlock` | | `EIP4844_FORK_VERSION` | `eip4844.SignedBeaconBlock` | #### BeaconBlocksByRoot v2 @@ -140,6 +141,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | | `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | | `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | +| `CAPELLA_FORK_VERSION` | `capella.SignedBeaconBlock` | | `EIP4844_FORK_VERSION` | `eip4844.SignedBeaconBlock` | #### BlobsSidecarsByRange v1 From cb9159fe588178f7cd835380aecc8b1fa5b9be4b Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 8 Nov 2022 10:41:48 -0800 Subject: [PATCH 14/49] EIP4844: Add block and sidecar retrival by root --- specs/eip4844/p2p-interface.md | 46 +++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 1576fe96f..4e08195a5 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -23,6 +23,7 @@ The specification of these changes continues in the same format as the network s - [Messages](#messages) - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) + - [BeaconBlockAndBlobsSidecarByRoot v1](#beaconblockandblobssidecarbyroot-v1) - [BlobsSidecarsByRange v1](#blobssidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) @@ -129,18 +130,45 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` -The EIP-4844 fork-digest is introduced to the `context` enum to specify EIP-4844 beacon block type. +`BeaconBlocksByRoot v2` will be disabled in favor of `BeaconBlockAndBlobsSidecarByRoot v1` for tighter coupling of blocks and sidecars. -Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: +Clients MUST disable `BeaconBlocksByRoot v2` after `EIP4844_FORK_EPOCH`. -[1]: # (eth2spec: skip) +#### BeaconBlockAndBlobsSidecarByRoot v1 -| `fork_version` | Chunk SSZ type | -| ------------------------ | -------------------------- | -| `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | -| `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | -| `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | -| `EIP4844_FORK_VERSION` | `eip4844.SignedBeaconBlock` | +**Protocol ID:** `/eth2/beacon_chain/req/beacon_block_and_blobs_sidecar_by_root/1/` + +Request Content: + +``` +( + List[Root, MAX_REQUEST_BLOCKS] +) +``` + +Response Content: + +``` +( + List[SignedBeaconBlockAndBlobsSidecar, MAX_REQUEST_BLOCKS] +) +``` + +Requests blocks by block root (= `hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)`). +The response is a list of `SignedBeaconBlockAndBlobsSidecar` whose length is less than or equal to the number of requests. +It may be less in the case that the responding peer is missing blocks and sidecars. + +No more than `MAX_REQUEST_BLOCKS` may be requested at a time. + +`BeaconBlockAndBlobsSidecarByRoot` is primarily used to recover recent blocks and sidecars (e.g. when receiving a block or attestation whose parent is unknown). + +The response MUST consist of zero or more `response_chunk`. +Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlockAndBlobsSidecar` payload. + +Clients MUST support requesting blocks and sidecars since the latest finalized epoch. + +Clients MUST respond with at least one block and sidecar, if they have it. +Clients MAY limit the number of blocks and sidecars in the response. #### BlobsSidecarsByRange v1 From c447662ba078905ee64c54100a614a71450ee839 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 8 Nov 2022 12:55:43 -0800 Subject: [PATCH 15/49] Fix `BeaconBlocksByRootV2` for pre-4844 support --- specs/eip4844/p2p-interface.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 4e08195a5..b431b7927 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -130,9 +130,17 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` -`BeaconBlocksByRoot v2` will be disabled in favor of `BeaconBlockAndBlobsSidecarByRoot v1` for tighter coupling of blocks and sidecars. +After `EIP4844_FORK_EPOCH`, `BeaconBlocksByRootV2` is replaced by `BeaconBlockAndBlobsSidecarByRootV1` -Clients MUST disable `BeaconBlocksByRoot v2` after `EIP4844_FORK_EPOCH`. +Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: + +[1]: # (eth2spec: skip) + +| `fork_version` | Chunk SSZ type | +| ------------------------ | -------------------------- | +| `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | +| `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | +| `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | #### BeaconBlockAndBlobsSidecarByRoot v1 From a04f06bd11cc782c8f02d2890e2a4a51b2d311e8 Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 10 Nov 2022 13:29:33 -0500 Subject: [PATCH 16/49] Fix merge conflict --- setup.py | 2 -- tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 589b5fdf1..205e8c400 100644 --- a/setup.py +++ b/setup.py @@ -669,8 +669,6 @@ def get_empty_list_result(fn): # type: ignore return wrapper -process_full_withdrawals = no_op(process_full_withdrawals) -process_partial_withdrawals = no_op(process_partial_withdrawals) process_withdrawals = no_op(process_withdrawals) process_bls_to_execution_change = no_op(process_bls_to_execution_change) get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index 6ca2e9432..3c406e524 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -170,7 +170,7 @@ def _perform_valid_withdrawal(spec, state): return pre_state, signed_block_1, pre_next_withdrawal_index -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_withdrawal_success_two_blocks(spec, state): pre_state, signed_block_1, pre_next_withdrawal_index = _perform_valid_withdrawal(spec, state) @@ -187,7 +187,7 @@ def test_withdrawal_success_two_blocks(spec, state): yield 'post', state -@with_capella_and_later +@with_phases([CAPELLA]) @spec_state_test def test_withdrawal_fail_second_block_payload_isnt_compatible(spec, state): _perform_valid_withdrawal(spec, state) From bed1df00a2452d4f0f6d26f8f2b70b35fb1b461c Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 10 Nov 2022 13:57:30 -0500 Subject: [PATCH 17/49] Remove withdrawal_queue from BeaconState upgrade --- specs/eip4844/fork.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index b13eb5a32..bdd26e811 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -111,7 +111,6 @@ def upgrade_to_eip4844(pre: capella.BeaconState) -> BeaconState: # Execution-layer latest_execution_payload_header=pre.latest_execution_payload_header, # Withdrawals - withdrawal_queue=pre.withdrawal_queue, next_withdrawal_index=pre.next_withdrawal_index, next_withdrawal_validator_index=pre.next_withdrawal_validator_index, ) From 2fbb1eddd2e9b064c794c826c73cd0004af832da Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 10 Nov 2022 14:01:50 -0500 Subject: [PATCH 18/49] fix test_process_withdrawals --- .../block_processing/test_process_withdrawals.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py index 86ac7e4ae..a7db37e42 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/block_processing/test_process_withdrawals.py @@ -6,20 +6,6 @@ from eth2spec.test.helpers.execution_payload import ( from eth2spec.test.helpers.state import next_slot -def prepare_withdrawal_queue(spec, state, num_withdrawals): - pre_queue_len = len(state.withdrawal_queue) - - for i in range(num_withdrawals): - withdrawal = spec.Withdrawal( - index=i + 5, - address=b'\x42' * 20, - amount=200000 + i, - ) - state.withdrawal_queue.append(withdrawal) - - assert len(state.withdrawal_queue) == num_withdrawals + pre_queue_len - - def run_withdrawals_processing(spec, state, execution_payload, valid=True): """ Run ``process_execution_payload``, yielding: @@ -49,8 +35,6 @@ def run_withdrawals_processing(spec, state, execution_payload, valid=True): @with_eip4844_and_later @spec_state_test def test_no_op(spec, state): - prepare_withdrawal_queue(spec, state, 1) - next_slot(spec, state) execution_payload = build_empty_execution_payload(spec, state) From fcafdc14a388373c12e59a0c72d169b83b865b5e Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 10 Nov 2022 14:11:05 -0500 Subject: [PATCH 19/49] remove eip4844 partial/full withdrawwals tests --- .../test_process_full_withdrawals.py | 43 ------------------- .../test_process_partial_withdrawals.py | 43 ------------------- 2 files changed, 86 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py delete mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py deleted file mode 100644 index 1a7cda91d..000000000 --- a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_full_withdrawals.py +++ /dev/null @@ -1,43 +0,0 @@ -from eth2spec.test.context import ( - with_eip4844_and_later, - spec_state_test, -) -from eth2spec.test.helpers.epoch_processing import ( - run_epoch_processing_to, -) -from eth2spec.test.helpers.withdrawals import ( - set_validator_fully_withdrawable, -) - - -def run_process_full_withdrawals_no_op(spec, state, num_expected_withdrawals=None): - run_epoch_processing_to(spec, state, 'process_full_withdrawals') - - state.next_withdrawal_index = 0 - to_be_withdrawn_indices = [ - index for index, validator in enumerate(state.validators) - if spec.is_fully_withdrawable_validator(validator, state.balances[index], spec.get_current_epoch(state)) - ] - - if num_expected_withdrawals is not None: - assert len(to_be_withdrawn_indices) == num_expected_withdrawals - else: - num_expected_withdrawals = len(to_be_withdrawn_indices) - - pre_state = state.copy() - - yield 'pre', state - spec.process_full_withdrawals(state) - yield 'post', state - - # Make sure state has NOT been changed - assert state == pre_state - - -@with_eip4844_and_later -@spec_state_test -def test_no_op(spec, state): - # Make one validator withdrawable - set_validator_fully_withdrawable(spec, state, 0) - - yield from run_process_full_withdrawals_no_op(spec, state, 1) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py deleted file mode 100644 index 78be3eb6c..000000000 --- a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_partial_withdrawals.py +++ /dev/null @@ -1,43 +0,0 @@ -from eth2spec.test.context import ( - spec_state_test, - with_eip4844_and_later, -) -from eth2spec.test.helpers.epoch_processing import run_epoch_processing_to -from eth2spec.test.helpers.withdrawals import ( - set_validator_partially_withdrawable, -) - - -def run_process_partial_withdrawals_no_op(spec, state, num_expected_withdrawals=None): - # Run rest of epoch processing before predicting partial withdrawals as - # balance changes can affect withdrawability - run_epoch_processing_to(spec, state, 'process_partial_withdrawals') - - partially_withdrawable_indices = [ - index for index, validator in enumerate(state.validators) - if spec.is_partially_withdrawable_validator(validator, state.balances[index]) - ] - num_partial_withdrawals = min(len(partially_withdrawable_indices), spec.MAX_PARTIAL_WITHDRAWALS_PER_EPOCH) - - if num_expected_withdrawals is not None: - assert num_partial_withdrawals == num_expected_withdrawals - else: - num_expected_withdrawals = num_partial_withdrawals - - pre_state = state.copy() - - yield 'pre', state - spec.process_partial_withdrawals(state) - yield 'post', state - - # Make sure state has NOT been changed - assert state == pre_state - - -@with_eip4844_and_later -@spec_state_test -def test_no_op(spec, state): - validator_index = len(state.validators) // 2 - set_validator_partially_withdrawable(spec, state, validator_index) - - yield from run_process_partial_withdrawals_no_op(spec, state, 1) From 67ba28c9d0f18ae7645219afcabbcfade9352869 Mon Sep 17 00:00:00 2001 From: inphi Date: Thu, 10 Nov 2022 14:13:49 -0500 Subject: [PATCH 20/49] remove eip4844 epoch_processing package --- .../pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 104cba06a6ab38a32d00423ceae55b41ba8e0944 Mon Sep 17 00:00:00 2001 From: inphi Date: Fri, 11 Nov 2022 00:21:56 -0500 Subject: [PATCH 21/49] replace get_blobs_and_kzg_commitments --- specs/eip4844/validator.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index c919c6f18..57a5610ce 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -11,9 +11,7 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) -- [Protocols](#protocols) - - [`ExecutionEngine`](#executionengine) - - [`get_payload`](#get_payload) + - [`get_blobs_and_kzg_commitments`](#get_blobs_and_kzg_commitments) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block and sidecar proposal](#block-and-sidecar-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -39,13 +37,17 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers -## Protocols +### `get_blobs_and_kzg_commitments` -### `ExecutionEngine` +The interface to retrieve blobs and corresponding kzg commitments. -#### `get_payload` +Note: This API is *unstable*. `get_blobs_and_kzg_commitments` and `get_payload` may be unified. +Implementers may also retrieve blobs individually per transaction. -`get_payload` returns the upgraded EIP-4844 `ExecutionPayload` type. +```python +def get_blobs_and_kzg_commitments(payload_id: PayloadId) -> Tuple[Sequence[BLSFieldElement], Sequence[KZGCommitment]]: + ... +``` ## Beacon chain responsibilities From e2a2d68cae9326d0835c583383dca1a216fac3b5 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Fri, 11 Nov 2022 09:50:06 -0800 Subject: [PATCH 22/49] Added a comment to support pre-fork-epoch --- specs/eip4844/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index b431b7927..392dc589a 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -131,6 +131,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` After `EIP4844_FORK_EPOCH`, `BeaconBlocksByRootV2` is replaced by `BeaconBlockAndBlobsSidecarByRootV1` +clients MUST support requesting blocks by root for pre-fork-epoch blocks. Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: From 078bba2e3c33195d19c4a7c93b7769602748bcea Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 8 Nov 2022 11:44:41 -0700 Subject: [PATCH 23/49] clarify deprecation of `beacon_block` gossip topic in 4844 --- specs/eip4844/p2p-interface.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 03363e1b8..269caa8cc 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -17,6 +17,7 @@ The specification of these changes continues in the same format as the network s - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) - [`beacon_block_and_blobs_sidecar`](#beacon_block_and_blobs_sidecar) - [Transitioning the gossip](#transitioning-the-gossip) - [The Req/Resp domain](#the-reqresp-domain) @@ -64,9 +65,9 @@ Some gossip meshes are upgraded in the fork of EIP4844 to support upgraded types ### Topics and messages Topics follow the same specification as in prior upgrades. -All topics remain stable except the beacon block topic which is updated with the modified type. +The `beacon_block` topic is deprecated and replaced by the `beacon_block_and_blobs_sidecar` topic. All other topics remain stable. -The specification around the creation, validation, and dissemination of messages has not changed from the Bellatrix document unless explicitly noted here. +The specification around the creation, validation, and dissemination of messages has not changed from the Capella document unless explicitly noted here. The derivation of the `message-id` remains stable. @@ -81,11 +82,17 @@ The new topics along with the type of the `data` field of a gossipsub message ar EIP4844 introduces a new global topic for beacon block and blobs-sidecars. +##### `beacon_block` + +This topic is deprecated and clients **MUST NOT** expose in their topic set to any peer. Implementers do not need to do +anything beyond simply skip implementation and it is explicitly called out as it is a departure from previous versioning +of this topic. + ##### `beacon_block_and_blobs_sidecar` This topic is used to propagate new signed and coupled beacon blocks and blobs sidecars to all nodes on the networks. -In addition to the gossip validations for the `beacon_block` topic from prior specifications, the following validations MUST pass before forwarding the `signed_beacon_block_and_blobs_sidecar` on the network. +In addition to the gossip validations for the `beacon_block` topic from prior specifications, the following validations MUST pass before forwarding the `signed_beacon_block_and_blobs_sidecar` on the network. Alias `signed_beacon_block = signed_beacon_block_and_blobs_sidecar.beacon_block`, `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. - _[REJECT]_ The KZG commitments of the blobs are all correctly encoded compressed BLS G1 Points. -- i.e. `all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzg_commitments)` From 38669d4b7a418df35fc05598b2c771cacd0d3d60 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 11 Nov 2022 11:20:10 -0700 Subject: [PATCH 24/49] Update specs/eip4844/p2p-interface.md Co-authored-by: Danny Ryan --- specs/eip4844/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 269caa8cc..8ef452236 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -85,7 +85,7 @@ EIP4844 introduces a new global topic for beacon block and blobs-sidecars. ##### `beacon_block` This topic is deprecated and clients **MUST NOT** expose in their topic set to any peer. Implementers do not need to do -anything beyond simply skip implementation and it is explicitly called out as it is a departure from previous versioning +anything beyond simply skip implementation, and it is explicitly called out as it is a departure from previous versioning of this topic. ##### `beacon_block_and_blobs_sidecar` From 6327ffa687aac88d5f11f51294fb1c7afcf4d2d9 Mon Sep 17 00:00:00 2001 From: inphi Date: Fri, 11 Nov 2022 17:24:05 -0500 Subject: [PATCH 25/49] rename excess_blobs --- specs/eip4844/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 69cf4ff31..b9a46f7cb 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -111,7 +111,7 @@ class ExecutionPayload(Container): timestamp: uint64 extra_data: ByteList[MAX_EXTRA_DATA_BYTES] base_fee_per_gas: uint256 - excess_blobs: uint64 # [New in EIP-4844] + excess_data_gas: uint64 # [New in EIP-4844] # Extra payload fields block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] @@ -135,7 +135,7 @@ class ExecutionPayloadHeader(Container): timestamp: uint64 extra_data: ByteList[MAX_EXTRA_DATA_BYTES] base_fee_per_gas: uint256 - excess_blobs: uint64 # [New in EIP-4844] + excess_data_gas: uint64 # [New in EIP-4844] # Extra payload fields block_hash: Hash32 # Hash of execution block transactions_root: Root @@ -273,7 +273,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe timestamp=payload.timestamp, extra_data=payload.extra_data, base_fee_per_gas=payload.base_fee_per_gas, - excess_blobs=payload.excess_blobs, # [New in EIP-4844] + excess_data_gas=payload.excess_data_gas, # [New in EIP-4844] block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), From cd1e1133a5f1830480c5bfc81af22b9c49055859 Mon Sep 17 00:00:00 2001 From: inphi Date: Fri, 11 Nov 2022 17:25:06 -0500 Subject: [PATCH 26/49] excess_data_gas uint256 --- specs/eip4844/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index b9a46f7cb..4a6d4203b 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -111,7 +111,7 @@ class ExecutionPayload(Container): timestamp: uint64 extra_data: ByteList[MAX_EXTRA_DATA_BYTES] base_fee_per_gas: uint256 - excess_data_gas: uint64 # [New in EIP-4844] + excess_data_gas: uint256 # [New in EIP-4844] # Extra payload fields block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] @@ -135,7 +135,7 @@ class ExecutionPayloadHeader(Container): timestamp: uint64 extra_data: ByteList[MAX_EXTRA_DATA_BYTES] base_fee_per_gas: uint256 - excess_data_gas: uint64 # [New in EIP-4844] + excess_data_gas: uint256 # [New in EIP-4844] # Extra payload fields block_hash: Hash32 # Hash of execution block transactions_root: Root From 37144460be5a55249469bfe7beb3d5cb34a13e19 Mon Sep 17 00:00:00 2001 From: inphi Date: Fri, 11 Nov 2022 17:52:57 -0500 Subject: [PATCH 27/49] Fix merge conflict --- tests/core/pyspec/eth2spec/test/helpers/forks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index d6d88876a..82ff12ff1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -5,7 +5,7 @@ from .constants import ( def is_post_fork(a, b): if a == EIP4844: - return b in [PHASE0, ALTAIR, BELLATRIX, EIP4844] + return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844] if a == CAPELLA: return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA] if a == BELLATRIX: From ff340068079a26a28e1695beb43e75b5dc011110 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 9 Nov 2022 14:01:47 +0200 Subject: [PATCH 28/49] Refactor `verify_kzg_proof()` to receive bytes (used in precompile) This way, client devs don't need to convert to field elements themselves, and the KZG library takes care fo it. --- specs/eip4844/polynomial-commitments.md | 23 ++++++++++++++++--- .../test_polynomial_commitments.py | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 2c142c04b..05e19c22a 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -32,6 +32,7 @@ - [KZG](#kzg) - [`blob_to_kzg_commitment`](#blob_to_kzg_commitment) - [`verify_kzg_proof`](#verify_kzg_proof) + - [`verify_kzg_proof_impl`](#verify_kzg_proof_impl) - [`compute_kzg_proof`](#compute_kzg_proof) - [`compute_aggregated_poly_and_commitment`](#compute_aggregated_poly_and_commitment) - [`compute_aggregate_kzg_proof`](#compute_aggregate_kzg_proof) @@ -296,11 +297,27 @@ def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment: ```python def verify_kzg_proof(polynomial_kzg: KZGCommitment, - z: BLSFieldElement, - y: BLSFieldElement, + z: Bytes32, + y: Bytes32, kzg_proof: KZGProof) -> bool: """ Verify KZG proof that ``p(z) == y`` where ``p(z)`` is the polynomial represented by ``polynomial_kzg``. + Receives inputs as bytes. + Public method. + """ + return verify_kzg_proof_impl(polynomial_kzg, bytes_to_bls_field(z), bytes_to_bls_field(y), kzg_proof) +``` + + +#### `verify_kzg_proof_impl` + +```python +def verify_kzg_proof_impl(polynomial_kzg: KZGCommitment, + z: BLSFieldElement, + y: BLSFieldElement, + kzg_proof: KZGProof) -> bool: + """ + Verify KZG proof that ``p(z) == y`` where ``p(z)`` is the polynomial represented by ``polynomial_kzg``. """ # Verify: P - y = Q * (X - z) X_minus_z = bls.add(bls.bytes96_to_G2(KZG_SETUP_G2[1]), bls.multiply(bls.G2, BLS_MODULUS - z)) @@ -390,5 +407,5 @@ def verify_aggregate_kzg_proof(blobs: Sequence[Blob], y = evaluate_polynomial_in_evaluation_form(aggregated_poly, evaluation_challenge) # Verify aggregated proof - return verify_kzg_proof(aggregated_poly_commitment, evaluation_challenge, y, kzg_aggregated_proof) + return verify_kzg_proof_impl(aggregated_poly_commitment, evaluation_challenge, y, kzg_aggregated_proof) ``` diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/polynomial_commitments/test_polynomial_commitments.py index dea6aeb8c..3e9e2cb63 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -17,4 +17,4 @@ def test_verify_kzg_proof(spec, state): proof = spec.compute_kzg_proof(polynomial, x) y = spec.evaluate_polynomial_in_evaluation_form(polynomial, x) - assert spec.verify_kzg_proof(commitment, x, y, proof) + assert spec.verify_kzg_proof_impl(commitment, x, y, proof) From fcac0b5c6976648af7ae825d3e3daba4d04ed179 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 9 Nov 2022 14:02:56 +0200 Subject: [PATCH 29/49] Clarify which functions are public and provided by the KZG library --- specs/eip4844/polynomial-commitments.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 05e19c22a..62bd8441c 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -46,6 +46,8 @@ This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the EIP-4844 specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. +Functions flagged as "Public method" MUST be provided by the underlying KZG library as public functions. All other functions are private functions used internally by the KZG library. + ## Custom types | Name | SSZ equivalent | Description | @@ -290,6 +292,9 @@ KZG core functions. These are also defined in EIP-4844 execution specs. ```python def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment: + """ + Public method. + """ return g1_lincomb(bit_reversal_permutation(KZG_SETUP_LAGRANGE), blob_to_polynomial(blob)) ``` @@ -384,6 +389,9 @@ def compute_aggregated_poly_and_commitment( ```python def compute_aggregate_kzg_proof(blobs: Sequence[Blob]) -> KZGProof: + """ + Public method. + """ commitments = [blob_to_kzg_commitment(blob) for blob in blobs] aggregated_poly, aggregated_poly_commitment, evaluation_challenge = compute_aggregated_poly_and_commitment( blobs, @@ -398,6 +406,9 @@ def compute_aggregate_kzg_proof(blobs: Sequence[Blob]) -> KZGProof: def verify_aggregate_kzg_proof(blobs: Sequence[Blob], expected_kzg_commitments: Sequence[KZGCommitment], kzg_aggregated_proof: KZGCommitment) -> bool: + """ + Public method. + """ aggregated_poly, aggregated_poly_commitment, evaluation_challenge = compute_aggregated_poly_and_commitment( blobs, expected_kzg_commitments, From a5f8a158197827359a9d69937779b09deba9e317 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 15 Nov 2022 10:29:03 -0700 Subject: [PATCH 30/49] Clarify how to transition gossip --- specs/eip4844/p2p-interface.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 8ef452236..7bb652000 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -88,6 +88,8 @@ This topic is deprecated and clients **MUST NOT** expose in their topic set to a anything beyond simply skip implementation, and it is explicitly called out as it is a departure from previous versioning of this topic. +Refer to [the section below](#transitioning-the-gossip) for details on how to transition the gossip. + ##### `beacon_block_and_blobs_sidecar` This topic is used to propagate new signed and coupled beacon blocks and blobs sidecars to all nodes on the networks. From b737e5370fee75af306a93a4d86f4a9ed7ca2220 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Nov 2022 12:12:18 +0100 Subject: [PATCH 31/49] fix state upgrade --- specs/eip4844/fork.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index eaabba916..18d334261 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -66,8 +66,25 @@ Since the `eip4844.BeaconState` format is equal to the `bellatrix.BeaconState` f ```python def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: - # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState + # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState and Capella.ExecutionPayloadHeader epoch = bellatrix.get_current_epoch(pre) + latest_execution_payload_header = ExecutionPayloadHeader( + parent_hash=pre.latest_execution_payload_header.parent_hash, + fee_recipient=pre.latest_execution_payload_header.fee_recipient, + state_root=pre.latest_execution_payload_header.state_root, + receipts_root=pre.latest_execution_payload_header.receipts_root, + logs_bloom=pre.latest_execution_payload_header.logs_bloom, + prev_randao=pre.latest_execution_payload_header.prev_randao, + block_number=pre.latest_execution_payload_header.block_number, + gas_limit=pre.latest_execution_payload_header.gas_limit, + gas_used=pre.latest_execution_payload_header.gas_used, + timestamp=pre.latest_execution_payload_header.timestamp, + extra_data=pre.latest_execution_payload_header.extra_data, + base_fee_per_gas=pre.latest_execution_payload_header.base_fee_per_gas, + excess_blobs=uint64(0), # [New in EIP-4844] + block_hash=pre.latest_execution_payload_header.block_hash, + transactions_root=pre.latest_execution_payload_header.transactions_root, + ) post = BeaconState( # Versioning genesis_time=pre.genesis_time, @@ -108,7 +125,7 @@ def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: current_sync_committee=pre.current_sync_committee, next_sync_committee=pre.next_sync_committee, # Execution-layer - latest_execution_payload_header=pre.latest_execution_payload_header, + latest_execution_payload_header=latest_execution_payload_header, ) return post From e8e860e1f3f4ef2a5d6d5e9c071d93badda0c30a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 16 Nov 2022 10:46:04 -0500 Subject: [PATCH 32/49] make a gossip condition --- specs/eip4844/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 953aad523..010b1e66f 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -97,8 +97,8 @@ Alias `sidecar = signed_beacon_block_and_blobs_sidecar.blobs_sidecar`. - _[IGNORE]_ the `sidecar.beacon_block_slot` is for the current slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `sidecar.beacon_block_slot == block.slot`. - _[REJECT]_ the `sidecar.blobs` are all well formatted, i.e. the `BLSFieldElement` in valid range (`x < BLS_MODULUS`). - _[REJECT]_ The KZG proof is a correctly encoded compressed BLS G1 Point -- i.e. `bls.KeyValidate(blobs_sidecar.kzg_aggregated_proof)` - -Once the sidecar and beacon block are received together, `validate_blobs_sidecar` can unlock the data-availability fork-choice dependency. +- _[REJECT]_ The KZG commitments in the block are valid against the provided blobs sidecar. + -- i.e. `validate_blobs_sidecar(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments, sidecar)` ### Transitioning the gossip From 620943b2756fa28a767a3f6dd470e68449a233de Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Wed, 16 Nov 2022 19:19:39 +0100 Subject: [PATCH 33/49] Update specs/eip4844/fork.md Co-authored-by: Alex Stokes --- specs/eip4844/fork.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index 18d334261..fcae85c25 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -66,7 +66,7 @@ Since the `eip4844.BeaconState` format is equal to the `bellatrix.BeaconState` f ```python def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: - # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState and Capella.ExecutionPayloadHeader + # TODO: if Capella gets scheduled, add sync it with Capella.BeaconState epoch = bellatrix.get_current_epoch(pre) latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=pre.latest_execution_payload_header.parent_hash, From 30dce7f232b1864ff4afdce8f3a2a80e0b69b11c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 02:25:56 +0800 Subject: [PATCH 34/49] Apply suggestions from code review Co-authored-by: Alex Stokes --- specs/eip4844/fork.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/eip4844/fork.md b/specs/eip4844/fork.md index fcae85c25..8e3a727df 100644 --- a/specs/eip4844/fork.md +++ b/specs/eip4844/fork.md @@ -81,9 +81,10 @@ def upgrade_to_eip4844(pre: bellatrix.BeaconState) -> BeaconState: timestamp=pre.latest_execution_payload_header.timestamp, extra_data=pre.latest_execution_payload_header.extra_data, base_fee_per_gas=pre.latest_execution_payload_header.base_fee_per_gas, - excess_blobs=uint64(0), # [New in EIP-4844] + excess_data_gas=uint256(0), # [New in EIP-4844] block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, + withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, ) post = BeaconState( # Versioning From 2a40b513cbbeeaf2fa739d8a74670544a477727b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 10:38:08 -0500 Subject: [PATCH 35/49] Add EIP4844 testgen --- .../capella/fork/test_capella_fork_random.py | 1 + .../pyspec/eth2spec/test/eip4844/__init__.py | 0 .../eth2spec/test/eip4844/fork/__init__.py | 0 .../eip4844/fork/test_eip4844_fork_basic.py | 82 +++++++++++++++++++ .../eip4844/fork/test_eip4844_fork_random.py | 1 + .../eth2spec/test/helpers/eip4844/__init__.py | 0 .../eth2spec/test/helpers/eip4844/fork.py | 63 ++++++++++++++ tests/generators/epoch_processing/main.py | 7 +- tests/generators/finality/main.py | 4 +- tests/generators/fork_choice/main.py | 5 +- tests/generators/forks/main.py | 5 +- tests/generators/genesis/main.py | 4 +- tests/generators/light_client/main.py | 4 +- tests/generators/operations/main.py | 10 ++- tests/generators/rewards/main.py | 4 +- tests/generators/sanity/main.py | 8 +- 16 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/fork/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_basic.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/eip4844/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py diff --git a/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py b/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py index e69de29bb..464090415 100644 --- a/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py +++ b/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py @@ -0,0 +1 @@ +# TODO diff --git a/tests/core/pyspec/eth2spec/test/eip4844/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/fork/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/fork/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_basic.py b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_basic.py new file mode 100644 index 000000000..aca7cb852 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_basic.py @@ -0,0 +1,82 @@ +from eth2spec.test.context import ( + with_phases, + with_custom_state, + with_presets, + spec_test, with_state, + low_balances, misc_balances, large_validator_set, +) +from eth2spec.test.utils import with_meta_tags +from eth2spec.test.helpers.constants import ( + CAPELLA, EIP4844, + MINIMAL, +) +from eth2spec.test.helpers.state import ( + next_epoch, + next_epoch_via_block, +) +from eth2spec.test.helpers.eip4844.fork import ( + EIP4844_FORK_TEST_META_TAGS, + run_fork_test, +) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_base_state(spec, phases, state): + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_next_epoch(spec, phases, state): + next_epoch(spec, state) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_next_epoch_with_block(spec, phases, state): + next_epoch_via_block(spec, state) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_many_next_epoch(spec, phases, state): + for _ in range(3): + next_epoch(spec, state) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@spec_test +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_random_low_balances(spec, phases, state): + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@spec_test +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_random_misc_balances(spec, phases, state): + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@with_presets([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@spec_test +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_fork_random_large_validator_set(spec, phases, state): + yield from run_fork_test(phases[EIP4844], state) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py new file mode 100644 index 000000000..464090415 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py @@ -0,0 +1 @@ +# TODO diff --git a/tests/core/pyspec/eth2spec/test/helpers/eip4844/__init__.py b/tests/core/pyspec/eth2spec/test/helpers/eip4844/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py b/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py new file mode 100644 index 000000000..1818557c0 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py @@ -0,0 +1,63 @@ +from eth2spec.test.helpers.constants import ( + EIP4844, +) + + +EIP4844_FORK_TEST_META_TAGS = { + 'fork': EIP4844, +} + + +def run_fork_test(post_spec, pre_state): + yield 'pre', pre_state + + post_state = post_spec.upgrade_to_eip4844(pre_state) + + # Stable fields + stable_fields = [ + 'genesis_time', 'genesis_validators_root', 'slot', + # History + 'latest_block_header', 'block_roots', 'state_roots', 'historical_roots', + # Eth1 + 'eth1_data', 'eth1_data_votes', 'eth1_deposit_index', + # Registry + 'validators', 'balances', + # Randomness + 'randao_mixes', + # Slashings + 'slashings', + # Participation + 'previous_epoch_participation', 'current_epoch_participation', + # Finality + 'justification_bits', 'previous_justified_checkpoint', 'current_justified_checkpoint', 'finalized_checkpoint', + # Inactivity + 'inactivity_scores', + # Sync + 'current_sync_committee', 'next_sync_committee', + # Execution + 'latest_execution_payload_header', + ] + for field in stable_fields: + assert getattr(pre_state, field) == getattr(post_state, field) + + # Modified fields + modified_fields = ['fork'] + for field in modified_fields: + assert getattr(pre_state, field) != getattr(post_state, field) + + assert len(pre_state.validators) == len(post_state.validators) + for pre_validator, post_validator in zip(pre_state.validators, post_state.validators): + stable_validator_fields = [ + 'pubkey', 'withdrawal_credentials', + 'effective_balance', + 'slashed', + 'activation_eligibility_epoch', 'activation_epoch', 'exit_epoch', 'withdrawable_epoch', + ] + for field in stable_validator_fields: + assert getattr(pre_validator, field) == getattr(post_validator, field) + + assert pre_state.fork.current_version == post_state.fork.previous_version + assert post_state.fork.current_version == post_spec.config.EIP4844_FORK_VERSION + assert post_state.fork.epoch == post_spec.get_current_epoch(post_state) + + yield 'post', post_state diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index fc57fbe4e..84421e749 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -31,6 +31,10 @@ if __name__ == "__main__": # so no additional tests required. capella_mods = bellatrix_mods + # No epoch-processing changes in EIP4844 and previous testing repeats with new types, + # so no additional tests required. + eip4844_mods = capella_mods + # TODO Custody Game testgen is disabled for now # custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [ # 'reveal_deadlines', @@ -43,6 +47,7 @@ if __name__ == "__main__": ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index cb40e7cc9..de5af9b11 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -7,12 +7,14 @@ if __name__ == "__main__": altair_mods = phase_0_mods # No additional Altair specific finality tests bellatrix_mods = altair_mods # No additional Bellatrix specific finality tests capella_mods = bellatrix_mods # No additional Capella specific finality tests + eip4844_mods = capella_mods # No additional EIP4844 specific finality tests all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="finality", all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 7982902c5..40e19a8ac 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -18,11 +18,14 @@ if __name__ == "__main__": ]} bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) capella_mods = bellatrix_mods # No additional Capella specific fork choice tests + eip4844_mods = capella_mods # No additional Capella specific fork choice tests + all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="fork_choice", all_mods=all_mods) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index 628114979..42f3f3a1f 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,13 +1,14 @@ from typing import Iterable from eth2spec.test.helpers.constants import ( - PHASE0, ALTAIR, BELLATRIX, CAPELLA, + PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844, MINIMAL, MAINNET, ) from eth2spec.test.helpers.typing import SpecForkName, PresetBaseName from eth2spec.test.altair.fork import test_altair_fork_basic, test_altair_fork_random from eth2spec.test.bellatrix.fork import test_bellatrix_fork_basic, test_bellatrix_fork_random from eth2spec.test.capella.fork import test_capella_fork_basic, test_capella_fork_random +from eth2spec.test.eip4844.fork import test_eip4844_fork_basic, test_eip4844_fork_random from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests @@ -39,6 +40,8 @@ def _get_fork_tests_providers(): yield create_provider(test_bellatrix_fork_random, preset, ALTAIR, BELLATRIX) yield create_provider(test_capella_fork_basic, preset, BELLATRIX, CAPELLA) yield create_provider(test_capella_fork_random, preset, BELLATRIX, CAPELLA) + yield create_provider(test_eip4844_fork_basic, preset, CAPELLA, EIP4844) + yield create_provider(test_eip4844_fork_random, preset, CAPELLA, EIP4844) if __name__ == "__main__": diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index c8f5a83d5..a5c4eba9d 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -16,11 +16,13 @@ if __name__ == "__main__": ]} bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) capella_mods = bellatrix_mods # No additional Capella specific genesis tests + eip4844_mods = capella_mods # No additional EIP4844 specific genesis tests all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="genesis", all_mods=all_mods) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 0ff3fe786..5d45bf39d 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, EIP4844 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators @@ -10,11 +10,13 @@ if __name__ == "__main__": ]} bellatrix_mods = altair_mods capella_mods = bellatrix_mods + eip4844_mods = capella_mods all_mods = { ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="light_client", all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 1977ec217..01e63c30d 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -31,6 +31,13 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) + _new_eip4844_mods = {key: 'eth2spec.test.eip4844.block_processing.test_process_' + key for key in [ + 'bls_to_execution_change', + 'withdrawals', + ]} + eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods) + + # TODO Custody Game testgen is disabled for now # _new_custody_game_mods = {key: 'eth2spec.test.custody_game.block_processing.test_process_' + key for key in [ # 'attestation', @@ -46,6 +53,7 @@ if __name__ == "__main__": ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="operations", all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 33763144b..8958074bc 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -16,12 +16,14 @@ if __name__ == "__main__": # Transaction fees are part of the execution-layer. bellatrix_mods = altair_mods capella_mods = bellatrix_mods + eip4844_mods = capella_mods all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="rewards", all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index bb3c954c2..9dd6d7ac0 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods @@ -23,11 +23,17 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) + _new_eip4844_mods = {key: 'eth2spec.test.eip4844.sanity.test_' + key for key in [ + 'blocks', + ]} + eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods) + all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="sanity", all_mods=all_mods) From 2f0157afad880d4a5d6b2c08743ca10513ee7546 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 11:22:06 -0500 Subject: [PATCH 36/49] Fix auto-rebase leftover --- setup.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/setup.py b/setup.py index 205e8c400..de446bb1c 100644 --- a/setup.py +++ b/setup.py @@ -624,7 +624,6 @@ class EIP4844SpecBuilder(CapellaSpecBuilder): @classmethod def imports(cls, preset_name: str): return super().imports(preset_name) + f''' -from eth2spec.utils import kzg from eth2spec.capella import {preset_name} as capella ''' @@ -638,20 +637,6 @@ T = TypeVar('T') # For generic function @classmethod def sundry_functions(cls) -> str: return super().sundry_functions() + '\n\n' + ''' -# TODO: for mainnet, load pre-generated trusted setup file to reduce building time. -# TESTING_FIELD_ELEMENTS_PER_BLOB is hardcoded copy from minimal presets -TESTING_FIELD_ELEMENTS_PER_BLOB = 4 -TESTING_SECRET = 1337 -TESTING_KZG_SETUP_G1 = kzg.generate_setup(bls.G1, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB) -TESTING_KZG_SETUP_G2 = kzg.generate_setup(bls.G2, TESTING_SECRET, TESTING_FIELD_ELEMENTS_PER_BLOB) -TESTING_KZG_SETUP_LAGRANGE = kzg.get_lagrange(TESTING_KZG_SETUP_G1) - -KZG_SETUP_G1 = [bls.G1_to_bytes48(p) for p in TESTING_KZG_SETUP_G1] -KZG_SETUP_G2 = [bls.G2_to_bytes96(p) for p in TESTING_KZG_SETUP_G2] -KZG_SETUP_LAGRANGE = TESTING_KZG_SETUP_LAGRANGE -ROOTS_OF_UNITY = kzg.compute_roots_of_unity(TESTING_FIELD_ELEMENTS_PER_BLOB) - - # # Temporarily disable Withdrawals functions for EIP4844 testnets # From 0c8885570e42bb790e70b87deeda0fedf353102b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 11:23:02 -0500 Subject: [PATCH 37/49] Add Capella and EIP4844 fork random tests --- .../capella/fork/test_capella_fork_random.py | 85 ++++++++++++++++++- .../eip4844/fork/test_eip4844_fork_random.py | 85 ++++++++++++++++++- 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py b/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py index 464090415..87b4667e0 100644 --- a/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py +++ b/tests/core/pyspec/eth2spec/test/capella/fork/test_capella_fork_random.py @@ -1 +1,84 @@ -# TODO +from random import Random + +from eth2spec.test.context import ( + with_phases, + with_custom_state, + with_presets, + spec_test, with_state, + low_balances, misc_balances, large_validator_set, +) +from eth2spec.test.utils import with_meta_tags +from eth2spec.test.helpers.constants import ( + BELLATRIX, CAPELLA, + MINIMAL, +) +from eth2spec.test.helpers.capella.fork import ( + CAPELLA_FORK_TEST_META_TAGS, + run_fork_test, +) +from eth2spec.test.helpers.random import randomize_state + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_state +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_0(spec, phases, state): + randomize_state(spec, state, rng=Random(1010)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_state +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_1(spec, phases, state): + randomize_state(spec, state, rng=Random(2020)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_state +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_2(spec, phases, state): + randomize_state(spec, state, rng=Random(3030)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_state +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_3(spec, phases, state): + randomize_state(spec, state, rng=Random(4040)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_low_balances(spec, phases, state): + randomize_state(spec, state, rng=Random(5050)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@spec_test +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_misc_balances(spec, phases, state): + randomize_state(spec, state, rng=Random(6060)) + yield from run_fork_test(phases[CAPELLA], state) + + +@with_phases(phases=[BELLATRIX], other_phases=[CAPELLA]) +@with_presets([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") +@spec_test +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS) +def test_capella_fork_random_large_validator_set(spec, phases, state): + randomize_state(spec, state, rng=Random(7070)) + yield from run_fork_test(phases[CAPELLA], state) diff --git a/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py index 464090415..a22de4b59 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/fork/test_eip4844_fork_random.py @@ -1 +1,84 @@ -# TODO +from random import Random + +from eth2spec.test.context import ( + with_phases, + with_custom_state, + with_presets, + spec_test, with_state, + low_balances, misc_balances, large_validator_set, +) +from eth2spec.test.utils import with_meta_tags +from eth2spec.test.helpers.constants import ( + CAPELLA, EIP4844, + MINIMAL, +) +from eth2spec.test.helpers.eip4844.fork import ( + EIP4844_FORK_TEST_META_TAGS, + run_fork_test, +) +from eth2spec.test.helpers.random import randomize_state + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_0(spec, phases, state): + randomize_state(spec, state, rng=Random(1010)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_1(spec, phases, state): + randomize_state(spec, state, rng=Random(2020)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_2(spec, phases, state): + randomize_state(spec, state, rng=Random(3030)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_state +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_3(spec, phases, state): + randomize_state(spec, state, rng=Random(4040)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_low_balances(spec, phases, state): + randomize_state(spec, state, rng=Random(5050)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@spec_test +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_misc_balances(spec, phases, state): + randomize_state(spec, state, rng=Random(6060)) + yield from run_fork_test(phases[EIP4844], state) + + +@with_phases(phases=[CAPELLA], other_phases=[EIP4844]) +@with_presets([MINIMAL], + reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated") +@spec_test +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE) +@with_meta_tags(EIP4844_FORK_TEST_META_TAGS) +def test_eip4844_fork_random_large_validator_set(spec, phases, state): + randomize_state(spec, state, rng=Random(7070)) + yield from run_fork_test(phases[EIP4844], state) From 438a7e55ea92fa61bda5554f975ebbfd631e55fa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 11 Nov 2022 12:57:41 -0800 Subject: [PATCH 38/49] Conver `roots_of_unity_brp` to integers --- specs/eip4844/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 2c142c04b..4e7a3a31e 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -276,7 +276,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, result = 0 for i in range(width): - result += div(int(polynomial[i]) * int(roots_of_unity_brp[i]), (int(z) - roots_of_unity_brp[i])) + result += div(int(polynomial[i]) * int(roots_of_unity_brp[i]), (int(z) - int(roots_of_unity_brp[i]))) result = result * (pow(z, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS return result ``` From 5c5639f8f21c17bc8b37d8383b06647dc168d91d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 15 Nov 2022 08:45:05 -0800 Subject: [PATCH 39/49] fix lint --- tests/generators/operations/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 01e63c30d..bb879a647 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -37,7 +37,6 @@ if __name__ == "__main__": ]} eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods) - # TODO Custody Game testgen is disabled for now # _new_custody_game_mods = {key: 'eth2spec.test.custody_game.block_processing.test_process_' + key for key in [ # 'attestation', From 812618cdaf7f40b7e5a28f5cc4419b5ebc4c80ed Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 03:37:24 +0800 Subject: [PATCH 40/49] Fix fork test verifier --- tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py b/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py index 1818557c0..ed4ae057d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py +++ b/tests/core/pyspec/eth2spec/test/helpers/eip4844/fork.py @@ -34,14 +34,14 @@ def run_fork_test(post_spec, pre_state): 'inactivity_scores', # Sync 'current_sync_committee', 'next_sync_committee', - # Execution - 'latest_execution_payload_header', + # Withdrawals + 'next_withdrawal_index', 'next_withdrawal_validator_index', ] for field in stable_fields: assert getattr(pre_state, field) == getattr(post_state, field) # Modified fields - modified_fields = ['fork'] + modified_fields = ['fork', 'latest_execution_payload_header'] for field in modified_fields: assert getattr(pre_state, field) != getattr(post_state, field) From dd8b67075e71a22d4a13d2ffd1512ccdab9beaef Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 04:51:19 +0800 Subject: [PATCH 41/49] Add note comment --- specs/eip4844/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 4a6d4203b..afab21dca 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -89,7 +89,7 @@ class BeaconBlockBody(Container): voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] sync_aggregate: SyncAggregate # Execution - execution_payload: ExecutionPayload + execution_payload: ExecutionPayload # [Modified in EIP-4844] bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in EIP-4844] ``` From d12c2a3b738859934de7942bdaa2dbd72ff00f47 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 04:55:59 +0800 Subject: [PATCH 42/49] Add EIP4844 random tests --- .../eth2spec/test/eip4844/random/__init__.py | 0 .../test/eip4844/random/test_random.py | 438 ++++++++++++++++++ .../test/utils/randomized_block_tests.py | 24 + tests/generators/random/Makefile | 2 + tests/generators/random/generate.py | 11 +- tests/generators/random/main.py | 6 +- 6 files changed, 479 insertions(+), 2 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/random/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip4844/random/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/random/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/random/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/eip4844/random/test_random.py b/tests/core/pyspec/eth2spec/test/eip4844/random/test_random.py new file mode 100644 index 000000000..b90b858b2 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip4844/random/test_random.py @@ -0,0 +1,438 @@ +""" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +""" + +from eth2spec.test.helpers.constants import EIP4844 +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_0(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_1(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_2(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_3(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_4(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_5(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_6(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_7(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_8(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_9(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_10(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_11(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_12(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_13(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_14(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([EIP4844]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_15(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_eip4844 + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_eip4844', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_eip4844'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 5504a53d7..37fbdc417 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -20,6 +20,9 @@ from eth2spec.test.helpers.random import ( randomize_state as randomize_state_helper, patch_state_to_non_leaking, ) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, +) from eth2spec.test.helpers.state import ( next_slot, next_epoch, @@ -78,6 +81,17 @@ def randomize_state_capella(spec, state, stats, exit_fraction=0.1, slash_fractio stats, exit_fraction=exit_fraction, slash_fraction=slash_fraction) + # TODO: randomize withdrawals + return scenario_state + + +def randomize_state_eip4844(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): + scenario_state = randomize_state_capella(spec, + state, + stats, + exit_fraction=exit_fraction, + slash_fraction=slash_fraction) + # TODO: randomize execution payload return scenario_state @@ -215,6 +229,16 @@ def random_block_capella(spec, state, signed_blocks, scenario_state, rng=Random( return block +def random_block_eip4844(spec, state, signed_blocks, scenario_state, rng=Random(3456)): + block = random_block_capella(spec, state, signed_blocks, scenario_state) + # TODO: more commitments. blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] + opaque_tx, _, blob_kzg_commitments = get_sample_opaque_tx(spec, blob_count=1) + block.body.execution_payload.transactions = [opaque_tx] + block.body.blob_kzg_commitments = blob_kzg_commitments + + return block + + # validations def no_op_validation(_spec, _state): diff --git a/tests/generators/random/Makefile b/tests/generators/random/Makefile index 8b77a3617..f57221ab4 100644 --- a/tests/generators/random/Makefile +++ b/tests/generators/random/Makefile @@ -6,7 +6,9 @@ all: rm -f ../../core/pyspec/eth2spec/test/altair/random/test_random.py rm -f ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py rm -f ../../core/pyspec/eth2spec/test/capella/random/test_random.py + rm -f ../../core/pyspec/eth2spec/test/eip4844/random/test_random.py python3 generate.py phase0 > ../../core/pyspec/eth2spec/test/phase0/random/test_random.py python3 generate.py altair > ../../core/pyspec/eth2spec/test/altair/random/test_random.py python3 generate.py bellatrix > ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py python3 generate.py capella > ../../core/pyspec/eth2spec/test/capella/random/test_random.py + python3 generate.py eip4844 > ../../core/pyspec/eth2spec/test/eip4844/random/test_random.py diff --git a/tests/generators/random/generate.py b/tests/generators/random/generate.py index 39d43b0aa..129d670fd 100644 --- a/tests/generators/random/generate.py +++ b/tests/generators/random/generate.py @@ -21,10 +21,12 @@ from eth2spec.test.utils.randomized_block_tests import ( randomize_state_altair, randomize_state_bellatrix, randomize_state_capella, + randomize_state_eip4844, random_block, random_block_altair_with_cycling_sync_committee_participation, random_block_bellatrix, random_block_capella, + random_block_eip4844, last_slot_in_epoch, random_slot_in_epoch, penultimate_slot_in_epoch, @@ -34,7 +36,7 @@ from eth2spec.test.utils.randomized_block_tests import ( transition_to_leaking, transition_without_leak, ) -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 # Ensure this many blocks are present in *each* randomized scenario @@ -272,5 +274,12 @@ if __name__ == "__main__": state_randomizer=randomize_state_capella, block_randomizer=random_block_capella, ) + if EIP4844 in sys.argv: + did_generate = True + run_generate_tests_to_std_out( + EIP4844, + state_randomizer=randomize_state_eip4844, + block_randomizer=random_block_eip4844, + ) if not did_generate: warnings.warn("no phase given for test generation") diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py index 9983da96c..e36678771 100644 --- a/tests/generators/random/main.py +++ b/tests/generators/random/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP4844 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators @@ -15,12 +15,16 @@ if __name__ == "__main__": capella_mods = {key: 'eth2spec.test.capella.random.test_' + key for key in [ 'random', ]} + eip4844_mods = {key: 'eth2spec.test.eip4844.random.test_' + key for key in [ + 'random', + ]} all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="random", all_mods=all_mods) From 8824259131ce00c564a6e4fbdd683316140b7f4c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 16:33:50 -0500 Subject: [PATCH 43/49] Fix --preset-list argument and enhance error output --- .../gen_helpers/gen_base/gen_runner.py | 25 ++++++++++++++----- tests/generators/README.md | 4 +-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index dccb9313f..0a831a592 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -4,7 +4,6 @@ import time import shutil import argparse from pathlib import Path -from filelock import FileLock import sys import json from typing import Iterable, AnyStr, Any, Callable @@ -13,6 +12,7 @@ from ruamel.yaml import ( YAML, ) +from filelock import FileLock from snappy import compress from eth2spec.test import context @@ -141,6 +141,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): tprov.prepare() for test_case in tprov.make_cases(): + # If preset list is assigned, filter by presets. + if len(presets) != 0 and test_case.preset_name not in presets: + continue + case_dir = ( Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) / Path(test_case.runner_name) / Path(test_case.handler_name) @@ -179,7 +183,16 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): try: fn(case_dir) except IOError as e: - sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}') + error_message = ( + f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' + ) + # Write to error log file + with log_file.open("a+") as f: + f.write(error_message) + traceback.print_exc(file=f) + f.write('\n') + + sys.exit(error_message) meta = dict() @@ -210,13 +223,13 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if not written_part: print(f"test case {case_dir} did not produce any test case parts") except Exception as e: - print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") - traceback.print_exc() - # Write to log file + error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" + # Write to error log file with log_file.open("a+") as f: - f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + f.write(error_message) traceback.print_exc(file=f) f.write('\n') + traceback.print_exc() else: # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. if not written_part: diff --git a/tests/generators/README.md b/tests/generators/README.md index a84331dcb..0146ca35e 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -186,7 +186,7 @@ if __name__ == "__main__": ALTAIR: altair_mods, } - run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) + run_state_test_generators(runner_name="sanity", all_mods=all_mods) ``` Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. @@ -210,7 +210,7 @@ To add a new test generator that builds `New Tests`: with any dependencies it may need. Leave it empty if your generator has none. 3. Your generator is assumed to have a `main.py` file in its root. By adding the base generator to your requirements, you can make a generator really easily. See docs below. -4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. +4. Your generator is called with `-o some/file/path/for_testing/can/be_anything --preset-list mainnet minimal`. The base generator helps you handle this; you only have to define test case providers. 5. Finally, add any linting or testing commands to the [circleci config file](../../.circleci/config.yml) if desired to increase code quality. From aac1af7db0342d385886b380d70200b1289c8afb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 17:27:59 +0800 Subject: [PATCH 44/49] Add `test_fail_double_bls_changes_in_same_block` --- .../test/capella/sanity/test_blocks.py | 87 ++++++++++++------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index 3c406e524..e42551f6f 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -21,9 +21,13 @@ from eth2spec.test.helpers.withdrawals import ( from eth2spec.test.helpers.voluntary_exits import prepare_signed_exits +# +# BLSToExecutionChange +# + @with_phases([CAPELLA]) @spec_state_test -def test_successful_bls_change(spec, state): +def test_success_bls_change(spec, state): index = 0 signed_address_change = get_signed_address_change(spec, state, validator_index=index) pre_credentials = state.validators[index].withdrawal_credentials @@ -44,6 +48,58 @@ def test_successful_bls_change(spec, state): assert post_credentials[12:] == signed_address_change.message.to_execution_address +@with_phases([CAPELLA]) +@spec_state_test +def test_success_exit_and_bls_change(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + index = 0 + signed_address_change = get_signed_address_change(spec, state, validator_index=index) + signed_exit = prepare_signed_exits(spec, state, [index])[0] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.voluntary_exits.append(signed_exit) + block.body.bls_to_execution_changes.append(signed_address_change) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + validator = state.validators[index] + balance = state.balances[index] + current_epoch = spec.get_current_epoch(state) + assert not spec.is_fully_withdrawable_validator(validator, balance, current_epoch) + assert validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + assert spec.is_fully_withdrawable_validator(validator, balance, validator.withdrawable_epoch) + + +@with_phases([CAPELLA]) +@spec_state_test +def test_fail_double_bls_changes_in_same_block(spec, state): + index = 0 + signed_address_change = get_signed_address_change(spec, state, validator_index=index) + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + + # Double BLSToExecutionChange of the same validator + for _ in range(2): + block.body.bls_to_execution_changes.append(signed_address_change) + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +# +# Withdrawals +# + @with_phases([CAPELLA]) @spec_state_test def test_full_withdrawal_in_epoch_transition(spec, state): @@ -112,35 +168,6 @@ def test_many_partial_withdrawals_in_epoch_transition(spec, state): assert len(spec.get_expected_withdrawals(state)) == 1 -@with_phases([CAPELLA]) -@spec_state_test -def test_exit_and_bls_change(spec, state): - # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - index = 0 - signed_address_change = get_signed_address_change(spec, state, validator_index=index) - signed_exit = prepare_signed_exits(spec, state, [index])[0] - - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - block.body.voluntary_exits.append(signed_exit) - block.body.bls_to_execution_changes.append(signed_address_change) - - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - validator = state.validators[index] - balance = state.balances[index] - current_epoch = spec.get_current_epoch(state) - assert not spec.is_fully_withdrawable_validator(validator, balance, current_epoch) - assert validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - assert spec.is_fully_withdrawable_validator(validator, balance, validator.withdrawable_epoch) - - def _perform_valid_withdrawal(spec, state): fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( spec, state, num_partial_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD * 4, From 82007c33d58f1ab4dbbcbfbb799f451fdb1ed628 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 18 Nov 2022 08:01:21 -0700 Subject: [PATCH 45/49] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 000e3207c..94dbaa2cf 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.3.0-alpha.0 +1.3.0-alpha.1 From b78bd1fae036dcd74bd3c15246734a76751a2f6d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 23:14:57 +0800 Subject: [PATCH 46/49] Add other double-operations (attestations, deposits) tests --- .../test/capella/sanity/test_blocks.py | 2 +- .../test/phase0/sanity/test_blocks.py | 72 +++++++++++++++++-- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index e42551f6f..0e910ed76 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -79,7 +79,7 @@ def test_success_exit_and_bls_change(spec, state): @with_phases([CAPELLA]) @spec_state_test -def test_fail_double_bls_changes_in_same_block(spec, state): +def test_fail_double_bls_changes_same_block(spec, state): index = 0 signed_address_change = get_signed_address_change(spec, state, validator_index=index) yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index b94a01f7f..29757074d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -433,7 +433,7 @@ def test_proposer_slashing(spec, state): @with_all_phases @spec_state_test -def test_double_same_proposer_slashings_same_block(spec, state): +def test_invalid_double_same_proposer_slashings_same_block(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) slashed_index = proposer_slashing.signed_header_1.message.proposer_index assert not state.validators[slashed_index].slashed @@ -450,7 +450,7 @@ def test_double_same_proposer_slashings_same_block(spec, state): @with_all_phases @spec_state_test -def test_double_similar_proposer_slashings_same_block(spec, state): +def test_invalid_double_similar_proposer_slashings_same_block(spec, state): slashed_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # Same validator, but different slashable offences in the same block @@ -549,7 +549,7 @@ def test_attester_slashing(spec, state): @with_all_phases @spec_state_test -def test_duplicate_attester_slashing(spec, state): +def test_invalid_double_same_attester_slashing_same_block(spec, state): if spec.MAX_ATTESTER_SLASHINGS < 2: return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") @@ -744,6 +744,27 @@ def test_deposit_in_block(spec, state): assert state.validators[validator_index].pubkey == pubkeys[validator_index] +@with_all_phases +@spec_state_test +def test_invalid_double_same_deposit_same_block(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + + # The same deposit of the same validator + for _ in range(2): + block.body.deposits.append(deposit) + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + @with_all_phases @spec_state_test def test_deposit_top_up(spec, state): @@ -831,6 +852,49 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root +@with_all_phases +@spec_state_test +def test_double_same_attestation_same_block(spec, state): + next_epoch(spec, state) + + yield 'pre', state + + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + + attestation = get_valid_attestation(spec, state, index=index, signed=True) + + if not is_post_altair(spec): + pre_current_attestations_len = len(state.current_epoch_attestations) + + # Add to state via block transition + for _ in range(2): + attestation_block.body.attestations.append(attestation) + signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) + + if not is_post_altair(spec): + assert len(state.current_epoch_attestations) == pre_current_attestations_len + 2 + # Epoch transition should move to previous_epoch_attestations + pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + else: + pre_current_epoch_participation_root = spec.hash_tree_root(state.current_epoch_participation) + + epoch_block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + signed_epoch_block = state_transition_and_sign_block(spec, state, epoch_block) + + yield 'blocks', [signed_attestation_block, signed_epoch_block] + yield 'post', state + + if not is_post_altair(spec): + assert len(state.current_epoch_attestations) == 0 + assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + else: + for index in range(len(state.validators)): + assert state.current_epoch_participation[index] == spec.ParticipationFlags(0b0000_0000) + assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root + + # After SHARDING is enabled, a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, # exceeding the minimal-config randao mixes memory size. # Applies to all voluntary-exit sanity block tests. @@ -866,7 +930,7 @@ def test_voluntary_exit(spec, state): @with_all_phases @spec_state_test -def test_double_validator_exit_same_block(spec, state): +def test_invalid_double_validator_exit_same_block(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit From 9f4adfde9df34e1e066bcf5804a6311008ee3213 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 18 Nov 2022 23:23:21 +0800 Subject: [PATCH 47/49] Add `test_invalid_two_bls_changes_of_different_addresses_same_validator_same_block` --- .../test/capella/sanity/test_blocks.py | 26 ++++++++++++++++++- .../test/helpers/bls_to_execution_changes.py | 7 +++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index 0e910ed76..490253bef 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -79,7 +79,7 @@ def test_success_exit_and_bls_change(spec, state): @with_phases([CAPELLA]) @spec_state_test -def test_fail_double_bls_changes_same_block(spec, state): +def test_invalid_double_bls_changes_same_block(spec, state): index = 0 signed_address_change = get_signed_address_change(spec, state, validator_index=index) yield 'pre', state @@ -96,6 +96,30 @@ def test_fail_double_bls_changes_same_block(spec, state): yield 'post', None +@with_phases([CAPELLA]) +@spec_state_test +def test_invalid_two_bls_changes_of_different_addresses_same_validator_same_block(spec, state): + index = 0 + + signed_address_change_1 = get_signed_address_change(spec, state, validator_index=index, + to_execution_address=b'\x12' * 20) + signed_address_change_2 = get_signed_address_change(spec, state, validator_index=index, + to_execution_address=b'\x34' * 20) + assert signed_address_change_1 != signed_address_change_2 + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + + block.body.bls_to_execution_changes.append(signed_address_change_1) + block.body.bls_to_execution_changes.append(signed_address_change_2) + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + # # Withdrawals # diff --git a/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py b/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py index 61c84b515..446fa9710 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py +++ b/tests/core/pyspec/eth2spec/test/helpers/bls_to_execution_changes.py @@ -2,7 +2,7 @@ from eth2spec.utils import bls from eth2spec.test.helpers.keys import pubkeys, privkeys, pubkey_to_privkey -def get_signed_address_change(spec, state, validator_index=None, withdrawal_pubkey=None): +def get_signed_address_change(spec, state, validator_index=None, withdrawal_pubkey=None, to_execution_address=None): if validator_index is None: validator_index = 0 @@ -13,11 +13,14 @@ def get_signed_address_change(spec, state, validator_index=None, withdrawal_pubk else: withdrawal_privkey = pubkey_to_privkey[withdrawal_pubkey] + if to_execution_address is None: + to_execution_address = b'\x42' * 20 + domain = spec.get_domain(state, spec.DOMAIN_BLS_TO_EXECUTION_CHANGE) address_change = spec.BLSToExecutionChange( validator_index=validator_index, from_bls_pubkey=withdrawal_pubkey, - to_execution_address=b'\x42' * 20, + to_execution_address=to_execution_address, ) signing_root = spec.compute_signing_root(address_change, domain) From fbb13f0cf0ed5bb3d82e93ad5c1172432f7e0d74 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 19 Nov 2022 00:03:29 +0800 Subject: [PATCH 48/49] Apply naming suggestion from @djrtwo --- .../eth2spec/test/capella/sanity/test_blocks.py | 2 +- .../eth2spec/test/phase0/sanity/test_blocks.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index 490253bef..229d8ecc5 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -79,7 +79,7 @@ def test_success_exit_and_bls_change(spec, state): @with_phases([CAPELLA]) @spec_state_test -def test_invalid_double_bls_changes_same_block(spec, state): +def test_invalid_duplicate_bls_changes_same_block(spec, state): index = 0 signed_address_change = get_signed_address_change(spec, state, validator_index=index) yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 29757074d..0e19e65b7 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -433,7 +433,7 @@ def test_proposer_slashing(spec, state): @with_all_phases @spec_state_test -def test_invalid_double_same_proposer_slashings_same_block(spec, state): +def test_invalid_duplicate_proposer_slashings_same_block(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) slashed_index = proposer_slashing.signed_header_1.message.proposer_index assert not state.validators[slashed_index].slashed @@ -450,7 +450,7 @@ def test_invalid_double_same_proposer_slashings_same_block(spec, state): @with_all_phases @spec_state_test -def test_invalid_double_similar_proposer_slashings_same_block(spec, state): +def test_invalid_similar_proposer_slashings_same_block(spec, state): slashed_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # Same validator, but different slashable offences in the same block @@ -549,7 +549,7 @@ def test_attester_slashing(spec, state): @with_all_phases @spec_state_test -def test_invalid_double_same_attester_slashing_same_block(spec, state): +def test_invalid_duplicate_attester_slashing_same_block(spec, state): if spec.MAX_ATTESTER_SLASHINGS < 2: return dump_skipping_message("Skip test if config cannot handle multiple AttesterSlashings per block") @@ -746,7 +746,7 @@ def test_deposit_in_block(spec, state): @with_all_phases @spec_state_test -def test_invalid_double_same_deposit_same_block(spec, state): +def test_invalid_duplicate_deposit_same_block(spec, state): validator_index = len(state.validators) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) @@ -854,7 +854,7 @@ def test_attestation(spec, state): @with_all_phases @spec_state_test -def test_double_same_attestation_same_block(spec, state): +def test_duplicate_attestation_same_block(spec, state): next_epoch(spec, state) yield 'pre', state @@ -930,7 +930,7 @@ def test_voluntary_exit(spec, state): @with_all_phases @spec_state_test -def test_invalid_double_validator_exit_same_block(spec, state): +def test_invalid_duplicate_validator_exit_same_block(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit From b71ad2f8be5375e2c6aa15fc0be9351d0b3f340b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 19 Nov 2022 01:55:01 +0800 Subject: [PATCH 49/49] Add EIP4844 sync tests --- tests/generators/sync/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py index eba3ac152..8fb395053 100644 --- a/tests/generators/sync/main.py +++ b/tests/generators/sync/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA +from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, EIP4844 if __name__ == "__main__": @@ -7,10 +7,12 @@ if __name__ == "__main__": 'optimistic', ]} capella_mods = bellatrix_mods + eip4844_mods = capella_mods all_mods = { BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, + EIP4844: eip4844_mods, } run_state_test_generators(runner_name="sync", all_mods=all_mods)