From 21cf61ea8f339c36c276003844f002e70f491741 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 15 Dec 2022 11:28:20 +0700 Subject: [PATCH 01/36] De-duplicate range references in BlobSidecarsByRange --- specs/deneb/p2p-interface.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 107c90b9a..51180f8a2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -256,18 +256,18 @@ The request MUST be encoded as an SSZ-container. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. -Clients MUST keep a record of signed blobs sidecars seen on the epoch range -`[max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]` +Let `blob_serve_range` be `[max(current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH), current_epoch]`. +Clients MUST keep a record of signed blob sidecars seen on the epoch range `blob_serve_range` where `current_epoch` is defined by the current wall-clock time, and clients MUST support serving requests of blobs on this range. -Peers that are unable to reply to blob sidecar requests within the `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` -epoch range SHOULD respond with error code `3: ResourceUnavailable`. +Peers that are unable to reply to blob sidecar requests within the +range `blob_serve_range` SHOULD respond with error code `3: ResourceUnavailable`. Such peers that are unable to successfully reply to this range of requests MAY get descored or disconnected at any time. *Note*: The above requirement implies that nodes that start from a recent weak subjectivity checkpoint -MUST backfill the local blobs database to at least epoch `current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` +MUST backfill the local blobs database to at least the range `blob_serve_range` to be fully compliant with `BlobSidecarsByRange` requests. *Note*: Although clients that bootstrap from a weak subjectivity checkpoint can begin From 1c35eb1c3303fe1e0b101323106d766d8f848cd6 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:37:18 +0800 Subject: [PATCH 02/36] Lock voluntary_exit domain on capella --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7e14fa951..e5eb12006 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1903,7 +1903,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From 47f078fc1fdbcc6274d42ee891718ed73bc731d9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 11 Mar 2023 20:28:49 +0800 Subject: [PATCH 03/36] Move change to deneb --- specs/deneb/beacon-chain.md | 44 +++++++++++++++++++++++++++++++++++- specs/phase0/beacon-chain.md | 2 +- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a0ac783b7..672d0a633 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -203,7 +203,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) + process_operations(state, block.body) # [Modified in Deneb] process_sync_aggregate(state, block.body.sync_aggregate) process_blob_kzg_commitments(state, block.body) # [New in Deneb] ``` @@ -245,6 +245,48 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ) ``` +#### Modified `process_operations` + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) # [Modified in Deneb] + for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) +``` + +##### Modified `process_voluntary_exit` + +```python +def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: + voluntary_exit = signed_voluntary_exit.message + validator = state.validators[voluntary_exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify exit has not been initiated + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= voluntary_exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD + # Verify signature + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) # [Modified in Deneb] + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) + # Initiate exit + initiate_validator_exit(state, voluntary_exit.validator_index) +``` + + #### Blob KZG commitments ```python diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index e5eb12006..7e14fa951 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1903,7 +1903,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From df4ba47e633dff7707d7a531136cfa785a65f5b0 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 13 Mar 2023 08:51:02 +0800 Subject: [PATCH 04/36] Update beacon-chain.md --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 672d0a633..fd983f29e 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -279,7 +279,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, min(voluntary_exit.epoch, CAPELLA_FORK_EPOCH)) # [Modified in Deneb] + domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) # [Modified in Deneb] signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit From 1a4db24919dbdf438c5f45b2bb4c51874266d6f6 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 17 May 2023 11:49:46 -0600 Subject: [PATCH 05/36] Update fork-choice.md Remove duplicate check --- specs/deneb/fork-choice.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index ce9973f13..1390b3d4e 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -30,8 +30,6 @@ This is the modification of the fork choice accompanying the Deneb upgrade. def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], blobs: Sequence[Blob], proofs: Sequence[KZGProof]) -> None: - assert len(expected_kzg_commitments) == len(blobs) == len(proofs) - assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) ``` From 32036d84a3542ce5b9bd4a2f4e288edfb6267a25 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 19 May 2023 23:32:49 +0800 Subject: [PATCH 06/36] Fix tests --- specs/deneb/beacon-chain.md | 5 ++- .../test_process_voluntary_exit.py | 28 +++++++----- .../test/deneb/block_processing/__init__.py | 0 .../test_process_voluntary_exit.py | 45 +++++++++++++++++++ .../test_process_voluntary_exit.py | 44 ++++++++++++++++++ .../eth2spec/test/helpers/voluntary_exits.py | 20 +++++---- tests/generators/operations/main.py | 6 ++- 7 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index f1fe48e82..7d2f9bdc7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,6 +30,8 @@ - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) + - [Modified `process_operations`](#modified-process_operations) + - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -281,7 +283,8 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) # [Modified in Deneb] + # [Modified in Deneb] + domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py index f4fcaac68..12b554da5 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py @@ -2,6 +2,12 @@ from eth2spec.test.context import ( spec_state_test, always_bls, with_bellatrix_and_later, + with_phases, +) +from eth2spec.test.helpers.constants import ( + BELLATRIX, + CAPELLA, + DENEB, ) from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import ( @@ -12,8 +18,10 @@ from eth2spec.test.helpers.voluntary_exits import ( sign_voluntary_exit, ) +BELLATRIX_AND_CAPELLA = [BELLATRIX, CAPELLA] -def _run_voluntary_exit_processing_test( + +def run_voluntary_exit_processing_test( spec, state, fork_version, @@ -51,7 +59,7 @@ def _run_voluntary_exit_processing_test( @spec_state_test @always_bls def test_invalid_voluntary_exit_with_current_fork_version_is_before_fork_epoch(spec, state): - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.current_version, @@ -60,11 +68,11 @@ def test_invalid_voluntary_exit_with_current_fork_version_is_before_fork_epoch(s ) -@with_bellatrix_and_later +@with_phases(BELLATRIX_AND_CAPELLA) @spec_state_test @always_bls def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, state): - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.current_version, @@ -72,13 +80,13 @@ def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, ) -@with_bellatrix_and_later +@with_phases([BELLATRIX, CAPELLA, DENEB]) @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): assert state.fork.previous_version != state.fork.current_version - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.previous_version, @@ -86,13 +94,13 @@ def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, st ) -@with_bellatrix_and_later +@with_phases(BELLATRIX_AND_CAPELLA) @spec_state_test @always_bls def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): assert state.fork.previous_version != state.fork.current_version - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=state.fork.previous_version, @@ -107,7 +115,7 @@ def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_ep def test_invalid_voluntary_exit_with_genesis_fork_version_is_before_fork_epoch(spec, state): assert spec.config.GENESIS_FORK_VERSION not in (state.fork.previous_version, state.fork.current_version) - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=spec.config.GENESIS_FORK_VERSION, @@ -122,7 +130,7 @@ def test_invalid_voluntary_exit_with_genesis_fork_version_is_before_fork_epoch(s def test_invalid_voluntary_exit_with_genesis_fork_version_not_is_before_fork_epoch(spec, state): assert spec.config.GENESIS_FORK_VERSION not in (state.fork.previous_version, state.fork.current_version) - yield from _run_voluntary_exit_processing_test( + yield from run_voluntary_exit_processing_test( spec, state, fork_version=spec.config.GENESIS_FORK_VERSION, diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py new file mode 100644 index 000000000..371fcfed4 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -0,0 +1,45 @@ +from eth2spec.test.context import ( + always_bls, + spec_state_test, + with_phases, + with_deneb_and_later, +) +from eth2spec.test.helpers.constants import ( + DENEB, +) +from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( + run_voluntary_exit_processing_test, +) + + +@with_deneb_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.current_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_phases([DENEB]) +@spec_state_test +@always_bls +def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py new file mode 100644 index 000000000..4128a1181 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py @@ -0,0 +1,44 @@ +from eth2spec.test.context import ( + always_bls, + spec_state_test, + with_eip6110_and_later, +) +from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( + run_voluntary_exit_processing_test, +) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_invalid_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + valid=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py b/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py index cac101dff..2e8139db6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -1,29 +1,31 @@ from random import Random from eth2spec.utils import bls from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.forks import is_post_deneb from eth2spec.test.helpers.keys import privkeys def prepare_signed_exits(spec, state, indices, fork_version=None): - if fork_version is None: - domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) - else: - domain = spec.compute_domain(spec.DOMAIN_VOLUNTARY_EXIT, fork_version, state.genesis_validators_root) - def create_signed_exit(index): - exit = spec.VoluntaryExit( + voluntary_exit = spec.VoluntaryExit( epoch=spec.get_current_epoch(state), validator_index=index, ) - signing_root = spec.compute_signing_root(exit, domain) - return spec.SignedVoluntaryExit(message=exit, signature=bls.Sign(privkeys[index], signing_root)) + return sign_voluntary_exit(spec, state, voluntary_exit, privkeys[index], fork_version=fork_version) return [create_signed_exit(index) for index in indices] def sign_voluntary_exit(spec, state, voluntary_exit, privkey, fork_version=None): if fork_version is None: - domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + if is_post_deneb(spec): + domain = spec.compute_domain( + spec.DOMAIN_VOLUNTARY_EXIT, + spec.config.CAPELLA_FORK_VERSION, + state.genesis_validators_root, + ) + else: + domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) else: domain = spec.compute_domain(spec.DOMAIN_VOLUNTARY_EXIT, fork_version, state.genesis_validators_root) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc2217917..2cff8bdf6 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -36,10 +36,14 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods + _new_deneb_mods = {key: 'eth2spec.test.deneb.block_processing.test_process_' + key for key in [ + 'voluntary_exit', + ]} + deneb_mods = combine_mods(_new_deneb_mods, capella_mods) _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', + 'voluntary_exit', ]} eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) From 296efec2561a44580a7c321bf22c51aff2e52714 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 00:39:13 +0800 Subject: [PATCH 07/36] Add `data_gas_used` field to `ExecutionPayload` --- specs/_features/eip6110/beacon-chain.md | 3 +++ specs/_features/eip6110/fork.md | 1 + specs/deneb/beacon-chain.md | 3 +++ specs/deneb/fork.md | 1 + specs/deneb/light-client/fork.md | 1 + specs/deneb/light-client/full-node.md | 1 + specs/deneb/light-client/sync-protocol.md | 2 +- tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 3 +++ 8 files changed, 14 insertions(+), 1 deletion(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index e1384fdae..95dc7fd3c 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -92,6 +92,7 @@ class ExecutionPayload(Container): transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] excess_data_gas: uint256 + data_gas_used: uint256 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -117,6 +118,7 @@ class ExecutionPayloadHeader(Container): transactions_root: Root withdrawals_root: Root excess_data_gas: uint256 + data_gas_used: uint256 deposit_receipts_root: Root # [New in EIP6110] ``` @@ -269,6 +271,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), excess_data_gas=payload.excess_data_gas, + data_gas_used=payload.data_gas_used, deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP6110] ) ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 2145a9d1a..8a603b530 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -89,6 +89,7 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, excess_data_gas=uint256(0), + data_gas_used=uint256(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 175a80edf..26206384a 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -127,6 +127,7 @@ class ExecutionPayload(Container): transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] excess_data_gas: uint256 # [New in Deneb] + data_gas_used: uint256 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -151,6 +152,7 @@ class ExecutionPayloadHeader(Container): transactions_root: Root withdrawals_root: Root excess_data_gas: uint256 # [New in Deneb] + data_gas_used: uint256 # [New in Deneb] ``` ## Helper functions @@ -258,6 +260,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), excess_data_gas=payload.excess_data_gas, # [New in Deneb] + data_gas_used=payload.data_gas_used, # [New in Deneb] ) ``` diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 23b3f23c7..8c7fd5319 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -84,6 +84,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, excess_data_gas=uint256(0), # [New in Deneb] + data_gas_used=uint256(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 46a093028..2ae59cc7e 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -42,6 +42,7 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, excess_data_gas=uint256(0), # [New in Deneb] + data_gas_used=uint256(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 275194036..0ef0ee96f 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -50,6 +50,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: # [New in Deneb] if epoch >= DENEB_FORK_EPOCH: execution_header.excess_data_gas = payload.excess_data_gas + execution_header.data_gas_used = payload.data_gas_used execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) else: diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index c691a113d..194be0ae4 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -68,7 +68,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: # [New in Deneb] if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint256(0) or header.execution.data_gas_used != uint256(0): return False if epoch < CAPELLA_FORK_EPOCH: diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 747d678ef..1fa3f4c4b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -32,6 +32,7 @@ def get_execution_payload_header(spec, execution_payload): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) if is_post_deneb(spec): payload_header.excess_data_gas = execution_payload.excess_data_gas + payload_header.data_gas_used = execution_payload.data_gas_used if is_post_eip6110(spec): payload_header.deposit_receipts_root = spec.hash_tree_root(execution_payload.deposit_receipts) return payload_header @@ -99,6 +100,7 @@ def compute_el_header_block_hash(spec, if is_post_deneb(spec): # excess_data_gas execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) + execution_payload_header_rlp.append((big_endian_int, payload_header.data_gas_used)) if is_post_eip6110(spec): # deposit_receipts_root assert deposit_receipts_trie_root is not None @@ -202,6 +204,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): payload.withdrawals = spec.get_expected_withdrawals(state) if is_post_deneb(spec): payload.excess_data_gas = 0 + payload.data_gas_used = 0 if is_post_eip6110(spec): # just to be clear payload.deposit_receipts = [] From f6fe8e27292c121c133a882998b1dacf900556ee Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 20:36:58 +0800 Subject: [PATCH 08/36] Change `ExecutionPayload.excess_data_gas` type from `uint256` to `uint64` --- specs/_features/eip6110/beacon-chain.md | 4 ++-- specs/_features/eip6110/fork.md | 2 +- specs/_features/eip6110/light-client/sync-protocol.md | 2 +- specs/deneb/beacon-chain.md | 4 ++-- specs/deneb/fork.md | 2 +- specs/deneb/light-client/fork.md | 2 +- specs/deneb/light-client/sync-protocol.md | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index e1384fdae..3e260006f 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -91,7 +91,7 @@ class ExecutionPayload(Container): block_hash: Hash32 transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 + excess_data_gas: uint64 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -116,7 +116,7 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 + excess_data_gas: uint64 deposit_receipts_root: Root # [New in EIP6110] ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 2145a9d1a..468f432e6 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -88,7 +88,7 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), + excess_data_gas=uint64(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md index bcb9d50e4..b454a2ebe 100644 --- a/specs/_features/eip6110/light-client/sync-protocol.md +++ b/specs/_features/eip6110/light-client/sync-protocol.md @@ -70,7 +70,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: return False if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint64(0): return False if epoch < CAPELLA_FORK_EPOCH: diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 175a80edf..ba84d33c0 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -126,7 +126,7 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 # [New in Deneb] + excess_data_gas: uint64 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -150,7 +150,7 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 # [New in Deneb] + excess_data_gas: uint64 # [New in Deneb] ``` ## Helper functions diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 23b3f23c7..a4014b10a 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,7 +83,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] + excess_data_gas=uint64(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 46a093028..dba50beb1 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,7 +41,7 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] + excess_data_gas=uint64(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index c691a113d..697d12c04 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -68,7 +68,7 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: # [New in Deneb] if epoch < DENEB_FORK_EPOCH: - if header.execution.excess_data_gas != uint256(0): + if header.execution.excess_data_gas != uint64(0): return False if epoch < CAPELLA_FORK_EPOCH: From 653e03e70223335d270940cc81640ff875080c7d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 30 May 2023 21:12:47 +0800 Subject: [PATCH 09/36] Fix ToC --- specs/deneb/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a929db859..335723626 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -33,8 +33,8 @@ - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - - [`process_execution_payload`](#process_execution_payload) - - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) + - [Modified `process_execution_payload`](#modified-process_execution_payload) + - [Modified `process_voluntary_exit`](#modified-process_voluntary_exit) - [Testing](#testing) @@ -218,7 +218,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, #### Execution payload -##### `process_execution_payload` +##### Modified `process_execution_payload` ```python def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: @@ -262,7 +262,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi ) ``` -##### Modified `process_voluntary_exit` +#### Modified `process_voluntary_exit` ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: From 7bc5d51ce84d47e9b495225c02ebade181438e0e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 18:09:23 +0800 Subject: [PATCH 10/36] Add non-zero data_gas_used case --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index ba0797a04..a6025092b 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,7 +16,7 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, data_gas_used=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -24,6 +24,7 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.excess_data_gas = excess_data_gas + block.body.execution_payload.data_gas_used = data_gas_used block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) if valid: From 60721b9d8aeab0f83dd317edceb84c437cfc3848 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 22:21:50 +0800 Subject: [PATCH 11/36] switch order --- specs/_features/eip6110/beacon-chain.md | 6 +++--- specs/_features/eip6110/fork.md | 2 +- specs/deneb/beacon-chain.md | 6 +++--- specs/deneb/fork.md | 2 +- specs/deneb/light-client/fork.md | 2 +- specs/deneb/light-client/full-node.md | 2 +- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 4 ++-- .../core/pyspec/eth2spec/test/helpers/execution_payload.py | 6 +++--- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 95dc7fd3c..dc68d4ab0 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -91,8 +91,8 @@ class ExecutionPayload(Container): block_hash: Hash32 transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 data_gas_used: uint256 + excess_data_gas: uint256 deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` @@ -117,8 +117,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 data_gas_used: uint256 + excess_data_gas: uint256 deposit_receipts_root: Root # [New in EIP6110] ``` @@ -270,8 +270,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - excess_data_gas=payload.excess_data_gas, data_gas_used=payload.data_gas_used, + excess_data_gas=payload.excess_data_gas, deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP6110] ) ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index 8a603b530..b702472b8 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -88,8 +88,8 @@ def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), data_gas_used=uint256(0), + excess_data_gas=uint256(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 26206384a..d8815762e 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -126,8 +126,8 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - excess_data_gas: uint256 # [New in Deneb] data_gas_used: uint256 # [New in Deneb] + excess_data_gas: uint256 # [New in Deneb] ``` #### `ExecutionPayloadHeader` @@ -151,8 +151,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - excess_data_gas: uint256 # [New in Deneb] data_gas_used: uint256 # [New in Deneb] + excess_data_gas: uint256 # [New in Deneb] ``` ## Helper functions @@ -259,8 +259,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - excess_data_gas=payload.excess_data_gas, # [New in Deneb] data_gas_used=payload.data_gas_used, # [New in Deneb] + excess_data_gas=payload.excess_data_gas, # [New in Deneb] ) ``` diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 8c7fd5319..910a6dbff 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,8 +83,8 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] data_gas_used=uint256(0), # [New in Deneb] + excess_data_gas=uint256(0), # [New in Deneb] ) post = BeaconState( # Versioning diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 2ae59cc7e..c10bec2b9 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,8 +41,8 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - excess_data_gas=uint256(0), # [New in Deneb] data_gas_used=uint256(0), # [New in Deneb] + excess_data_gas=uint256(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 0ef0ee96f..69ba0d3ba 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -49,8 +49,8 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: # [New in Deneb] if epoch >= DENEB_FORK_EPOCH: - execution_header.excess_data_gas = payload.excess_data_gas execution_header.data_gas_used = payload.data_gas_used + execution_header.excess_data_gas = payload.excess_data_gas execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) else: diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index a6025092b..2af330efb 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,15 +16,15 @@ from eth2spec.test.helpers.sharding import ( ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, data_gas_used=1, valid=True): +def run_block_with_blobs(spec, state, blob_count, data_gas_used=1, excess_data_gas=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.data_gas_used = data_gas_used + block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) if valid: diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 1fa3f4c4b..b08b1975e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -31,8 +31,8 @@ def get_execution_payload_header(spec, execution_payload): if is_post_capella(spec): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) if is_post_deneb(spec): - payload_header.excess_data_gas = execution_payload.excess_data_gas payload_header.data_gas_used = execution_payload.data_gas_used + payload_header.excess_data_gas = execution_payload.excess_data_gas if is_post_eip6110(spec): payload_header.deposit_receipts_root = spec.hash_tree_root(execution_payload.deposit_receipts) return payload_header @@ -99,8 +99,8 @@ def compute_el_header_block_hash(spec, execution_payload_header_rlp.append((Binary(32, 32), withdrawals_trie_root)) if is_post_deneb(spec): # excess_data_gas - execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) execution_payload_header_rlp.append((big_endian_int, payload_header.data_gas_used)) + execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) if is_post_eip6110(spec): # deposit_receipts_root assert deposit_receipts_trie_root is not None @@ -203,8 +203,8 @@ def build_empty_execution_payload(spec, state, randao_mix=None): if is_post_capella(spec): payload.withdrawals = spec.get_expected_withdrawals(state) if is_post_deneb(spec): - payload.excess_data_gas = 0 payload.data_gas_used = 0 + payload.excess_data_gas = 0 if is_post_eip6110(spec): # just to be clear payload.deposit_receipts = [] From c9b84ecc1a75d3a3d3d028955a2e5646cc62b54a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 23:40:46 +0800 Subject: [PATCH 12/36] Use `verify_blob_kzg_proof_batch` directly in `is_data_available` and remove `validate_blob` wrapper --- specs/deneb/fork-choice.md | 14 +---- specs/deneb/p2p-interface.md | 2 +- .../deneb/unittests/fork_choice/__init__.py | 0 .../fork_choice/test_validate_blobs.py | 54 ------------------- 4 files changed, 3 insertions(+), 67 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 1390b3d4e..4636b42a1 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -8,7 +8,6 @@ - [Introduction](#introduction) - [Containers](#containers) - [Helpers](#helpers) - - [`validate_blobs`](#validate_blobs) - [`is_data_available`](#is_data_available) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [`on_block`](#on_block) @@ -24,19 +23,10 @@ This is the modification of the fork choice accompanying the Deneb upgrade. ## Helpers -#### `validate_blobs` - -```python -def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], - blobs: Sequence[Blob], - proofs: Sequence[KZGProof]) -> None: - assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) -``` - #### `is_data_available` The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. -Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `validate_blobs`. +Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. The block MUST NOT be considered valid until all valid `Blob`s have been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `Blob`s have subsequently been pruned. @@ -52,7 +42,7 @@ def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZ if isinstance(blobs, str) or isinstance(proofs, str): return True - validate_blobs(blob_kzg_commitments, blobs, proofs) + assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) return True ``` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3c6f3c88a..f7c962e46 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -264,7 +264,7 @@ Requests blob sidecars in the slot range `[start_slot, start_slot + count)`, lea The response is unsigned, i.e. `BlobSidecarsByRange`, as the signature of the beacon block proposer may not be available beyond the initial distribution via gossip. -Before consuming the next response chunk, the response reader SHOULD verify the blob sidecar is well-formatted and correct w.r.t. the expected KZG commitments through `validate_blobs`. +Before consuming the next response chunk, the response reader SHOULD verify the blob sidecar is well-formatted and correct w.r.t. the expected KZG commitments through `verify_blob_kzg_proof_batch`. `BlobSidecarsByRange` is primarily used to sync blobs that may have been missed on gossip and to sync within the `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` window. diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py deleted file mode 100644 index 0d7bd53e5..000000000 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/fork_choice/test_validate_blobs.py +++ /dev/null @@ -1,54 +0,0 @@ -from eth2spec.test.helpers.state import ( - state_transition_and_sign_block, -) -from eth2spec.test.helpers.block import ( - build_empty_block_for_next_slot -) -from eth2spec.test.context import ( - spec_state_test, - with_deneb_and_later, -) -from eth2spec.test.helpers.execution_payload import ( - compute_el_block_hash, -) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, -) - - -def _run_validate_blobs(spec, state, blob_count): - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, kzg_proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - state_transition_and_sign_block(spec, state, block) - - blob_sidecars = spec.get_blob_sidecars(block, blobs, kzg_proofs) - blobs = [sidecar.blob for sidecar in blob_sidecars] - kzg_proofs = [sidecar.kzg_proof for sidecar in blob_sidecars] - spec.validate_blobs(blob_kzg_commitments, blobs, kzg_proofs) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_zero_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=0) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_one_blob(spec, state): - _run_validate_blobs(spec, state, blob_count=1) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_two_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=2) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_max_blobs(spec, state): - _run_validate_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) From 4bf3895864ceb670a4dfba16b7c4bf3ead0aea55 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Jun 2023 23:53:01 +0800 Subject: [PATCH 13/36] Update VERSION.txt to 1.4.0-alpha.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 779260991..2a6fa03c0 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.0 +1.4.0-alpha.1 From e1a55238f2576aa4829cdf32357d530eb7fa0301 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 2 Jun 2023 00:06:47 +0800 Subject: [PATCH 14/36] Delete eip6110_mods --- tests/generators/light_client/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index c6b0e01b9..178ef4f44 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -15,14 +15,12 @@ if __name__ == "__main__": ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) deneb_mods = capella_mods - eip6110_mods = deneb_mods all_mods = { ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, - EIP6110: eip6110_mods, } run_state_test_generators(runner_name="light_client", all_mods=all_mods) From dde9301610797cbbeb45278b5511f03c382140d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 2 Jun 2023 00:15:45 +0800 Subject: [PATCH 15/36] fix lint --- tests/generators/light_client/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 178ef4f44..cfe34aee4 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators From f5e00848d0f74cd440b996ae716d58947b868173 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 7 Jun 2023 11:59:51 +0600 Subject: [PATCH 16/36] Remove assert from is_data_available --- specs/deneb/fork-choice.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 283e76b1d..8b3c22423 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -42,8 +42,7 @@ def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZ if isinstance(blobs, str) or isinstance(proofs, str): return True - assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) - return True + return verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, proofs) ``` ## Updated fork-choice handlers From a547d47264e40adca4b06fce1954e22ba84e0c27 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 7 Jun 2023 17:45:39 +0800 Subject: [PATCH 17/36] specially mark the EIP4844 changes --- specs/deneb/beacon-chain.md | 38 ++++++++++++----------- specs/deneb/fork-choice.md | 4 ++- specs/deneb/fork.md | 6 ++-- specs/deneb/light-client/fork.md | 4 +-- specs/deneb/light-client/full-node.md | 2 +- specs/deneb/light-client/sync-protocol.md | 2 +- specs/deneb/p2p-interface.md | 14 +++++++-- specs/deneb/polynomial-commitments.md | 2 +- specs/deneb/validator.md | 2 ++ 9 files changed, 44 insertions(+), 30 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 489ae6151..5c3b4e8a4 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -41,16 +41,15 @@ ## Introduction -This upgrade adds blobs to the beacon chain as part of Deneb. This is an extension of the Capella upgrade. - -The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. +Deneb is a consensus-layer upgrade containing a number of features. Including: +* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner. ## Custom types | Name | SSZ equivalent | Description | | - | - | - | -| `VersionedHash` | `Bytes32` | | -| `BlobIndex` | `uint64` | | +| `VersionedHash` | `Bytes32` | [New in Deneb:EIP4844] | +| `BlobIndex` | `uint64` | [New in Deneb:EIP4844] | ## Constants @@ -73,8 +72,11 @@ The blob transactions are packed into the execution payload by the EL/builder wi | Name | Value | Description | | - | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | [New in Deneb:EIP4844] hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | [New in Deneb:EIP4844] Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | + +*Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted +and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. ## Configuration @@ -100,9 +102,9 @@ class BeaconBlockBody(Container): voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] sync_aggregate: SyncAggregate # Execution - execution_payload: ExecutionPayload # [Modified in Deneb] + execution_payload: ExecutionPayload # [Modified in Deneb:EIP4844] bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] - blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb] + blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb:EIP4844] ``` #### `ExecutionPayload` @@ -126,8 +128,8 @@ class ExecutionPayload(Container): block_hash: Hash32 # Hash of execution block transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - data_gas_used: uint64 # [New in Deneb] - excess_data_gas: uint64 # [New in Deneb] + data_gas_used: uint64 # [New in Deneb:EIP4844] + excess_data_gas: uint64 # [New in Deneb:EIP4844] ``` #### `ExecutionPayloadHeader` @@ -151,8 +153,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 # Hash of execution block transactions_root: Root withdrawals_root: Root - data_gas_used: uint64 # [New in Deneb] - excess_data_gas: uint64 # [New in Deneb] + data_gas_used: uint64 # [New in Deneb:EIP4844] + excess_data_gas: uint64 # [New in Deneb:EIP4844] ``` ## Helper functions @@ -205,7 +207,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, if not self.is_valid_block_hash(new_payload_request.execution_payload): return False - # [New in Deneb] + # [New in Deneb:EIP4844] if not self.is_valid_versioned_hashes(new_payload_request): return False @@ -232,11 +234,11 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) - # [New in Deneb] Verify commitments are under limit + # [New in Deneb:EIP4844] Verify commitments are under limit assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid - # [Modified in Deneb] Pass `versioned_hashes` to Execution Engine + # [Modified in Deneb:EIP4844] Pass `versioned_hashes` to Execution Engine versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) @@ -259,8 +261,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - data_gas_used=payload.data_gas_used, # [New in Deneb] - excess_data_gas=payload.excess_data_gas, # [New in Deneb] + data_gas_used=payload.data_gas_used, # [New in Deneb:EIP4844] + excess_data_gas=payload.excess_data_gas, # [New in Deneb:EIP4844] ) ``` diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 283e76b1d..60e05f826 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -25,6 +25,8 @@ This is the modification of the fork choice accompanying the Deneb upgrade. #### `is_data_available` +[New in Deneb:EIP4844] + The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. @@ -76,7 +78,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: ) assert store.finalized_checkpoint.root == finalized_checkpoint_block - # [New in Deneb] + # [New in Deneb:EIP4844] # Check if blob data is available # If not, this block MAY be queued and subsequently considered when blob data becomes available assert is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments) diff --git a/specs/deneb/fork.md b/specs/deneb/fork.md index 7fc80275d..ffced6a59 100644 --- a/specs/deneb/fork.md +++ b/specs/deneb/fork.md @@ -83,8 +83,8 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, - data_gas_used=uint64(0), # [New in Deneb] - excess_data_gas=uint64(0), # [New in Deneb] + data_gas_used=uint64(0), # [New in Deneb:EIP4844] + excess_data_gas=uint64(0), # [New in Deneb:EIP4844] ) post = BeaconState( # Versioning @@ -126,7 +126,7 @@ def upgrade_to_deneb(pre: capella.BeaconState) -> BeaconState: current_sync_committee=pre.current_sync_committee, next_sync_committee=pre.next_sync_committee, # Execution-layer - latest_execution_payload_header=latest_execution_payload_header, # [Modified in Deneb] + latest_execution_payload_header=latest_execution_payload_header, # [Modified in Deneb:EIP4844] # Withdrawals next_withdrawal_index=pre.next_withdrawal_index, next_withdrawal_validator_index=pre.next_withdrawal_validator_index, diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 2fd410fc5..f4fd1b396 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,8 +41,8 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - data_gas_used=uint64(0), # [New in Deneb] - excess_data_gas=uint64(0), # [New in Deneb] + data_gas_used=uint64(0), # [New in Deneb:EIP4844] + excess_data_gas=uint64(0), # [New in Deneb:EIP4844] ), execution_branch=pre.execution_branch, ) diff --git a/specs/deneb/light-client/full-node.md b/specs/deneb/light-client/full-node.md index 69ba0d3ba..876a6c258 100644 --- a/specs/deneb/light-client/full-node.md +++ b/specs/deneb/light-client/full-node.md @@ -47,7 +47,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: withdrawals_root=hash_tree_root(payload.withdrawals), ) - # [New in Deneb] + # [New in Deneb:EIP4844] if epoch >= DENEB_FORK_EPOCH: execution_header.data_gas_used = payload.data_gas_used execution_header.excess_data_gas = payload.excess_data_gas diff --git a/specs/deneb/light-client/sync-protocol.md b/specs/deneb/light-client/sync-protocol.md index b7b9b5ab1..38909ddbf 100644 --- a/specs/deneb/light-client/sync-protocol.md +++ b/specs/deneb/light-client/sync-protocol.md @@ -66,7 +66,7 @@ def get_lc_execution_root(header: LightClientHeader) -> Root: def is_valid_light_client_header(header: LightClientHeader) -> bool: epoch = compute_epoch_at_slot(header.beacon.slot) - # [New in Deneb] + # [New in Deneb:EIP4844] if epoch < DENEB_FORK_EPOCH: if header.execution.data_gas_used != uint64(0) or header.execution.excess_data_gas != uint64(0): return False diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index caf2b5bae..e9123c309 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -50,6 +50,8 @@ The specification of these changes continues in the same format as the network s #### `BlobSidecar` +[New in Deneb:EIP4844] + ```python class BlobSidecar(Container): block_root: Root @@ -64,6 +66,8 @@ class BlobSidecar(Container): #### `SignedBlobSidecar` +[New in Deneb:EIP4844] + ```python class SignedBlobSidecar(Container): message: BlobSidecar @@ -72,6 +76,8 @@ class SignedBlobSidecar(Container): #### `BlobIdentifier` +[New in Deneb:EIP4844] + ```python class BlobIdentifier(Container): block_root: Root @@ -107,7 +113,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar | Name | Message Type | | - | - | -| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` (new) | +| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` [New in Deneb:EIP4844] | ##### Global topics @@ -124,6 +130,8 @@ New validation: ###### `blob_sidecar_{subnet_id}` +[New in Deneb:EIP4844] + This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: @@ -191,7 +199,7 @@ No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/1/` -New in deneb. +[New in Deneb:EIP4844] The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -240,7 +248,7 @@ Clients MAY limit the number of blocks and sidecars in the response. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` -New in deneb. +[New in Deneb:EIP4844] The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e800d8cc2..8a26b9ade 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -336,7 +336,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, ### KZG -KZG core functions. These are also defined in Deneb execution specs. +KZG core functions. These are also defined in Deneb execution specs for EIP-4844. #### `blob_to_kzg_commitment` diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index ae5f26673..5af7c4938 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -98,6 +98,8 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments +[New in Deneb:EIP4844] + 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` via `get_payload(payload_id).blobs_bundle`. From bab93e8d44b27ccf41a0ddc4993eaae95b144939 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 7 Jun 2023 18:40:02 +0800 Subject: [PATCH 18/36] specially mark EIP7044 changes --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index ccf0d6933..7ce8bb1e7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -282,7 +282,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify the validator has been active long enough assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature - # [Modified in Deneb] + # [Modified in Deneb:EIP7044] domain = compute_domain(DOMAIN_VOLUNTARY_EXIT, CAPELLA_FORK_VERSION, state.genesis_validators_root) signing_root = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) From b7e21d6caf94980318f4892d35096c2061b10f2c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 8 Jun 2023 15:05:46 +0800 Subject: [PATCH 19/36] PR feedback and make the marks italic --- specs/deneb/beacon-chain.md | 8 ++++---- specs/deneb/fork-choice.md | 2 +- specs/deneb/p2p-interface.md | 12 ++++++------ specs/deneb/polynomial-commitments.md | 4 ++-- specs/deneb/validator.md | 10 +++++++--- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 5c3b4e8a4..41ce76bd2 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -48,8 +48,8 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | SSZ equivalent | Description | | - | - | - | -| `VersionedHash` | `Bytes32` | [New in Deneb:EIP4844] | -| `BlobIndex` | `uint64` | [New in Deneb:EIP4844] | +| `VersionedHash` | `Bytes32` | *[New in Deneb:EIP4844]* | +| `BlobIndex` | `uint64` | *[New in Deneb:EIP4844]* | ## Constants @@ -72,8 +72,8 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | Value | Description | | - | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | [New in Deneb:EIP4844] hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | [New in Deneb:EIP4844] Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | *[New in Deneb:EIP4844]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | *Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 60e05f826..cc7373c09 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -25,7 +25,7 @@ This is the modification of the fork choice accompanying the Deneb upgrade. #### `is_data_available` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. Initially, verification requires every verifying actor to retrieve all matching `Blob`s and `KZGProof`s, and validate them with `verify_blob_kzg_proof_batch`. diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index e9123c309..0a61ef800 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -50,7 +50,7 @@ The specification of these changes continues in the same format as the network s #### `BlobSidecar` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class BlobSidecar(Container): @@ -66,7 +66,7 @@ class BlobSidecar(Container): #### `SignedBlobSidecar` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class SignedBlobSidecar(Container): @@ -76,7 +76,7 @@ class SignedBlobSidecar(Container): #### `BlobIdentifier` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* ```python class BlobIdentifier(Container): @@ -130,7 +130,7 @@ New validation: ###### `blob_sidecar_{subnet_id}` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. @@ -199,7 +199,7 @@ No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/1/` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -248,7 +248,7 @@ Clients MAY limit the number of blocks and sidecars in the response. **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* The `` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`: diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index 8a26b9ade..94299aaa2 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -49,7 +49,7 @@ ## Introduction -This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the Deneb specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. +This document specifies basic polynomial operations and KZG polynomial commitment operations that are essential for the implementation of the EIP-4844 feature in the Deneb specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. Functions flagged as "Public method" MUST be provided by the underlying KZG library as public functions. All other functions are private functions used internally by the KZG library. @@ -336,7 +336,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, ### KZG -KZG core functions. These are also defined in Deneb execution specs for EIP-4844. +KZG core functions. These are also defined in Deneb execution specs. #### `blob_to_kzg_commitment` diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 5af7c4938..b53c0a042 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -46,12 +46,14 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | Value | Unit | | - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | The number of blob sidecar subnets used in the gossipsub protocol. | +| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | ## Helpers ### `BlobsBundle` +*[New in Deneb:EIP4844]* + ```python @dataclass class BlobsBundle(object): @@ -67,7 +69,7 @@ class BlobsBundle(object): class GetPayloadResponse(object): execution_payload: ExecutionPayload block_value: uint256 - blobs_bundle: BlobsBundle + blobs_bundle: BlobsBundle # [New in Deneb:EIP4844] ``` ## Protocol @@ -98,7 +100,7 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments -[New in Deneb:EIP4844] +*[New in Deneb:EIP4844]* 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` @@ -107,6 +109,8 @@ via `get_payload(payload_id).blobs_bundle`. #### Constructing the `SignedBlobSidecar`s +*[New in Deneb:EIP4844]* + To construct a `SignedBlobSidecar`, a `signed_blob_sidecar` is defined with the necessary context for block and sidecar proposal. ##### Sidecar From 241e52a20230a970a156b7d9501d9a98ed8954c1 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:35:03 +0300 Subject: [PATCH 20/36] Whisk (SSLE) with Curdleproofs - rebased (#3342) * Introduce consensus code for Whisk * polish, simplify, clean up (~100 fewer lines) @asn-d6: As discussed, I fixed a few bugs along the way but likely also introduced some bugs :) * minor cleanups and fixes * simplify is_k_commitment_unique * Update beacon-chain.md * Update beacon-chain.md * Initialize `k` in `get_validator_from_deposit()` * minor cleanups * Update beacon-chain.md * Create beacon-chain.md This PR changes the Whisk tracker format to be of the form `(r * pubkey, r * BLS_GT_GENERATOR)` instead of `(r * k * BLS_G1_GENERATOR, r * BLS_G1_GENERATOR)`. This allows for non-interactive tracker registrations from validator pubkeys, removing ~50 lines the code. It also significantly reduces the amount of state overhead. This PR also removes permutation commitments, though those can be easily readded if deemed necessary. * A couple of fixes to the no-registration simplification @asn-d6: Readded a consistency check for `IsValidWhiskOpeningProof` (involving `pubkey` instead of `k_commitment`). * remove unused helpers * use Mary's suggested tracker * Update beacon-chain.md * Revert G_t element optimization This needs its own ethresearch post, and some additional analysis to see if we can do the shuffle ZKP in the allowed timeframe. This reverts commit 8517acabfc1dbb1a35789e6170b5db0bb2c19c9a. * Implement new shuffling strategy Ditch the Feistel logic and instead have each shuffler pick the row they shuffle using their RANDAO reveal. * Curdleproofs edits * working whisk eth2spec * working whisk dummy test * add more boilerplate set up code * rebase constants * Implement even newer and simplified shuffling strategy This commit further simplifies 0faef30fc131d1b471b63a7f16772eeeef548ef8 by removing the entire squareshuffle. The latest version of https://eprint.iacr.org/2022/560 proposes that each shuffler picks random indices from the entire candidate set instead of organizing validators into a square. * Move to _features * remove dummy test * Run doctoc * Change Whisk's previous fork to Capella instead of Bellatrix. Make linter happier. * Fix lint * Fix pylint * Fix mypy issues * Clean-up get_beacon_proposer_index * Fix doc headers * Fix capella link * Update apply_deposit * Rename process_shuffled_trackers --------- Co-authored-by: George Kadianakis Co-authored-by: Justin Co-authored-by: Nalin Bhardwaj Co-authored-by: Hsiao-Wei Wang --- .gitignore | 1 + Makefile | 2 +- setup.py | 39 ++- specs/_features/whisk/beacon-chain.md | 482 ++++++++++++++++++++++++++ specs/_features/whisk/fork.md | 138 ++++++++ 5 files changed, 656 insertions(+), 6 deletions(-) create mode 100644 specs/_features/whisk/beacon-chain.md create mode 100644 specs/_features/whisk/fork.md diff --git a/.gitignore b/.gitignore index 82026c27b..c7012d51b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ tests/core/pyspec/eth2spec/bellatrix/ tests/core/pyspec/eth2spec/capella/ tests/core/pyspec/eth2spec/deneb/ tests/core/pyspec/eth2spec/eip6110/ +tests/core/pyspec/eth2spec/whisk/ # coverage reports .htmlcov diff --git a/Makefile b/Makefile index ab5521663..5c25d59d3 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \ $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \ $(wildcard $(SSZ_DIR)/*.md) -ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb eip6110 +ALL_EXECUTABLE_SPECS = phase0 altair bellatrix capella deneb eip6110 whisk # The parameters for commands. Use `foreach` to avoid listing specs again. COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), --cov=eth2spec.$S.$(TEST_PRESET_TYPE)) PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPECS), ./eth2spec/$S) diff --git a/setup.py b/setup.py index c9041010d..f4d0c8719 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ BELLATRIX = 'bellatrix' CAPELLA = 'capella' DENEB = 'deneb' EIP6110 = 'eip6110' +WHISK = 'whisk' # The helper functions that are used when defining constants @@ -733,9 +734,31 @@ from eth2spec.deneb import {preset_name} as deneb ''' +# +# WhiskSpecBuilder +# +class WhiskSpecBuilder(CapellaSpecBuilder): + fork: str = WHISK + + @classmethod + def imports(cls, preset_name: str): + return super().imports(preset_name) + f''' +from eth2spec.capella import {preset_name} as capella +''' + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + # Necessary for custom types `WhiskShuffleProof` and `WhiskTrackerProof` + constants = { + 'WHISK_MAX_SHUFFLE_PROOF_SIZE': spec_object.constant_vars['WHISK_MAX_SHUFFLE_PROOF_SIZE'].value, + 'WHISK_MAX_OPENING_PROOF_SIZE': spec_object.constant_vars['WHISK_MAX_OPENING_PROOF_SIZE'].value, + } + return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants} + + spec_builders = { builder.fork: builder - for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, EIP6110SpecBuilder) + for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder, EIP6110SpecBuilder, WhiskSpecBuilder) } @@ -1045,7 +1068,7 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -1053,7 +1076,7 @@ class PySpecCommand(Command): specs/phase0/weak-subjectivity.md specs/phase0/p2p-interface.md """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/altair/light-client/full-node.md specs/altair/light-client/light-client.md @@ -1065,7 +1088,7 @@ class PySpecCommand(Command): specs/altair/validator.md specs/altair/p2p-interface.md """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110): + if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/bellatrix/beacon-chain.md specs/bellatrix/fork.md @@ -1074,7 +1097,7 @@ class PySpecCommand(Command): specs/bellatrix/p2p-interface.md sync/optimistic.md """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110): + if self.spec_fork in (CAPELLA, DENEB, EIP6110, WHISK): self.md_doc_paths += """ specs/capella/light-client/fork.md specs/capella/light-client/full-node.md @@ -1104,6 +1127,11 @@ class PySpecCommand(Command): specs/_features/eip6110/beacon-chain.md specs/_features/eip6110/fork.md """ + if self.spec_fork == WHISK: + self.md_doc_paths += """ + specs/_features/whisk/beacon-chain.md + specs/_features/whisk/fork.md + """ if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) @@ -1259,5 +1287,6 @@ setup( "lru-dict==1.1.8", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", + "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", ] ) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md new file mode 100644 index 000000000..7d2ee078f --- /dev/null +++ b/specs/_features/whisk/beacon-chain.md @@ -0,0 +1,482 @@ +# Whisk -- The Beacon Chain + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Constants](#constants) +- [Cryptography](#cryptography) + - [BLS](#bls) + - [Curdleproofs and opening proofs](#curdleproofs-and-opening-proofs) +- [Epoch processing](#epoch-processing) + - [`WhiskTracker`](#whisktracker) + - [`Validator`](#validator) + - [`BeaconState`](#beaconstate) +- [Block processing](#block-processing) + - [Block header](#block-header) + - [`BeaconBlock`](#beaconblock) + - [Whisk](#whisk) + - [`BeaconBlockBody`](#beaconblockbody) + - [Deposits](#deposits) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) + + + + +## Introduction + +This document details the beacon chain additions and changes of to support the Whisk SSLE, + +*Note:* This specification is built upon [Capella](../../capella/beacon-chain.md) and is under active development. + +## Constants + +| Name | Value | Description | +| ---------------------------------- | -------------------------- | ----------------------------------------------------------- | +| `WHISK_CANDIDATE_TRACKERS_COUNT` | `uint64(2**14)` (= 16,384) | number of candidate trackers | +| `WHISK_PROPOSER_TRACKERS_COUNT` | `uint64(2**13)` (= 8,192) | number of proposer trackers | +| `WHISK_EPOCHS_PER_SHUFFLING_PHASE` | `Epoch(2**8)` (= 256) | epochs per shuffling phase | +| `WHISK_VALIDATORS_PER_SHUFFLE` | `uint64(2**7)` (= 128) | number of validators shuffled per shuffle step | +| `WHISK_PROPOSER_SELECTION_GAP` | `Epoch(2)` | gap between proposer selection and the block proposal phase | +| `WHISK_MAX_SHUFFLE_PROOF_SIZE` | `uint64(2**15)` | max size of a shuffle proof | +| `WHISK_MAX_OPENING_PROOF_SIZE` | `uint64(2**10)` | max size of a opening proof | + +| Name | Value | +| ---------------------------------- | -------------------------- | +| `DOMAIN_WHISK_CANDIDATE_SELECTION` | `DomainType('0x07000000')` | +| `DOMAIN_WHISK_SHUFFLE` | `DomainType('0x07100000')` | +| `DOMAIN_WHISK_PROPOSER_SELECTION` | `DomainType('0x07200000')` | + +## Cryptography + +### BLS + +| Name | SSZ equivalent | Description | +| ------------------- | ---------------------------------------- | ----------------------------- | +| `BLSFieldElement` | `uint256` | BLS12-381 scalar | +| `BLSG1Point` | `Bytes48` | compressed BLS12-381 G1 point | +| `WhiskShuffleProof` | `ByteList[WHISK_MAX_SHUFFLE_PROOF_SIZE]` | Serialized shuffle proof | +| `WhiskTrackerProof` | `ByteList[WHISK_MAX_OPENING_PROOF_SIZE]` | Serialized tracker proof | + +*Note*: A subgroup check MUST be performed when deserializing a `BLSG1Point` for use in any of the functions below. + +```python +def BLSG1ScalarMultiply(scalar: BLSFieldElement, point: BLSG1Point) -> BLSG1Point: + return bls.G1_to_bytes48(bls.multiply(bls.bytes48_to_G1(point), scalar)) +``` + +```python +def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: + """ + Convert bytes to a BLS field scalar. The output is not uniform over the BLS field. + TODO: Deneb will introduces this helper too. Should delete it once it's rebased to post-Deneb. + """ + field_element = int.from_bytes(b, ENDIANNESS) + assert field_element < BLS_MODULUS + return BLSFieldElement(field_element) +``` + +| Name | Value | +| ------------------ | ------------------------------------------------------------------------------- | +| `BLS_G1_GENERATOR` | `bls.G1_to_bytes48(bls.G1)` | +| `BLS_MODULUS` | `52435875175126190479447740508185965837690552500527637822603658699938581184513` | + +### Curdleproofs and opening proofs + +Note that Curdleproofs (Whisk Shuffle Proofs), the tracker opening proofs and all related data structures and verifier code (along with tests) is specified in [curdleproofs.pie](https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only) repository. + +```python +def IsValidWhiskShuffleProof(pre_shuffle_trackers: Sequence[WhiskTracker], + post_shuffle_trackers: Sequence[WhiskTracker], + M: BLSG1Point, + shuffle_proof: WhiskShuffleProof) -> bool: + """ + Verify `post_shuffle_trackers` is a permutation of `pre_shuffle_trackers`. + Defined in https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only. + """ + # pylint: disable=unused-argument + return True +``` + +```python +def IsValidWhiskOpeningProof(tracker: WhiskTracker, + k_commitment: BLSG1Point, + tracker_proof: WhiskTrackerProof) -> bool: + """ + Verify knowledge of `k` such that `tracker.k_r_G == k * tracker.r_G` and `k_commitment == k * BLS_G1_GENERATOR`. + Defined in https://github.com/nalinbhardwaj/curdleproofs.pie/tree/verifier-only. + """ + # pylint: disable=unused-argument + return True +``` + +## Epoch processing + +### `WhiskTracker` + +```python +class WhiskTracker(Container): + r_G: BLSG1Point # r * G + k_r_G: BLSG1Point # k * r * G +``` + +### `Validator` + +```python +class Validator(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals + effective_balance: Gwei # Balance at stake + slashed: boolean + # Status epochs + activation_eligibility_epoch: Epoch # When criteria for activation were met + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw funds + whisk_tracker: WhiskTracker # Whisk tracker (r * G, k * r * G) [New in Whisk] + whisk_k_commitment: BLSG1Point # Whisk k commitment k * BLS_G1_GENERATOR [New in Whisk] +``` + +### `BeaconState` + +```python +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] # Frozen in Capella, replaced by historical_summaries + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] # [Modified in Whisk] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + # Inactivity + inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT] + # Sync + current_sync_committee: SyncCommittee + next_sync_committee: SyncCommittee + # Execution + latest_execution_payload_header: ExecutionPayloadHeader + # Withdrawals + next_withdrawal_index: WithdrawalIndex + next_withdrawal_validator_index: ValidatorIndex + # Deep history valid from Capella onwards + historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT] + whisk_candidate_trackers: Vector[WhiskTracker, WHISK_CANDIDATE_TRACKERS_COUNT] # [New in Whisk] + whisk_proposer_trackers: Vector[WhiskTracker, WHISK_PROPOSER_TRACKERS_COUNT] # [New in Whisk] +``` + +```python +def select_whisk_trackers(state: BeaconState, epoch: Epoch) -> None: + # Select proposer trackers from candidate trackers + proposer_seed = get_seed(state, epoch - WHISK_PROPOSER_SELECTION_GAP, DOMAIN_WHISK_PROPOSER_SELECTION) + for i in range(WHISK_PROPOSER_TRACKERS_COUNT): + index = compute_shuffled_index(uint64(i), uint64(len(state.whisk_candidate_trackers)), proposer_seed) + state.whisk_proposer_trackers[i] = state.whisk_candidate_trackers[index] + + # Select candidate trackers from active validator trackers + active_validator_indices = get_active_validator_indices(state, epoch) + for i in range(WHISK_CANDIDATE_TRACKERS_COUNT): + seed = hash(get_seed(state, epoch, DOMAIN_WHISK_CANDIDATE_SELECTION) + uint_to_bytes(i)) + candidate_index = compute_proposer_index(state, active_validator_indices, seed) # sample by effective balance + state.whisk_candidate_trackers[i] = state.validators[candidate_index].whisk_tracker +``` + +```python +def process_whisk_updates(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + if next_epoch % WHISK_EPOCHS_PER_SHUFFLING_PHASE == 0: # select trackers at the start of shuffling phases + select_whisk_trackers(state, next_epoch) +``` + +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_inactivity_updates(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_summaries_update(state) + process_participation_flag_updates(state) + process_sync_committee_updates(state) + process_whisk_updates(state) # [New in Whisk] +``` + +## Block processing + +### Block header + +#### `BeaconBlock` + +```python +class BeaconBlock(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body: BeaconBlockBody + whisk_opening_proof: WhiskTrackerProof # [New in Whisk] +``` + +```python +def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None: + tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT] + k_commitment = state.validators[block.proposer_index].whisk_k_commitment + assert IsValidWhiskOpeningProof(tracker, k_commitment, block.whisk_opening_proof) +``` + +Removed `assert block.proposer_index == get_beacon_proposer_index(state)` check in Whisk. + +```python +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + + # # Verify that proposer index is the correct index + # assert block.proposer_index == get_beacon_proposer_index(state) + + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), + ) + + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed + process_whisk_opening_proof(state, block) # [New in Whisk] +``` + +### Whisk + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(capella.BeaconBlockBody): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] + sync_aggregate: SyncAggregate + # Execution + execution_payload: ExecutionPayload + # Capella operations + bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] + whisk_post_shuffle_trackers: Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE] # [New in Whisk] + whisk_shuffle_proof: WhiskShuffleProof # [New in Whisk] + whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk] + whisk_registration_proof: WhiskTrackerProof # [New in Whisk] + whisk_tracker: WhiskTracker # [New in Whisk] + whisk_k_commitment: BLSG1Point # [New in Whisk] +``` + +```python +def get_shuffle_indices(randao_reveal: BLSSignature) -> Sequence[uint64]: + """ + Given a `randao_reveal` return the list of indices that got shuffled from the entire candidate set + """ + indices = [] + for i in WHISK_VALIDATORS_PER_SHUFFLE: + # XXX ensure we are not suffering from modulo bias + shuffle_index = uint256(hash(randao_reveal + uint_to_bytes(i))) % WHISK_CANDIDATE_TRACKERS_COUNT + indices.append(shuffle_index) + + return indices +``` + +```python +def process_shuffled_trackers(state: BeaconState, body: BeaconBlockBody) -> None: + # Check the shuffle proof + shuffle_indices = get_shuffle_indices(body.randao_reveal) + pre_shuffle_trackers = [state.whisk_candidate_trackers[i] for i in shuffle_indices] + post_shuffle_trackers = body.whisk_post_shuffle_trackers + + shuffle_epoch = get_current_epoch(state) % WHISK_EPOCHS_PER_SHUFFLING_PHASE + if shuffle_epoch + WHISK_PROPOSER_SELECTION_GAP + 1 >= WHISK_EPOCHS_PER_SHUFFLING_PHASE: + # Require unchanged trackers during cooldown + assert pre_shuffle_trackers == post_shuffle_trackers + else: + # Require shuffled trackers during shuffle + assert IsValidWhiskShuffleProof( + pre_shuffle_trackers, + post_shuffle_trackers, + body.whisk_shuffle_proof_M_commitment, + body.whisk_shuffle_proof, + ) + + # Shuffle candidate trackers + for i, shuffle_index in enumerate(shuffle_indices): + state.whisk_candidate_trackers[shuffle_index] = post_shuffle_trackers[i] +``` + +```python +def is_k_commitment_unique(state: BeaconState, k_commitment: BLSG1Point) -> bool: + return all([validator.whisk_k_commitment != k_commitment for validator in state.validators]) +``` + +```python +def process_whisk(state: BeaconState, body: BeaconBlockBody) -> None: + process_shuffled_trackers(state, body) + + # Overwrite all validator Whisk fields (first Whisk proposal) or just the permutation commitment (next proposals) + proposer = state.validators[get_beacon_proposer_index(state)] + if proposer.whisk_tracker.r_G == BLS_G1_GENERATOR: # first Whisk proposal + assert body.whisk_tracker.r_G != BLS_G1_GENERATOR + assert is_k_commitment_unique(state, body.whisk_k_commitment) + assert IsValidWhiskOpeningProof( + body.whisk_tracker, + body.whisk_k_commitment, + body.whisk_registration_proof, + ) + proposer.whisk_tracker = body.whisk_tracker + proposer.whisk_k_commitment = body.whisk_k_commitment + else: # next Whisk proposals + assert body.whisk_registration_proof == WhiskTrackerProof() + assert body.whisk_tracker == WhiskTracker() + assert body.whisk_k_commitment == BLSG1Point() + assert body.whisk_shuffle_proof_M_commitment == BLSG1Point() +``` + +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + if is_execution_enabled(state, block.body): + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) + process_sync_aggregate(state, block.body.sync_aggregate) + process_whisk(state, block.body) # [New in Whisk] +``` + +### Deposits + +```python +def get_unique_whisk_k(state: BeaconState, validator_index: ValidatorIndex) -> BLSFieldElement: + counter = 0 + while True: + # hash `validator_index || counter` + k = BLSFieldElement(bytes_to_bls_field(hash(uint_to_bytes(validator_index) + uint_to_bytes(uint64(counter))))) + if is_k_commitment_unique(state, BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)): + return k # unique by trial and error + counter += 1 +``` + +```python +def get_initial_commitments(k: BLSFieldElement) -> Tuple[BLSG1Point, WhiskTracker]: + return ( + BLSG1ScalarMultiply(k, BLS_G1_GENERATOR), + WhiskTracker(r_G=BLS_G1_GENERATOR, k_r_G=BLSG1ScalarMultiply(k, BLS_G1_GENERATOR)) + ) +``` + +```python +def get_validator_from_deposit_whisk( + state: BeaconState, + pubkey: BLSPubkey, + withdrawal_credentials: Bytes32, + amount: uint64 +) -> Validator: + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + k = get_unique_whisk_k(state, ValidatorIndex(len(state.validators))) + whisk_k_commitment, whisk_tracker = get_initial_commitments(k) + + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + # Whisk fields + whisk_tracker=whisk_tracker, + whisk_k_commitment=whisk_k_commitment, + ) + return validator +``` + +```python +def apply_deposit(state: BeaconState, + pubkey: BLSPubkey, + withdrawal_credentials: Bytes32, + amount: uint64, + signature: BLSSignature) -> None: + validator_pubkeys = [validator.pubkey for validator in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + amount=amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + # Initialize validator if the deposit signature is valid + if bls.Verify(pubkey, signing_root, signature): + index = get_index_for_new_validator(state) + validator = get_validator_from_deposit_whisk(state, pubkey, withdrawal_credentials, amount) + set_or_append_list(state.validators, index, validator) + set_or_append_list(state.balances, index, amount) + # [New in Altair] + set_or_append_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.inactivity_scores, index, uint64(0)) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + +### `get_beacon_proposer_index` + +```python +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: + """ + Return the beacon proposer index at the current slot. + """ + assert state.latest_block_header.slot == state.slot # sanity check `process_block_header` has been called + return state.latest_block_header.proposer_index +``` diff --git a/specs/_features/whisk/fork.md b/specs/_features/whisk/fork.md new file mode 100644 index 000000000..189d8c5b3 --- /dev/null +++ b/specs/_features/whisk/fork.md @@ -0,0 +1,138 @@ +# Whisk -- Fork Logic + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Configuration](#configuration) +- [Fork to Whisk](#fork-to-whisk) + + + + +## Introduction + +This document describes the process of Whisk upgrade. + + +``` +""" + WHISK_FORK_EPOCH + | cooldown + | | || + v vsvv + --+~~~~~~~~~~~~~~~~~~~~~----+- + shuffling ^ + | + | + proposer selection + candidate selection +""" +``` + +## Configuration + +Warning: this configuration is not definitive. + +| Name | Value | +| -------------------- | ----------------------- | +| `WHISK_FORK_VERSION` | `Version('0x05000000')` | +| `WHISK_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | + +## Fork to Whisk + +If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == WHISK_FORK_EPOCH`, an irregular state change is made to upgrade to Whisk. `WHISK_FORK_EPOCH` must be a multiple of `WHISK_RUN_DURATION_IN_EPOCHS`. + +The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `WHISK_FORK_EPOCH * SLOTS_PER_EPOCH`. + +This ensures that we drop right into the beginning of the shuffling phase but without `process_whisk_epoch()` triggering for this Whisk run. Hence we handle all the setup ourselves in `upgrade_to_whisk()` below. + +```python +def whisk_candidate_selection(state: BeaconState, epoch: Epoch) -> None: + # TODO + # pylint: disable=unused-argument + pass +``` + +```python +def whisk_proposer_selection(state: BeaconState, epoch: Epoch) -> None: + # TODO + # pylint: disable=unused-argument + pass +``` + +```python +def upgrade_to_whisk(pre: bellatrix.BeaconState) -> BeaconState: + epoch = bellatrix.get_current_epoch(pre) + post = BeaconState( + # Versioning + genesis_time=pre.genesis_time, + genesis_validators_root=pre.genesis_validators_root, + slot=pre.slot, + fork=Fork( + previous_version=pre.fork.current_version, + current_version=WHISK_FORK_VERSION, + epoch=epoch, + ), + # History + latest_block_header=pre.latest_block_header, + block_roots=pre.block_roots, + state_roots=pre.state_roots, + historical_roots=pre.historical_roots, + # Eth1 + eth1_data=pre.eth1_data, + eth1_data_votes=pre.eth1_data_votes, + eth1_deposit_index=pre.eth1_deposit_index, + # Registry + validators=[], + balances=pre.balances, + # Randomness + randao_mixes=pre.randao_mixes, + # Slashings + slashings=pre.slashings, + # Participation + previous_epoch_participation=pre.previous_epoch_participation, + current_epoch_participation=pre.current_epoch_participation, + # Finality + justification_bits=pre.justification_bits, + previous_justified_checkpoint=pre.previous_justified_checkpoint, + current_justified_checkpoint=pre.current_justified_checkpoint, + finalized_checkpoint=pre.finalized_checkpoint, + # Inactivity + inactivity_scores=pre.inactivity_Scores, + ) + + # Initialize all validators with predictable commitments + for val_index, pre_validator in enumerate(pre.validators): + whisk_commitment, whisk_tracker = get_initial_commitments(get_unique_whisk_k(post, ValidatorIndex(val_index))) + + post_validator = Validator( + pubkey=pre_validator.pubkey, + withdrawal_credentials=pre_validator.withdrawal_credentials, + effective_balance=pre_validator.effective_balance, + slashed=pre_validator.slashed, + activation_eligibility_epoch=pre_validator.activation_eligibility_epoch, + activation_epoch=pre_validator.activation_epoch, + exit_epoch=pre_validator.exit_epoch, + withdrawable_epoch=pre_validator.withdrawable_epoch, + whisk_commitment=whisk_commitment, + whisk_tracker=whisk_tracker, + ) + post.validators.append(post_validator) + + # Do a candidate selection followed by a proposer selection so that we have proposers for the upcoming day + # Use an old epoch when selecting candidates so that we don't get the same seed as in the next candidate selection + whisk_candidate_selection(post, epoch - WHISK_PROPOSER_SELECTION_GAP - 1) + whisk_proposer_selection(post, epoch) + + # Do a final round of candidate selection. + # We need it so that we have something to shuffle over the upcoming shuffling phase. + whisk_candidate_selection(post, epoch) + + return post +``` From 68bb5ee5079ababf7be4cf7b6a1e0a9cada09f43 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 15:00:49 +0300 Subject: [PATCH 21/36] Ensure make lint resolves to the right pylint and mypy versions --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5c25d59d3..6c852a1e9 100644 --- a/Makefile +++ b/Makefile @@ -146,8 +146,8 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ - && mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) + && python -m pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ + && python -m mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ From 706824cf959df3aaa5aef543d573aaae00c411a3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jun 2023 20:25:03 +0300 Subject: [PATCH 22/36] Drop need to declare spec feature files --- setup.py | 121 ++++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/setup.py b/setup.py index f4d0c8719..f9b2438d4 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,25 @@ DENEB = 'deneb' EIP6110 = 'eip6110' WHISK = 'whisk' +PREVIOUS_FORK_OF = { + PHASE0: None, + ALTAIR: PHASE0, + BELLATRIX: ALTAIR, + CAPELLA: BELLATRIX, + DENEB: CAPELLA, + EIP6110: DENEB, + WHISK: CAPELLA, +} + +ALL_FORKS = list(PREVIOUS_FORK_OF.keys()) + +IGNORE_SPEC_FILES = [ + "specs/phase0/deposit-contract.md" +] + +EXTRA_SPEC_FILES = { + BELLATRIX: "sync/optimistic.md" +} # The helper functions that are used when defining constants CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' @@ -96,6 +115,30 @@ class SpecObject(NamedTuple): dataclasses: Dict[str, str] +def is_post_fork(a, b) -> bool: + """ + Returns true if fork a is after b, or if a == b + """ + if a == b: + return True + + prev_fork = PREVIOUS_FORK_OF[a] + if prev_fork == b: + return True + elif prev_fork == None: + return False + else: + return is_post_fork(prev_fork, b) + +def get_fork_directory(fork): + dir1 = f'specs/{fork}' + if os.path.exists(dir1): + return dir1 + dir2 = f'specs/_features/{fork}' + if os.path.exists(dir2): + return dir2 + raise FileNotFoundError(f"No directory found for fork: {fork}") + def _get_name_from_heading(heading: Heading) -> Optional[str]: last_child = heading.children[-1] if isinstance(last_child, CodeSpan): @@ -1068,70 +1111,20 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths = """ - specs/phase0/beacon-chain.md - specs/phase0/fork-choice.md - specs/phase0/validator.md - specs/phase0/weak-subjectivity.md - specs/phase0/p2p-interface.md - """ - if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/altair/light-client/full-node.md - specs/altair/light-client/light-client.md - specs/altair/light-client/p2p-interface.md - specs/altair/light-client/sync-protocol.md - specs/altair/beacon-chain.md - specs/altair/bls.md - specs/altair/fork.md - specs/altair/validator.md - specs/altair/p2p-interface.md - """ - if self.spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/bellatrix/beacon-chain.md - specs/bellatrix/fork.md - specs/bellatrix/fork-choice.md - specs/bellatrix/validator.md - specs/bellatrix/p2p-interface.md - sync/optimistic.md - """ - if self.spec_fork in (CAPELLA, DENEB, EIP6110, WHISK): - self.md_doc_paths += """ - specs/capella/light-client/fork.md - specs/capella/light-client/full-node.md - specs/capella/light-client/p2p-interface.md - specs/capella/light-client/sync-protocol.md - specs/capella/beacon-chain.md - specs/capella/fork.md - specs/capella/fork-choice.md - specs/capella/validator.md - specs/capella/p2p-interface.md - """ - if self.spec_fork in (DENEB, EIP6110): - self.md_doc_paths += """ - specs/deneb/light-client/fork.md - specs/deneb/light-client/full-node.md - specs/deneb/light-client/p2p-interface.md - specs/deneb/light-client/sync-protocol.md - specs/deneb/beacon-chain.md - specs/deneb/fork.md - specs/deneb/fork-choice.md - specs/deneb/polynomial-commitments.md - specs/deneb/p2p-interface.md - specs/deneb/validator.md - """ - if self.spec_fork == EIP6110: - self.md_doc_paths += """ - specs/_features/eip6110/beacon-chain.md - specs/_features/eip6110/fork.md - """ - if self.spec_fork == WHISK: - self.md_doc_paths += """ - specs/_features/whisk/beacon-chain.md - specs/_features/whisk/fork.md - """ + self.md_doc_paths = "" + + for fork in ALL_FORKS: + if is_post_fork(self.spec_fork, fork): + # Append all files in fork directory recursively + for root, dirs, files in os.walk(get_fork_directory(fork)): + for filename in files: + filepath = os.path.join(root, filename) + if filepath.endswith('.md') and filepath not in IGNORE_SPEC_FILES: + self.md_doc_paths += filepath + "\n" + # Append extra files if any + if fork in EXTRA_SPEC_FILES: + self.md_doc_paths += EXTRA_SPEC_FILES[fork] + "\n" + if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) From efd2429b3f14f7bccaad342a75fdf843b21c0951 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:28:06 +0300 Subject: [PATCH 23/36] Move whisk_opening_proof to block body (#3408) --- specs/_features/whisk/beacon-chain.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/specs/_features/whisk/beacon-chain.md b/specs/_features/whisk/beacon-chain.md index 7d2ee078f..bad818837 100644 --- a/specs/_features/whisk/beacon-chain.md +++ b/specs/_features/whisk/beacon-chain.md @@ -19,7 +19,6 @@ - [`BeaconState`](#beaconstate) - [Block processing](#block-processing) - [Block header](#block-header) - - [`BeaconBlock`](#beaconblock) - [Whisk](#whisk) - [`BeaconBlockBody`](#beaconblockbody) - [Deposits](#deposits) @@ -235,23 +234,11 @@ def process_epoch(state: BeaconState) -> None: ### Block header -#### `BeaconBlock` - -```python -class BeaconBlock(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body: BeaconBlockBody - whisk_opening_proof: WhiskTrackerProof # [New in Whisk] -``` - ```python def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None: tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT] k_commitment = state.validators[block.proposer_index].whisk_k_commitment - assert IsValidWhiskOpeningProof(tracker, k_commitment, block.whisk_opening_proof) + assert IsValidWhiskOpeningProof(tracker, k_commitment, block.body.whisk_opening_proof) ``` Removed `assert block.proposer_index == get_beacon_proposer_index(state)` check in Whisk. @@ -303,6 +290,8 @@ class BeaconBlockBody(capella.BeaconBlockBody): execution_payload: ExecutionPayload # Capella operations bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] + # Whisk + whisk_opening_proof: WhiskTrackerProof # [New in Whisk] whisk_post_shuffle_trackers: Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE] # [New in Whisk] whisk_shuffle_proof: WhiskShuffleProof # [New in Whisk] whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk] From 7b69f176e2e836ccff5b58f655b39b9c583f00cd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 9 Jun 2023 08:54:16 -0600 Subject: [PATCH 24/36] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 2a6fa03c0..54dd11e46 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.1 +1.4.0-alpha.2 From 45f3432019336b5cb1196037e6f5d92a6dc5aeba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Jun 2023 23:10:00 +0800 Subject: [PATCH 25/36] Update `MAX_BLOBS_PER_BLOCK` to `6` and add Deneb networking configs to yaml files (#3410) --- configs/mainnet.yaml | 8 ++++++++ configs/minimal.yaml | 8 ++++++++ presets/mainnet/deneb.yaml | 4 ++-- presets/minimal/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- specs/deneb/fork-choice.md | 3 ++- .../test/deneb/unittests/test_config_invariants.py | 8 ++++++++ 7 files changed, 31 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index baf489739..80a240185 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -124,3 +124,11 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 43f1fc83f..82a813a9b 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -125,3 +125,11 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index 10c5025ed..23889fd18 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -6,5 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 # `uint64(2**12)` (= 4096) MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 diff --git a/presets/minimal/deneb.yaml b/presets/minimal/deneb.yaml index 91120f9da..e21d38777 100644 --- a/presets/minimal/deneb.yaml +++ b/presets/minimal/deneb.yaml @@ -6,5 +6,5 @@ FIELD_ELEMENTS_PER_BLOB: 4 # [customized] MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 41ce76bd2..148dc96d8 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -73,7 +73,7 @@ Deneb is a consensus-layer upgrade containing a number of features. Including: | Name | Value | Description | | - | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | *[New in Deneb:EIP4844]* hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | *[New in Deneb:EIP4844]* Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOBS_PER_BLOCK` | `uint64(6)` | *[New in Deneb:EIP4844]* maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | *Note*: The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index c67906259..bbc7fa0f8 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -36,7 +36,8 @@ The block MUST NOT be considered valid until all valid `Blob`s have been downloa def is_data_available(beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: # `retrieve_blobs_and_proofs` is implementation and context dependent # It returns all the blobs for the given block root, and raises an exception if not available - # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` + # Note: the p2p network does not guarantee sidecar retrieval outside of + # `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` blobs, proofs = retrieve_blobs_and_proofs(beacon_block_root) # For testing, `retrieve_blobs_and_proofs` returns ("TEST", "TEST"). diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 13a54225e..75955ed16 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -10,3 +10,11 @@ from eth2spec.test.context import ( @single_phase def test_length(spec): assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + + +@with_deneb_and_later +@spec_test +@single_phase +def test_networking(spec): + assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK + assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK From 5576d0e6850b0ddbffd1ec122841b35e31e66fee Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Sat, 10 Jun 2023 01:12:38 +1000 Subject: [PATCH 26/36] Moved configuration into network preset instead of constants. (#3394) * Moved configuration into network preset instead of constants. Now that `MAX_CHUNK_SIZE` and `GOSSIP_MAX_SIZE` are in configuration, we no longer need separate constants to represent them in the spec when they change in Bellatrix. I've changed the usage, and put the values into the presets, but I'm not sure if I've updated the descriptions in the best way... This is following on from the work in #3375 where a number of constants got moved into configuration, so we no longer need these constants to be separately represented, they can simply be updated in presets. * Update presets/minimal/bellatrix.yaml Co-authored-by: Hsiao-Wei Wang * Update presets/mainnet/bellatrix.yaml Co-authored-by: Hsiao-Wei Wang * Moved preset items into the correct section and updated TOC. It looked like the items listed in configuration about the max size and chunk size were no longer needed since we're updating preset values now and the preset changes seem to only be listed in the changes at the top. * review feedback * hopefully correct this time! Moved the 2 fields from configs into presets completely as suggested. * WIP - changing back to being in config and updating the phase 0 value... I think this should be close but want to see what's outstanding. * fix intellij's formatting of table. * more fixes --------- Co-authored-by: Hsiao-Wei Wang --- configs/mainnet.yaml | 8 ++++---- configs/minimal.yaml | 8 ++++---- presets/mainnet/phase0.yaml | 2 +- presets/minimal/phase0.yaml | 2 +- specs/bellatrix/p2p-interface.md | 28 ++-------------------------- specs/phase0/p2p-interface.md | 4 ++-- 6 files changed, 14 insertions(+), 38 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 80a240185..fe405cb38 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -98,16 +98,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa # Networking # --------------------------------------------------------------- -# `2**20` (= 1048576, 1 MiB) -GOSSIP_MAX_SIZE: 1048576 +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 -# `2**20` (=1048576, 1 MiB) -MAX_CHUNK_SIZE: 1048576 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 # 5s TTFB_TIMEOUT: 5 # 10s diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 82a813a9b..dcc834963 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -99,16 +99,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 # Networking # --------------------------------------------------------------- -# `2**20` (= 1048576, 1 MiB) -GOSSIP_MAX_SIZE: 1048576 +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 -# `2**20` (=1048576, 1 MiB) -MAX_CHUNK_SIZE: 1048576 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 # 5s TTFB_TIMEOUT: 5 # 10s diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index 02bc96c8c..00133ba36 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index e7028f5a4..d9a6a2b6c 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128 # 2**4 (= 16) MAX_DEPOSITS: 16 # 2**4 (= 16) -MAX_VOLUNTARY_EXITS: 16 +MAX_VOLUNTARY_EXITS: 16 \ No newline at end of file diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index b8b3a11d6..7d80d40a8 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -14,7 +14,6 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas - [Warning](#warning) - [Modifications in Bellatrix](#modifications-in-bellatrix) - - [Configuration](#configuration) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) @@ -41,15 +40,6 @@ Refer to the note in the [validator guide](./validator.md) for further details. ## Modifications in Bellatrix -### Configuration - -This section outlines modifications constants that are used in this spec. - -| Name | Value | Description | -|---|---|---| -| `GOSSIP_MAX_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed gossip messages starting at Bellatrix upgrade. | -| `MAX_CHUNK_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses starting at Bellatrix upgrade. | - ### The gossip domain: gossipsub Some gossip meshes are upgraded in Bellatrix to support upgraded types. @@ -61,11 +51,6 @@ All topics remain stable except the beacon block topic which is updated with the The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 and Altair documents unless explicitly noted here. -Starting at Bellatrix upgrade, each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) -has a maximum size of `GOSSIP_MAX_SIZE_BELLATRIX`. -Clients MUST reject (fail validation) messages that are over this size limit. -Likewise, clients MUST NOT emit or propagate messages larger than this limit. - The derivation of the `message-id` remains stable. The new topics along with the type of the `data` field of a gossipsub message are given in this table: @@ -130,10 +115,6 @@ down-scoring or disconnection. Request and Response remain unchanged unless explicitly noted here. -Starting at Bellatrix upgrade, -a global maximum uncompressed byte size of `MAX_CHUNK_SIZE_BELLATRIX` MUST be applied to all method response chunks -regardless of type specific bounds that *MUST* also be respected. - Bellatrix fork-digest is introduced to the `context` enum to specify Bellatrix block type. Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: @@ -171,17 +152,12 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: With the addition of `ExecutionPayload` to `BeaconBlock`s, there is a dynamic field -- `transactions` -- which can validly exceed the `GOSSIP_MAX_SIZE` limit (1 MiB) put in -place at Phase 0. At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction +place at Phase 0, so GOSSIP_MAX_SIZE has increased to 10 Mib on the network. +At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction filled entirely with data at a cost of 16 gas per byte can create a valid `ExecutionPayload` of ~2 MiB. Thus we need a size limit to at least account for current mainnet conditions. -Geth currently has a [max gossip message size](https://github.com/ethereum/go-ethereum/blob/3ce9f6d96f38712f5d6756e97b59ccc20cc403b3/eth/protocols/eth/protocol.go#L49) of 10 MiB. -To support backward compatibility with this previously defined network limit, -we adopt `GOSSIP_MAX_SIZE_BELLATRIX` of 10 MiB for maximum gossip sizes at the -point of Bellatrix and beyond. Note, that clients SHOULD still reject objects -that exceed their maximum theoretical bounds which in most cases is less than `GOSSIP_MAX_SIZE_BELLATRIX`. - Note, that due to additional size induced by the `BeaconBlock` contents (e.g. proposer signature, operations lists, etc) this does reduce the theoretical max valid `ExecutionPayload` (and `transactions` list) size as diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index de3e0e529..c0d18b08f 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -189,11 +189,11 @@ This section outlines configurations that are used in this spec. | Name | Value | Description | |---|---|---| -| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `GOSSIP_MAX_SIZE` | `10 * 2**20` (= 10485760, 10 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | -| `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | +| `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | From cfd44dac56996b8195434277392686fc319b70d2 Mon Sep 17 00:00:00 2001 From: William Dowling Date: Sun, 11 Jun 2023 15:07:39 +0200 Subject: [PATCH 27/36] Update lru-dict package to latest stable release. lru-dict build fails when building under clang version 16 (https://github.com/amitdev/lru-dict/commit/6badf6376d12a2e0498f67bbc7d5c0332b96a4d7). This bumps the version to the latest stable release. Tested on MacOS M2 and Ubuntu Jammy x86-64. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c8719..adf24ba36 100644 --- a/setup.py +++ b/setup.py @@ -1284,7 +1284,7 @@ setup( "remerkleable==0.1.27", "trie==2.0.2", RUAMEL_YAML_VERSION, - "lru-dict==1.1.8", + "lru-dict==1.2.0", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", From 99f294cdd8a2d4461576eacaabbf4dc7ad60169c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Jun 2023 16:02:28 +0800 Subject: [PATCH 28/36] Add link to EIP PR7044. Need to change it to eips.ethereum.org path once the EIP is merged --- specs/deneb/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index e25751232..929ac39f6 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -43,7 +43,8 @@ ## Introduction Deneb is a consensus-layer upgrade containing a number of features. Including: -* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner. +* [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844): Shard Blob Transactions scale data-availability of Ethereum in a simple, forwards-compatible manner +* [EIP-7044](https://github.com/ethereum/EIPs/pull/7044): Perpetually Valid Signed Voluntary Exits ## Custom types From 7cf55c138ceb4bcbad889c362b9dce6d1c130afb Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 12 Jun 2023 15:34:05 +0300 Subject: [PATCH 29/36] Lock nalinbhardwaj/curdleproofs.pie version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f4d0c8719..f6d69cc13 100644 --- a/setup.py +++ b/setup.py @@ -1287,6 +1287,6 @@ setup( "lru-dict==1.1.8", MARKO_VERSION, "py_arkworks_bls12381==0.3.4", - "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs", + "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@805d06785b6ff35fde7148762277dd1ae678beeb#egg=curdleproofs&subdirectory=curdleproofs", ] ) From 512d2ca516d3aa6c3b65a831ff89c2bacaa6f9da Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 12 Jun 2023 20:24:22 +0530 Subject: [PATCH 30/36] Update blob side car subnet count to 6 in line with max blobs limit update --- specs/deneb/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index b53c0a042..ec9c54784 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -46,7 +46,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | Value | Unit | | - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | ## Helpers From 210c4827bc6824c02c7c0cfcc7aad26294767560 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Jun 2023 23:57:21 +0800 Subject: [PATCH 31/36] Move `BLOB_SIDECAR_SUBNET_COUNT` to networking config --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/deneb/p2p-interface.md | 7 +++++-- specs/deneb/validator.md | 10 ---------- .../test/deneb/unittests/test_config_invariants.py | 2 ++ 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index fe405cb38..365bc1136 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -132,3 +132,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index dcc834963..b22a7165e 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -133,3 +133,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128 MAX_REQUEST_BLOB_SIDECARS: 768 # `2**12` (= 4096 epochs, ~18 days) MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 32e0ee9bf..809e405a6 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -40,11 +40,14 @@ The specification of these changes continues in the same format as the network s ### Configuration +*[New in Deneb:EIP4844]* + | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | -| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | +| `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | +| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | The number of blob sidecar subnets used in the gossipsub protocol. | ### Containers diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index ec9c54784..3157ccf21 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -10,8 +10,6 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) -- [Constants](#constants) - - [Misc](#misc) - [Helpers](#helpers) - [`BlobsBundle`](#blobsbundle) - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) @@ -40,14 +38,6 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of Deneb](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. -## Constants - -### Misc - -| Name | Value | Unit | -| - | - | :-: | -| `BLOB_SIDECAR_SUBNET_COUNT` | `6` | *[New in Deneb:EIP4844]* The number of blob sidecar subnets used in the gossipsub protocol. | - ## Helpers ### `BlobsBundle` diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py index 75955ed16..087bd63c6 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -18,3 +18,5 @@ def test_length(spec): def test_networking(spec): assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK assert spec.config.MAX_REQUEST_BLOB_SIDECARS == spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.MAX_BLOBS_PER_BLOCK + # Start with the same size, but `BLOB_SIDECAR_SUBNET_COUNT` could potentially increase later. + assert spec.config.BLOB_SIDECAR_SUBNET_COUNT == spec.MAX_BLOBS_PER_BLOCK From 87f77a3a6d5d5cb240e9961611514606b6d44f2f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 13 Jun 2023 01:42:18 +0800 Subject: [PATCH 32/36] bump VERSION.txt --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 54dd11e46..45eafc27d 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.4.0-alpha.2 +1.4.0-alpha.3 From 530924020f84198675268830f54a879c21784d24 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:29:59 +0300 Subject: [PATCH 33/36] Lock doctoc version --- .circleci/config.yml | 2 +- .github/workflows/run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdf483d5..157c56ca5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,7 +176,7 @@ jobs: - checkout - run: name: Check table of contents - command: sudo npm install -g doctoc@2 && make check_toc + command: sudo npm install -g doctoc@2.2.0 && make check_toc codespell: docker: - image: circleci/python:3.9 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 41a80ab92..b27c90765 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,7 +50,7 @@ jobs: with: ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }} - name: Check table of contents - run: sudo npm install -g doctoc@2 && make check_toc + run: sudo npm install -g doctoc@2.2.0 && make check_toc codespell: runs-on: self-hosted From 12fabf5854622cec2b01cf652a9277534ea6c59a Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 14 Jun 2023 16:47:17 +0300 Subject: [PATCH 34/36] Update specs/deneb/beacon-chain.md Co-authored-by: Danny Ryan --- specs/deneb/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 929ac39f6..0d71e4ae7 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -270,6 +270,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi #### Modified `process_voluntary_exit` +Note: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 + ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: voluntary_exit = signed_voluntary_exit.message From 420f8baf6777cbf35e63ffe4ceb9029b168f65ad Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Jun 2023 23:04:46 +0800 Subject: [PATCH 35/36] Rework tests. Move all `process_voluntary_exit` tests to Deneb --- specs/deneb/beacon-chain.md | 2 +- .../test_process_voluntary_exit.py | 3 +- .../test_process_voluntary_exit.py | 54 ++++++++++++++++--- .../test_process_voluntary_exit.py | 44 --------------- tests/generators/operations/main.py | 1 - 5 files changed, 49 insertions(+), 55 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 0d71e4ae7..3189ee190 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -270,7 +270,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi #### Modified `process_voluntary_exit` -Note: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 +*Note*: The function `process_voluntary_exit` is modified to use the a fixed fork version -- `CAPELLA_FORK_VERSION` -- for EIP-7044 ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py index 12b554da5..ea3b57a97 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_voluntary_exit.py @@ -7,7 +7,6 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.constants import ( BELLATRIX, CAPELLA, - DENEB, ) from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import ( @@ -80,7 +79,7 @@ def test_voluntary_exit_with_current_fork_version_not_is_before_fork_epoch(spec, ) -@with_phases([BELLATRIX, CAPELLA, DENEB]) +@with_phases([BELLATRIX, CAPELLA]) @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py index 371fcfed4..06a111c86 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -19,6 +19,7 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` """ + assert state.fork.current_version != spec.config.CAPELLA_FORK_VERSION yield from run_voluntary_exit_processing_test( spec, state, @@ -28,7 +29,7 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo ) -@with_phases([DENEB]) +@with_deneb_and_later @spec_state_test @always_bls def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): @@ -37,9 +38,48 @@ def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec """ assert state.fork.previous_version != state.fork.current_version - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=False, - ) + if spec.fork == DENEB: + assert state.fork.previous_version == spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + ) + else: + assert state.fork.previous_version != spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=False, + valid=False, + ) + + +@with_deneb_and_later +@spec_state_test +@always_bls +def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): + """ + Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + """ + assert state.fork.previous_version != state.fork.current_version + + if spec.fork == DENEB: + assert state.fork.previous_version == spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + ) + else: + assert state.fork.previous_version != spec.config.CAPELLA_FORK_VERSION + yield from run_voluntary_exit_processing_test( + spec, + state, + fork_version=state.fork.previous_version, + is_before_fork_epoch=True, + valid=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py deleted file mode 100644 index 4128a1181..000000000 --- a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_voluntary_exit.py +++ /dev/null @@ -1,44 +0,0 @@ -from eth2spec.test.context import ( - always_bls, - spec_state_test, - with_eip6110_and_later, -) -from eth2spec.test.bellatrix.block_processing.test_process_voluntary_exit import ( - run_voluntary_exit_processing_test, -) - - -@with_eip6110_and_later -@spec_state_test -@always_bls -def test_invalid_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): - """ - Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` - """ - assert state.fork.previous_version != state.fork.current_version - - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=False, - valid=False, - ) - - -@with_eip6110_and_later -@spec_state_test -@always_bls -def test_invalid_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): - """ - Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` - """ - assert state.fork.previous_version != state.fork.current_version - - yield from run_voluntary_exit_processing_test( - spec, - state, - fork_version=state.fork.previous_version, - is_before_fork_epoch=True, - valid=False, - ) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 3b4e5d1f7..053236c8d 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -45,7 +45,6 @@ if __name__ == "__main__": _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', - 'voluntary_exit', ]} eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) From 11ab19c90a6228a696dea3df27b8834674fef348 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 14 Jun 2023 09:25:54 -0600 Subject: [PATCH 36/36] Apply suggestions from code review --- .../deneb/block_processing/test_process_voluntary_exit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py index 06a111c86..711d27eb9 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_voluntary_exit.py @@ -35,6 +35,8 @@ def test_invalid_voluntary_exit_with_current_fork_version_not_is_before_fork_epo def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec, state): """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + + Note: This test is valid for ``spec.fork == DENEB`` and invalid for subsequent forks """ assert state.fork.previous_version != state.fork.current_version @@ -63,6 +65,8 @@ def test_voluntary_exit_with_previous_fork_version_not_is_before_fork_epoch(spec def test_voluntary_exit_with_previous_fork_version_is_before_fork_epoch(spec, state): """ Since Deneb, the VoluntaryExit domain is fixed to `CAPELLA_FORK_VERSION` + + Note: This test is valid for ``spec.fork == DENEB`` and invalid for subsequent forks """ assert state.fork.previous_version != state.fork.current_version