From 6b94aab3af875d97e8574cbc23d6919b6540a235 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 3 Jan 2023 23:28:37 +0800 Subject: [PATCH 1/6] Move `is_data_available` check to fork-choice `on_block` --- specs/eip4844/beacon-chain.md | 28 --------- specs/eip4844/fork-choice.md | 105 ++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 specs/eip4844/fork-choice.md diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 19acb57d8..e62c455d6 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -23,7 +23,6 @@ - [Helper functions](#helper-functions) - [Misc](#misc) - [`validate_blobs_sidecar`](#validate_blobs_sidecar) - - [`is_data_available`](#is_data_available) - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) @@ -162,30 +161,6 @@ def validate_blobs_sidecar(slot: Slot, assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) ``` -#### `is_data_available` - -The implementation of `is_data_available` will become more sophisticated during later sharding upgrades. -Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`, -and validate the sidecar with `validate_blobs_sidecar`. - -The block MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `BlobsSidecar` has subsequently been pruned. - -```python -def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: - # `retrieve_blobs_sidecar` is implementation and context dependent, raises an exception if not available. - # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` - sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) - - # For testing, `retrieve_blobs_sidecar` returns "TEST". - # TODO: Remove it once we have a way to inject `BlobsSidecar` into tests. - if isinstance(sidecar, str): - return True - - validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) - return True -``` - - #### `kzg_commitment_to_versioned_hash` ```python @@ -241,9 +216,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) process_blob_kzg_commitments(state, block.body) # [New in EIP-4844] - - # New in EIP-4844 - assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) ``` #### Execution payload diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md new file mode 100644 index 000000000..bc25eeae7 --- /dev/null +++ b/specs/eip4844/fork-choice.md @@ -0,0 +1,105 @@ +# EIP-4844 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) +- [Helpers](#helpers) + - [`is_data_available`](#is_data_available) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + + + + +## Introduction + +This is the modification of the fork choice according to the executable beacon chain proposal. + +## Helpers + +#### `is_data_available` + +The implementation of `is_data_available` will become more sophisticated during later sharding upgrades. +Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`, +and validate the sidecar with `validate_blobs_sidecar`. + +The block MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `BlobsSidecar` has subsequently been pruned. + +```python +def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: + # `retrieve_blobs_sidecar` is implementation and context dependent, raises an exception if not available. + # Note: the p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS` + sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) + + # For testing, `retrieve_blobs_sidecar` returns "TEST". + # TODO: Remove it once we have a way to inject `BlobsSidecar` into tests. + if isinstance(sidecar, str): + return True + + validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) + return True +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The only modification is the addition of the verification of transition block conditions. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # [New in EIP-4844] + # Check if the block is available + assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + + # Check the merge transition + if is_merge_transition_block(pre_state, block.body): + validate_merge_block(block) + + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Add proposer score boost if the block is timely + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + if get_current_slot(store) == block.slot and is_before_attesting_interval: + store.proposer_boost_root = hash_tree_root(block) + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + store.justified_checkpoint = state.current_justified_checkpoint +``` From 5ff877cc26dab8ebdbd83ff08689ca3231a64607 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 4 Jan 2023 18:24:13 +0800 Subject: [PATCH 2/6] Add link to docs lookup table --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 07b78c0d5..bb69a452a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Features are researched and developed in parallel, and then consolidated into se | Code Name or Topic | Specs | Notes | | - | - | - | | Capella (tentative) | | -| EIP4844 (tentative) | | +| EIP4844 (tentative) | | | Sharding (outdated) | | | Custody Game (outdated) | | Dependent on sharding | | Data Availability Sampling (outdated) | | | From e937e2abbc48aad805d15df64480168ab993b98d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 6 Jan 2023 13:08:32 +0800 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Danny Ryan --- specs/eip4844/fork-choice.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index bc25eeae7..2613bbb7e 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -22,8 +22,8 @@ This is the modification of the fork choice according to the executable beacon c #### `is_data_available` -The implementation of `is_data_available` will become more sophisticated during later sharding upgrades. -Initially, it requires every verifying actor to retrieve the matching `BlobsSidecar`, +The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. +Initially, verification requires every verifying actor to retrieve the matching `BlobsSidecar`, and validate the sidecar with `validate_blobs_sidecar`. The block MUST NOT be considered valid until a valid `BlobsSidecar` has been downloaded. Blocks that have been previously validated as available SHOULD be considered available even if the associated `BlobsSidecar` has subsequently been pruned. @@ -69,7 +69,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root # [New in EIP-4844] - # Check if the block is available + # Check if blob data is available + # If not, this block MAY be queued and subsequently considered when blob data becomes available assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) # Check the block is valid and compute the post-state From e154b3414c30131ce970d8319d00d997ede4a162 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 6 Jan 2023 23:19:44 +0800 Subject: [PATCH 4/6] Move `validate_blobs_sidecar` to fork-choice and add spec into execution spec scope --- setup.py | 1 + specs/eip4844/beacon-chain.md | 17 ----------------- specs/eip4844/fork-choice.md | 17 +++++++++++++++++ .../{validator => fork_choice}/__init__.py | 0 .../test_validate_blobs_sidecar.py} | 0 5 files changed, 18 insertions(+), 17 deletions(-) rename tests/core/pyspec/eth2spec/test/eip4844/unittests/{validator => fork_choice}/__init__.py (100%) rename tests/core/pyspec/eth2spec/test/eip4844/unittests/{validator/test_validator.py => fork_choice/test_validate_blobs_sidecar.py} (100%) diff --git a/setup.py b/setup.py index be8be6d90..28d55f07e 100644 --- a/setup.py +++ b/setup.py @@ -1020,6 +1020,7 @@ class PySpecCommand(Command): self.md_doc_paths += """ specs/eip4844/beacon-chain.md specs/eip4844/fork.md + specs/eip4844/fork-choice.md specs/eip4844/polynomial-commitments.md specs/eip4844/p2p-interface.md specs/eip4844/validator.md diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index e62c455d6..8b0224f86 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -22,7 +22,6 @@ - [`ExecutionPayloadHeader`](#executionpayloadheader) - [Helper functions](#helper-functions) - [Misc](#misc) - - [`validate_blobs_sidecar`](#validate_blobs_sidecar) - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) @@ -145,22 +144,6 @@ class ExecutionPayloadHeader(Container): ### Misc -#### `validate_blobs_sidecar` - -```python -def validate_blobs_sidecar(slot: Slot, - beacon_block_root: Root, - expected_kzg_commitments: Sequence[KZGCommitment], - blobs_sidecar: BlobsSidecar) -> None: - assert slot == blobs_sidecar.beacon_block_slot - assert beacon_block_root == blobs_sidecar.beacon_block_root - blobs = blobs_sidecar.blobs - kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof - assert len(expected_kzg_commitments) == len(blobs) - - assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) -``` - #### `kzg_commitment_to_versioned_hash` ```python diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index 2613bbb7e..ead394234 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -7,6 +7,7 @@ - [Introduction](#introduction) - [Helpers](#helpers) + - [`validate_blobs_sidecar`](#validate_blobs_sidecar) - [`is_data_available`](#is_data_available) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [`on_block`](#on_block) @@ -20,6 +21,22 @@ This is the modification of the fork choice according to the executable beacon c ## Helpers +#### `validate_blobs_sidecar` + +```python +def validate_blobs_sidecar(slot: Slot, + beacon_block_root: Root, + expected_kzg_commitments: Sequence[KZGCommitment], + blobs_sidecar: BlobsSidecar) -> None: + assert slot == blobs_sidecar.beacon_block_slot + assert beacon_block_root == blobs_sidecar.beacon_block_root + blobs = blobs_sidecar.blobs + kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof + assert len(expected_kzg_commitments) == len(blobs) + + assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) +``` + #### `is_data_available` The implementation of `is_data_available` will become more sophisticated during later scaling upgrades. diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/__init__.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/fork_choice/__init__.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/__init__.py rename to tests/core/pyspec/eth2spec/test/eip4844/unittests/fork_choice/__init__.py diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/fork_choice/test_validate_blobs_sidecar.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py rename to tests/core/pyspec/eth2spec/test/eip4844/unittests/fork_choice/test_validate_blobs_sidecar.py From c9f8e4fef845b2b03a7790ede72c10e13f052187 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 6 Jan 2023 23:39:04 +0800 Subject: [PATCH 5/6] Move `BlobsSidecar` --- specs/eip4844/fork-choice.md | 14 ++++++++++++++ specs/eip4844/p2p-interface.md | 11 ----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index ead394234..0937b66d7 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -6,6 +6,8 @@ - [Introduction](#introduction) +- [Containers](#containers) + - [`BlobsSidecar`](#blobssidecar) - [Helpers](#helpers) - [`validate_blobs_sidecar`](#validate_blobs_sidecar) - [`is_data_available`](#is_data_available) @@ -19,6 +21,18 @@ This is the modification of the fork choice according to the executable beacon chain proposal. +## Containers + +### `BlobsSidecar` + +```python +class BlobsSidecar(Container): + beacon_block_root: Root + beacon_block_slot: Slot + blobs: List[Blob, MAX_BLOBS_PER_BLOCK] + kzg_aggregated_proof: KZGProof +``` + ## Helpers #### `validate_blobs_sidecar` diff --git a/specs/eip4844/p2p-interface.md b/specs/eip4844/p2p-interface.md index 420a6da05..582331203 100644 --- a/specs/eip4844/p2p-interface.md +++ b/specs/eip4844/p2p-interface.md @@ -12,7 +12,6 @@ The specification of these changes continues in the same format as the network s - [Configuration](#configuration) - [Containers](#containers) - - [`BlobsSidecar`](#blobssidecar) - [`SignedBeaconBlockAndBlobsSidecar`](#signedbeaconblockandblobssidecar) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - [Topics and messages](#topics-and-messages) @@ -41,16 +40,6 @@ The specification of these changes continues in the same format as the network s ## Containers -### `BlobsSidecar` - -```python -class BlobsSidecar(Container): - beacon_block_root: Root - beacon_block_slot: Slot - blobs: List[Blob, MAX_BLOBS_PER_BLOCK] - kzg_aggregated_proof: KZGProof -``` - ### `SignedBeaconBlockAndBlobsSidecar` ```python From 4ba2266fd55f5e0bc330ad2a70090443c23871ac Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 6 Jan 2023 08:45:20 -0700 Subject: [PATCH 6/6] Update specs/eip4844/fork-choice.md --- specs/eip4844/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index 0937b66d7..8dea28ded 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -19,7 +19,7 @@ ## Introduction -This is the modification of the fork choice according to the executable beacon chain proposal. +This is the modification of the fork choice accompanying the EIP-4844 upgrade. ## Containers