From b5bcec8d34904d2f46cea3a14cad55a996611b8d Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Mon, 26 Sep 2022 15:01:45 +0000 Subject: [PATCH 01/13] deprecate phase 0-only versions of beacon_blocks_by_{range,root} --- specs/phase0/p2p-interface.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 6277bee9f..f52752931 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -745,6 +745,8 @@ For example, if slot 4 were empty in the previous example, the returned array wo `step` is deprecated and must be set to 1. Clients may respond with a single block if a larger step is returned during the deprecation transition period. +`/eth2/beacon_chain/req/beacon_blocks_by_range/1/` is deprecated. Clients MAY respond with an empty list during the deprecation transition period. + `BeaconBlocksByRange` is primarily used to sync historical blocks. The request MUST be encoded as an SSZ-container. @@ -831,6 +833,8 @@ Clients MUST support requesting blocks since the latest finalized epoch. Clients MUST respond with at least one block, if they have it. Clients MAY limit the number of blocks in the response. +`/eth2/beacon_chain/req/beacon_blocks_by_root/1/` is deprecated. Clients MAY respond with an empty list during the deprecation transition period. + #### Ping **Protocol ID:** `/eth2/beacon_chain/req/ping/1/` From 19c8f21b6345690694d878c1c7db2665416cccde Mon Sep 17 00:00:00 2001 From: Ramana Kumar Date: Sat, 19 Nov 2022 09:17:11 +0000 Subject: [PATCH 02/13] Fix type annotation in verify_aggregate_kzg_proof --- specs/eip4844/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index b945d317a..6ffc7f0c5 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -405,7 +405,7 @@ def compute_aggregate_kzg_proof(blobs: Sequence[Blob]) -> KZGProof: ```python def verify_aggregate_kzg_proof(blobs: Sequence[Blob], expected_kzg_commitments: Sequence[KZGCommitment], - kzg_aggregated_proof: KZGCommitment) -> bool: + kzg_aggregated_proof: KZGProof) -> bool: """ Public method. """ From 4e3b700b86fa66aa4f62027601579e825119f7d7 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 22 Nov 2022 11:51:10 +0400 Subject: [PATCH 03/13] Fix initialization of VERSIONED_HASH_VERSION_KZG --- specs/eip4844/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index afab21dca..600793a3f 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -55,7 +55,7 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844. This is an exte | Name | Value | | - | - | | `BLOB_TX_TYPE` | `uint8(0x05)` | -| `VERSIONED_HASH_VERSION_KZG` | `Bytes1(0x01)` | +| `VERSIONED_HASH_VERSION_KZG` | `Bytes1('0x01')` | ## Preset From d202f620de036df40c2ef6268e7b10f8c1e5696b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 23 Nov 2022 05:09:12 +0800 Subject: [PATCH 04/13] Fix EIP4844 previous_version in tests --- tests/core/pyspec/eth2spec/test/helpers/fork_transition.py | 3 +++ tests/core/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index 0280bc7fb..1e3374a64 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -165,6 +165,9 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, operation_dict= elif post_spec.fork == CAPELLA: assert state.fork.previous_version == post_spec.config.BELLATRIX_FORK_VERSION assert state.fork.current_version == post_spec.config.CAPELLA_FORK_VERSION + elif post_spec.fork == EIP4844: + assert state.fork.previous_version == post_spec.config.CAPELLA_FORK_VERSION + assert state.fork.current_version == post_spec.config.EIP4844_FORK_VERSION if with_block: return state, _state_transition_and_sign_block_at_slot(post_spec, state, operation_dict=operation_dict) diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index ee3068788..d758936e9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -60,7 +60,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): previous_version = spec.config.BELLATRIX_FORK_VERSION current_version = spec.config.CAPELLA_FORK_VERSION elif spec.fork == EIP4844: - previous_version = spec.config.BELLATRIX_FORK_VERSION + previous_version = spec.config.CAPELLA_FORK_VERSION current_version = spec.config.EIP4844_FORK_VERSION state = spec.BeaconState( From f0ff15c14dca644470979c58bcaad0aaaf28ae46 Mon Sep 17 00:00:00 2001 From: kevaundray Date: Wed, 23 Nov 2022 14:52:47 +0000 Subject: [PATCH 05/13] EIP4844: Refactor Fiat-Shamir logic to handle empty sidecars (#3093) Additionally, it makes the Fiat-Shamir hashing logic more robust by making the challenges independent of each other. It also makes it more efficient to implement by moving both challenge computations to a single function needing a single transcript hash. Co-authored-by: George Kadianakis Co-authored-by: Dankrad Feist --- specs/eip4844/polynomial-commitments.md | 54 ++++++++++++------- .../unittests/validator/test_validator.py | 6 +++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 6ffc7f0c5..b18122b03 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -21,7 +21,7 @@ - [BLS12-381 helpers](#bls12-381-helpers) - [`bytes_to_bls_field`](#bytes_to_bls_field) - [`blob_to_polynomial`](#blob_to_polynomial) - - [`hash_to_bls_field`](#hash_to_bls_field) + - [`compute_challenges`](#compute_challenges) - [`bls_modular_inverse`](#bls_modular_inverse) - [`div`](#div) - [`g1_lincomb`](#g1_lincomb) @@ -41,7 +41,6 @@ - ## Introduction This document specifies basic polynomial operations and KZG polynomial commitment operations as they are needed for the EIP-4844 specification. The implementations are not optimized for performance, but readability. All practical implementations should optimize the polynomial operations. @@ -163,31 +162,44 @@ def blob_to_polynomial(blob: Blob) -> Polynomial: return polynomial ``` -#### `hash_to_bls_field` +#### `compute_challenges` ```python -def hash_to_bls_field(polys: Sequence[Polynomial], - comms: Sequence[KZGCommitment]) -> BLSFieldElement: +def compute_challenges(polynomials: Sequence[Polynomial], + commitments: Sequence[KZGCommitment]) -> BLSFieldElement: """ - Compute 32-byte hash of serialized polynomials and commitments concatenated. - This hash is then converted to a BLS field element, where the result is not uniform over the BLS field. - Return the BLS field element. + Return the Fiat-Shamir challenges required by the rest of the protocol. + The Fiat-Shamir logic works as per the following pseudocode: + + hashed_data = hash(DOMAIN_SEPARATOR, polynomials, commitments) + r = hash(hashed_data, 0) + r_powers = [r, r**2, r**3, ...] + eval_challenge = hash(hashed_data, 1) + + Then return `r_powers` and `eval_challenge` after converting them to BLS field elements. + The resulting field elements are not uniform over the BLS field. """ # Append the number of polynomials and the degree of each polynomial as a domain separator - num_polys = int.to_bytes(len(polys), 8, ENDIANNESS) + num_polynomials = int.to_bytes(len(polynomials), 8, ENDIANNESS) degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 8, ENDIANNESS) - data = FIAT_SHAMIR_PROTOCOL_DOMAIN + degree_poly + num_polys + data = FIAT_SHAMIR_PROTOCOL_DOMAIN + degree_poly + num_polynomials # Append each polynomial which is composed by field elements - for poly in polys: + for poly in polynomials: for field_element in poly: data += int.to_bytes(field_element, BYTES_PER_FIELD_ELEMENT, ENDIANNESS) # Append serialized G1 points - for commitment in comms: + for commitment in commitments: data += commitment - return bytes_to_bls_field(hash(data)) + # Transcript has been prepared: time to create the challenges + hashed_data = hash(data) + r = hash(hashed_data + b'\x00') + r_powers = compute_powers(bytes_to_bls_field(r), len(commitments)) + eval_challenge = hash(hashed_data + b'\x01') + + return r_powers, bytes_to_bls_field(eval_challenge) ``` #### `bls_modular_inverse` @@ -234,7 +246,8 @@ def poly_lincomb(polys: Sequence[Polynomial], Given a list of ``polynomials``, interpret it as a 2D matrix and compute the linear combination of each column with `scalars`: return the resulting polynomials. """ - result = [0] * len(polys[0]) + assert len(polys) == len(scalars) + result = [0] * FIELD_ELEMENTS_PER_BLOB for v, s in zip(polys, scalars): for i, x in enumerate(v): result[i] = (result[i] + int(s) * int(x)) % BLS_MODULUS @@ -256,6 +269,7 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]: return powers ``` + ### Polynomials #### `evaluate_polynomial_in_evaluation_form` @@ -367,14 +381,15 @@ def compute_aggregated_poly_and_commitment( """ Return (1) the aggregated polynomial, (2) the aggregated KZG commitment, and (3) the polynomial evaluation random challenge. + This function should also work with blobs == [] and kzg_commitments == [] """ + assert len(blobs) == len(kzg_commitments) + # Convert blobs to polynomials polynomials = [blob_to_polynomial(blob) for blob in blobs] - # Generate random linear combination challenges - r = hash_to_bls_field(polynomials, kzg_commitments) - r_powers = compute_powers(r, len(kzg_commitments)) - evaluation_challenge = int(r_powers[-1]) * r % BLS_MODULUS + # Generate random linear combination and evaluation challenges + r_powers, evaluation_challenge = compute_challenges(polynomials, kzg_commitments) # Create aggregated polynomial in evaluation form aggregated_poly = Polynomial(poly_lincomb(polynomials, r_powers)) @@ -390,6 +405,7 @@ def compute_aggregated_poly_and_commitment( ```python def compute_aggregate_kzg_proof(blobs: Sequence[Blob]) -> KZGProof: """ + Given a list of blobs, return the aggregated KZG proof that is used to verify them against their commitments. Public method. """ commitments = [blob_to_kzg_commitment(blob) for blob in blobs] @@ -407,6 +423,8 @@ def verify_aggregate_kzg_proof(blobs: Sequence[Blob], expected_kzg_commitments: Sequence[KZGCommitment], kzg_aggregated_proof: KZGProof) -> bool: """ + Given a list of blobs and an aggregated KZG proof, verify that they correspond to the provided commitments. + Public method. """ aggregated_poly, aggregated_poly_commitment, evaluation_challenge = compute_aggregated_poly_and_commitment( diff --git a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py index 634daca2d..d9e3877c3 100644 --- a/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/eip4844/unittests/validator/test_validator.py @@ -25,6 +25,12 @@ def _run_validate_blobs_sidecar_test(spec, state, blob_count): spec.validate_blobs_sidecar(block.slot, block.hash_tree_root(), expected_commitments, blobs_sidecar) +@with_eip4844_and_later +@spec_state_test +def test_validate_blobs_sidecar_zero_blobs(spec, state): + _run_validate_blobs_sidecar_test(spec, state, blob_count=0) + + @with_eip4844_and_later @spec_state_test def test_validate_blobs_sidecar_one_blob(spec, state): From 3ec25a875569dc4d9c8b6e0c0d13f25db7468b54 Mon Sep 17 00:00:00 2001 From: kevaundray Date: Thu, 24 Nov 2022 13:11:46 +0000 Subject: [PATCH 06/13] Fix docstring for compute_powers and compute_challenges methods (#3130) --- specs/eip4844/polynomial-commitments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index b18122b03..6f5a369e4 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -173,7 +173,7 @@ def compute_challenges(polynomials: Sequence[Polynomial], hashed_data = hash(DOMAIN_SEPARATOR, polynomials, commitments) r = hash(hashed_data, 0) - r_powers = [r, r**2, r**3, ...] + r_powers = [1, r, r**2, r**3, ...] eval_challenge = hash(hashed_data, 1) Then return `r_powers` and `eval_challenge` after converting them to BLS field elements. @@ -259,7 +259,7 @@ def poly_lincomb(polys: Sequence[Polynomial], ```python def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]: """ - Return ``x`` to power of [0, n-1]. + Return ``x`` to power of [0, n-1], if n > 0. When n==0, an empty array is returned. """ current_power = 1 powers = [] From a1e46d1ae47dd9d097725801575b46907c12a1f8 Mon Sep 17 00:00:00 2001 From: Roberto Bayardo Date: Thu, 24 Nov 2022 05:26:02 -0800 Subject: [PATCH 07/13] EIP4844: Fix return value of compute_challenges (#3129) Co-authored-by: George Kadianakis --- specs/eip4844/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 6f5a369e4..74c0b3cc5 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -166,7 +166,7 @@ def blob_to_polynomial(blob: Blob) -> Polynomial: ```python def compute_challenges(polynomials: Sequence[Polynomial], - commitments: Sequence[KZGCommitment]) -> BLSFieldElement: + commitments: Sequence[KZGCommitment]) -> Tuple[Sequence[BLSFieldElement], BLSFieldElement]: """ Return the Fiat-Shamir challenges required by the rest of the protocol. The Fiat-Shamir logic works as per the following pseudocode: From b5a0ff07e0f34ceced63ad7dba39ac2c7629c63e Mon Sep 17 00:00:00 2001 From: Potuz Date: Thu, 24 Nov 2022 10:29:46 -0300 Subject: [PATCH 08/13] Add link to Capella's P2P doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa22e05c5..07b78c0d5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Features are researched and developed in parallel, and then consolidated into se ### In-development Specifications | Code Name or Topic | Specs | Notes | | - | - | - | -| Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
| +| Capella (tentative) |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Validator additions](specs/capella/validator.md)
    • [P2P networking](specs/capella/p2p-interface.md)
| | EIP4844 (tentative) |
  • Core
    • [Beacon Chain changes](specs/eip4844/beacon-chain.md)
    • [EIP-4844 fork](specs/eip4844/fork.md)
    • [Polynomial commitments](specs/eip4844/polynomial-commitments.md)
  • Additions
    • [Honest validator guide changes](specs/eip4844/validator.md)
    • [P2P networking](specs/eip4844/p2p-interface.md)
| | Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/sharding/p2p-interface.md)
| | Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/custody_game/validator.md)
| Dependent on sharding | From 4f32fd95d6af22ac18379d5f7a3dc5b3c57bf327 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 28 Nov 2022 18:51:38 +0800 Subject: [PATCH 09/13] Enable EIP4844 lint and fix Pylint --- Makefile | 8 ++++---- linter.ini | 5 +++++ setup.py | 4 ++++ specs/eip4844/beacon-chain.md | 1 + specs/eip4844/validator.md | 1 + 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c38d39a6d..73450562b 100644 --- a/Makefile +++ b/Makefile @@ -105,12 +105,12 @@ install_test: # Testing against `minimal` config by default test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec # Testing against `minimal` config by default find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p $(TEST_REPORT_DIR); @@ -142,8 +142,8 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella \ - && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella + && pylint --rcfile $(LINTER_CONFIG_FILE) ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix ./eth2spec/capella ./eth2spec/eip4844 \ + && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix -p eth2spec.capella -p eth2spec.eip4844 lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ diff --git a/linter.ini b/linter.ini index 6575642f1..52a3aec0e 100644 --- a/linter.ini +++ b/linter.ini @@ -11,3 +11,8 @@ warn_unused_configs = True warn_redundant_casts = True ignore_missing_imports = True + +# pylint +[MESSAGES CONTROL] +disable = all +enable = unused-argument diff --git a/setup.py b/setup.py index de446bb1c..13ef3c701 100644 --- a/setup.py +++ b/setup.py @@ -588,6 +588,7 @@ class NoopExecutionEngine(ExecutionEngine): pass def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: + # pylint: disable=unused-argument raise NotImplementedError("no default block production") @@ -643,12 +644,14 @@ T = TypeVar('T') # For generic function def no_op(fn): # type: ignore + # pylint: disable=unused-argument def wrapper(*args, **kw): # type: ignore return None return wrapper def get_empty_list_result(fn): # type: ignore + # pylint: disable=unused-argument def wrapper(*args, **kw): # type: ignore return [] return wrapper @@ -664,6 +667,7 @@ get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals) # def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> Optional[BlobsSidecar]: + # pylint: disable=unused-argument return "TEST"''' @classmethod diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 600793a3f..1afd21e88 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -284,6 +284,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ```python def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody): + # pylint: disable=unused-argument assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` diff --git a/specs/eip4844/validator.md b/specs/eip4844/validator.md index 57a5610ce..1258358a2 100644 --- a/specs/eip4844/validator.md +++ b/specs/eip4844/validator.md @@ -46,6 +46,7 @@ Implementers may also retrieve blobs individually per transaction. ```python def get_blobs_and_kzg_commitments(payload_id: PayloadId) -> Tuple[Sequence[BLSFieldElement], Sequence[KZGCommitment]]: + # pylint: disable=unused-argument ... ``` From b3a176689d0d9712172a1a06a701e1ae9aa41764 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 28 Nov 2022 20:16:18 +0800 Subject: [PATCH 10/13] WIP. Fixing mypy errors --- setup.py | 15 +------------ specs/eip4844/beacon-chain.md | 13 ++++++----- specs/eip4844/polynomial-commitments.md | 30 ++++++++++++------------- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/setup.py b/setup.py index 13ef3c701..f15e4ca6d 100644 --- a/setup.py +++ b/setup.py @@ -666,7 +666,7 @@ get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals) # End # -def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> Optional[BlobsSidecar]: +def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> PyUnion[BlobsSidecar, str]: # pylint: disable=unused-argument return "TEST"''' @@ -686,10 +686,6 @@ spec_builders = { } -def is_spec_defined_type(value: str) -> bool: - return value.startswith(('ByteList', 'Union', 'Vector', 'List')) - - def objects_to_spec(preset_name: str, spec_object: SpecObject, builder: SpecBuilder, @@ -702,15 +698,6 @@ def objects_to_spec(preset_name: str, [ f"class {key}({value}):\n pass\n" for key, value in spec_object.custom_types.items() - if not is_spec_defined_type(value) - ] - ) - + ('\n\n' if len([key for key, value in spec_object.custom_types.items() if is_spec_defined_type(value)]) > 0 else '') - + '\n\n'.join( - [ - f"{key} = {value}\n" - for key, value in spec_object.custom_types.items() - if is_spec_defined_type(value) ] ) ) diff --git a/specs/eip4844/beacon-chain.md b/specs/eip4844/beacon-chain.md index 1afd21e88..63a6cd987 100644 --- a/specs/eip4844/beacon-chain.md +++ b/specs/eip4844/beacon-chain.md @@ -175,10 +175,13 @@ but MUST NOT be considered valid until a valid `BlobsSidecar` has been downloade def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments: Sequence[KZGCommitment]) -> bool: # `retrieve_blobs_sidecar` is implementation dependent, raises an exception if not available. sidecar = retrieve_blobs_sidecar(slot, beacon_block_root) - if sidecar == "TEST": - return True # For testing; remove once we have a way to inject `BlobsSidecar` into tests - validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) + # For testing, `retrieve_blobs_sidecar` returns "TEST. + # TODO: Remove it once we have a way to inject `BlobsSidecar` into tests. + if isinstance(sidecar, str): + return True + + validate_blobs_sidecar(slot, beacon_block_root, blob_kzg_commitments, sidecar) return True ``` @@ -216,7 +219,7 @@ def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedH ```python def verify_kzg_commitments_against_transactions(transactions: Sequence[Transaction], kzg_commitments: Sequence[KZGCommitment]) -> bool: - all_versioned_hashes = [] + all_versioned_hashes: List[VersionedHash] = [] for tx in transactions: if tx[0] == BLOB_TX_TYPE: all_versioned_hashes += tx_peek_blob_versioned_hashes(tx) @@ -283,7 +286,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe #### Blob KZG commitments ```python -def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody): +def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody) -> None: # pylint: disable=unused-argument assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 74c0b3cc5..726947134 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -144,7 +144,7 @@ def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: """ Convert 32-byte value to a BLS field scalar. The output is not uniform over the BLS field. """ - return int.from_bytes(b, ENDIANNESS) % BLS_MODULUS + return BLSFieldElement(int.from_bytes(b, ENDIANNESS) % BLS_MODULUS) ``` #### `blob_to_polynomial` @@ -210,7 +210,7 @@ def bls_modular_inverse(x: BLSFieldElement) -> BLSFieldElement: Compute the modular inverse of x i.e. return y such that x * y % BLS_MODULUS == 1 and return 0 for x == 0 """ - return pow(x, -1, BLS_MODULUS) if x != 0 else 0 + return BLSFieldElement(pow(x, -1, BLS_MODULUS)) if x != 0 else BLSFieldElement(0) ``` #### `div` @@ -220,7 +220,7 @@ def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement: """ Divide two field elements: ``x`` by `y``. """ - return (int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS + return BLSFieldElement((int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS) ``` #### `g1_lincomb` @@ -251,7 +251,7 @@ def poly_lincomb(polys: Sequence[Polynomial], for v, s in zip(polys, scalars): for i, x in enumerate(v): result[i] = (result[i] + int(s) * int(x)) % BLS_MODULUS - return [BLSFieldElement(x) for x in result] + return Polynomial([BLSFieldElement(x) for x in result]) ``` #### `compute_powers` @@ -284,7 +284,7 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, """ width = len(polynomial) assert width == FIELD_ELEMENTS_PER_BLOB - inverse_width = bls_modular_inverse(width) + inverse_width = bls_modular_inverse(BLSFieldElement(width)) # Make sure we won't divide by zero during division assert z not in ROOTS_OF_UNITY @@ -293,9 +293,11 @@ def evaluate_polynomial_in_evaluation_form(polynomial: Polynomial, result = 0 for i in range(width): - result += div(int(polynomial[i]) * int(roots_of_unity_brp[i]), (int(z) - int(roots_of_unity_brp[i]))) - result = result * (pow(z, width, BLS_MODULUS) - 1) * inverse_width % BLS_MODULUS - return result + a = BLSFieldElement(int(polynomial[i]) * int(roots_of_unity_brp[i]) % BLS_MODULUS) + b = BLSFieldElement((int(BLS_MODULUS) + int(z) - int(roots_of_unity_brp[i])) % BLS_MODULUS) + result += int(div(a, b) % BLS_MODULUS) + result = result * int(pow(z, width, BLS_MODULUS) - 1) * int(inverse_width) + return BLSFieldElement(result % BLS_MODULUS) ``` ### KZG @@ -355,17 +357,13 @@ def compute_kzg_proof(polynomial: Polynomial, z: BLSFieldElement) -> KZGProof: Compute KZG proof at point `z` with `polynomial` being in evaluation form Do this by computing the quotient polynomial in evaluation form: q(x) = (p(x) - p(z)) / (x - z) """ - - # To avoid SSZ overflow/underflow, convert element into int - polynomial = [int(i) for i in polynomial] - z = int(z) - y = evaluate_polynomial_in_evaluation_form(polynomial, z) - polynomial_shifted = [(p - int(y)) % BLS_MODULUS for p in polynomial] + polynomial_shifted = [BLSFieldElement((int(p) - int(y)) % BLS_MODULUS) for p in polynomial] # Make sure we won't divide by zero during division assert z not in ROOTS_OF_UNITY - denominator_poly = [(int(x) - z) % BLS_MODULUS for x in bit_reversal_permutation(ROOTS_OF_UNITY)] + denominator_poly = [BLSFieldElement((int(x) - int(z)) % BLS_MODULUS) + for x in bit_reversal_permutation(ROOTS_OF_UNITY)] # Calculate quotient polynomial by doing point-by-point division quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)] @@ -392,7 +390,7 @@ def compute_aggregated_poly_and_commitment( r_powers, evaluation_challenge = compute_challenges(polynomials, kzg_commitments) # Create aggregated polynomial in evaluation form - aggregated_poly = Polynomial(poly_lincomb(polynomials, r_powers)) + aggregated_poly = poly_lincomb(polynomials, r_powers) # Compute commitment to aggregated polynomial aggregated_poly_commitment = KZGCommitment(g1_lincomb(kzg_commitments, r_powers)) From edde563b3ac19aea4ff6290ce1e14c279b75b0df Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 28 Nov 2022 21:06:22 +0800 Subject: [PATCH 11/13] Workaround: ignore Invalid base class "ByteVector" error --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f15e4ca6d..432a41fe4 100644 --- a/setup.py +++ b/setup.py @@ -686,6 +686,10 @@ spec_builders = { } +def is_byte_vector(value: str) -> bool: + return value.startswith(('ByteVector')) + + def objects_to_spec(preset_name: str, spec_object: SpecObject, builder: SpecBuilder, @@ -696,7 +700,7 @@ def objects_to_spec(preset_name: str, new_type_definitions = ( '\n\n'.join( [ - f"class {key}({value}):\n pass\n" + f"class {key}({value}):\n pass\n" if not is_byte_vector(value) else f"class {key}({value}): # type: ignore\n pass\n" for key, value in spec_object.custom_types.items() ] ) From aae00a0bf667d64a99244d0a2d79c7e3f691adcd Mon Sep 17 00:00:00 2001 From: Tim Beiko Date: Wed, 30 Nov 2022 16:21:06 -0800 Subject: [PATCH 12/13] Clarify that full withdrawals only apply to exited validators --- specs/capella/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 32192a6ec..644ee476f 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -112,10 +112,10 @@ Up to `MAX_BLS_TO_EXECUTION_CHANGES`, [`BLSToExecutionChange`](./beacon-chain.md ## Enabling validator withdrawals -Validator balances are fully or partially withdrawn via an automatic process. +Validator balances are withdrawn periodically via an automatic process. For exited validators, the full balance is withdrawn. For active validators, the balance in excess of `MAX_EFFECTIVE_BALANCE` is withdrawn. -For validators, there is one prerequisite for this automated process: -withdrawal credentials pointing to an execution layer address, i.e. having an `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +There is one prerequisite for this automated process: +the validator's withdrawal credentials pointing to an execution layer address, i.e. having an `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. If a validator has a `BLS_WITHDRAWAL_PREFIX` withdrawal credential prefix, to participate in withdrawals the validator must create a one-time message to change their withdrawal credential from the version authenticated with a BLS key to the From 23d3aeebba3b5da0df4bd25108461b442199f406 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 1 Dec 2022 14:59:00 +0200 Subject: [PATCH 13/13] EIP4844 precompile: Be strict when parsing field elements from network (#3138) * EIP4844: bytes_to_bls_field() must not accept values >= BLS_MODULUS bytes_to_bls_field() will be used in the precompile and hence it should error out when provided with malicious inputs. * EIP4844: Add hash_to_bls_field() for use in compute_challenges() The previous commit made bytes_to_bls_field() be strict about its inputs. However in compute_challenges() we are dealing with Fiat-Shamir and hash outputs that could be innocuously higher than the modulus. For this reason we add the hash_to_bls_field() helper for use in compute_challenges(). * EIP4844: Further use of bytes_to_bls_field() // Fix executable spec Co-authored-by: Hsiao-Wei Wang --- specs/eip4844/polynomial-commitments.md | 33 ++++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/specs/eip4844/polynomial-commitments.md b/specs/eip4844/polynomial-commitments.md index 726947134..6e1c00ccd 100644 --- a/specs/eip4844/polynomial-commitments.md +++ b/specs/eip4844/polynomial-commitments.md @@ -19,6 +19,7 @@ - [`reverse_bits`](#reverse_bits) - [`bit_reversal_permutation`](#bit_reversal_permutation) - [BLS12-381 helpers](#bls12-381-helpers) + - [`hash_to_bls_field`](#hash_to_bls_field) - [`bytes_to_bls_field`](#bytes_to_bls_field) - [`blob_to_polynomial`](#blob_to_polynomial) - [`compute_challenges`](#compute_challenges) @@ -137,14 +138,29 @@ def bit_reversal_permutation(sequence: Sequence[T]) -> Sequence[T]: ### BLS12-381 helpers +#### `hash_to_bls_field` + +```python +def hash_to_bls_field(data: bytes) -> BLSFieldElement: + """ + Hash ``data`` and convert the output to a BLS scalar field element. + The output is not uniform over the BLS field. + """ + hashed_data = hash(data) + return BLSFieldElement(int.from_bytes(hashed_data, ENDIANNESS) % BLS_MODULUS) +``` + #### `bytes_to_bls_field` ```python def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: """ - Convert 32-byte value to a BLS field scalar. The output is not uniform over the BLS field. + Convert 32-byte value to a BLS scalar field element. + This function does not accept inputs greater than the BLS modulus. """ - return BLSFieldElement(int.from_bytes(b, ENDIANNESS) % BLS_MODULUS) + field_element = int.from_bytes(b, ENDIANNESS) + assert field_element < BLS_MODULUS + return BLSFieldElement(field_element) ``` #### `blob_to_polynomial` @@ -156,8 +172,7 @@ def blob_to_polynomial(blob: Blob) -> Polynomial: """ polynomial = Polynomial() for i in range(FIELD_ELEMENTS_PER_BLOB): - value = int.from_bytes(blob[i * BYTES_PER_FIELD_ELEMENT: (i + 1) * BYTES_PER_FIELD_ELEMENT], ENDIANNESS) - assert value < BLS_MODULUS + value = bytes_to_bls_field(blob[i * BYTES_PER_FIELD_ELEMENT: (i + 1) * BYTES_PER_FIELD_ELEMENT]) polynomial[i] = value return polynomial ``` @@ -195,11 +210,11 @@ def compute_challenges(polynomials: Sequence[Polynomial], # Transcript has been prepared: time to create the challenges hashed_data = hash(data) - r = hash(hashed_data + b'\x00') - r_powers = compute_powers(bytes_to_bls_field(r), len(commitments)) - eval_challenge = hash(hashed_data + b'\x01') + r = hash_to_bls_field(hashed_data + b'\x00') + r_powers = compute_powers(r, len(commitments)) + eval_challenge = hash_to_bls_field(hashed_data + b'\x01') - return r_powers, bytes_to_bls_field(eval_challenge) + return r_powers, eval_challenge ``` #### `bls_modular_inverse` @@ -422,7 +437,7 @@ def verify_aggregate_kzg_proof(blobs: Sequence[Blob], kzg_aggregated_proof: KZGProof) -> bool: """ Given a list of blobs and an aggregated KZG proof, verify that they correspond to the provided commitments. - + Public method. """ aggregated_poly, aggregated_poly_commitment, evaluation_challenge = compute_aggregated_poly_and_commitment(