From 7e080c18f7448b970cfe59fd5f36e5624d911336 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Oct 2021 01:15:31 +0200 Subject: [PATCH 01/15] update remerkleable --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3dcaaa44f..863761daa 100644 --- a/setup.py +++ b/setup.py @@ -1021,7 +1021,7 @@ setup( "py_ecc==5.2.0", "milagro_bls_binding==1.6.3", "dataclasses==0.6", - "remerkleable==0.1.22", + "remerkleable==0.1.24", RUAMEL_YAML_VERSION, "lru-dict==1.1.6", MARKO_VERSION, From afb62eebf0f4c9d10abfc620b87f4f3492f024a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 18 Oct 2021 17:42:47 +0800 Subject: [PATCH 02/15] Add pytest CLI option `--fork` so that we can just run with specific phase (fork) --- tests/core/pyspec/eth2spec/test/conftest.py | 20 ++++++++++++++++++++ tests/core/pyspec/eth2spec/test/context.py | 5 ++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index e6d8352e0..55613be0a 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,4 +1,7 @@ from eth2spec.test import context +from eth2spec.test.helpers.constants import ( + ALL_PHASES, +) from eth2spec.utils import bls as bls_utils # We import pytest only when it's present, i.e. when we are running tests. @@ -29,6 +32,13 @@ def pytest_addoption(parser): "--preset", action="store", type=str, default="minimal", help="preset: make the pyspec use the specified preset" ) + parser.addoption( + "--fork", action="append", type=str, + help=( + "fork: make the pyspec only run with the specified phase." + " To run multiple phases, e.g., --fork=phase0 --fork=altair" + ) + ) parser.addoption( "--disable-bls", action="store_true", default=False, help="bls-default: make tests that are not dependent on BLS run without BLS" @@ -44,6 +54,16 @@ def preset(request): context.DEFAULT_TEST_PRESET = request.config.getoption("--preset") +@fixture(autouse=True) +def run_phases(request): + phases = request.config.getoption("--fork") + if phases: + phases = [phase.lower() for phase in phases] + context.DEFAULT_PYTEST_FORKS = set(phases) + else: + context.DEFAULT_PYTEST_FORKS = ALL_PHASES + + @fixture(autouse=True) def bls_default(request): disable_bls = request.config.getoption("--disable-bls") diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 6062648d6..944351bfd 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -22,6 +22,9 @@ from lru import LRU # Without pytest CLI arg or pyspec-test-generator 'preset' argument, this will be the config to apply. DEFAULT_TEST_PRESET = MINIMAL +# Without pytest CLI arg or pyspec-test-generator 'run-phase' argument, this will be the config to apply. +DEFAULT_PYTEST_FORKS = ALL_PHASES + # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. @@ -351,7 +354,7 @@ def with_phases(phases, other_phases=None): """ def decorator(fn): def wrapper(*args, **kw): - run_phases = phases + run_phases = set(phases).intersection(DEFAULT_PYTEST_FORKS) # limit phases if one explicitly specified if 'phase' in kw: From 2f6e817f3d06286565ad2fbae5f089ae31d54185 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 18 Oct 2021 18:30:33 +0800 Subject: [PATCH 03/15] Set CI job for each fork --- .circleci/config.yml | 40 ++++++++++++++++++++++++++++++++++++---- Makefile | 9 ++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb1df9909..bcce74bd9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,7 +90,7 @@ jobs: name: Install pyspec requirements command: make install_test - save_pyspec_cached_venv - test: + test-phase0: docker: - image: circleci/python:3.8 working_directory: ~/specs-repo @@ -100,7 +100,33 @@ jobs: - restore_pyspec_cached_venv - run: name: Run py-tests - command: make citest + command: make citest fork=phase0 + - store_test_results: + path: tests/core/pyspec/test-reports + test-altair: + docker: + - image: circleci/python:3.8 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_pyspec_cached_venv + - run: + name: Run py-tests + command: make citest fork=altair + - store_test_results: + path: tests/core/pyspec/test-reports + test-merge: + docker: + - image: circleci/python:3.8 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_pyspec_cached_venv + - run: + name: Run py-tests + command: make citest fork=merge - store_test_results: path: tests/core/pyspec/test-reports table_of_contents: @@ -208,14 +234,20 @@ workflows: - install_pyspec_test: requires: - checkout_specs - - test: + - test-phase0: + requires: + - install_pyspec_test + - test-altair: + requires: + - install_pyspec_test + - test-merge: requires: - install_pyspec_test - table_of_contents - codespell - lint: requires: - - test + - install_pyspec_test # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. # - install_deposit_contract_web3_tester: # requires: diff --git a/Makefile b/Makefile index 977141078..547ed215f 100644 --- a/Makefile +++ b/Makefile @@ -105,8 +105,15 @@ find_test: pyspec python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec - mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ + mkdir -p tests/core/pyspec/test-reports/eth2spec; +ifdef fork + . venv/bin/activate; cd $(PY_SPEC_DIR); \ + python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=eth2spec/test_results.xml eth2spec +else + . venv/bin/activate; cd $(PY_SPEC_DIR); \ python3 -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec +endif + open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & From 190ef9fb503743b779ce137527cdf5dc96b6a7d0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Oct 2021 12:07:33 -0600 Subject: [PATCH 04/15] call notify_forkchoice_updated with finalized_block_hash == 0x00..00 if not yet finalized --- specs/merge/fork-choice.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index ea4a5588a..32bb8215e 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -49,6 +49,7 @@ def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, fi ``` *Note*: The call of the `notify_forkchoice_updated` function maps on the `POS_FORKCHOICE_UPDATED` event defined in the [EIP-3675](https://eips.ethereum.org/EIPS/eip-3675#definitions). +As per EIP-3675, before a post-transition block is finalized, `notify_forkchoice_updated` must be called with `finalized_block_hash = Hash32()`. ## Helpers From 3bfdc917e13ec56c50ca8f0ff4b60485ab0b1a72 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Oct 2021 13:38:08 -0600 Subject: [PATCH 05/15] ad TBH_ACTIVATION_EPOCH --- configs/mainnet.yaml | 4 +++- configs/minimal.yaml | 4 +++- specs/merge/beacon-chain.md | 3 ++- specs/merge/client-settings.md | 9 +++++---- specs/merge/fork-choice.md | 6 ++++-- specs/merge/validator.md | 5 +++++ 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 77d7a136f..74896c90f 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -7,8 +7,10 @@ PRESET_BASE: 'mainnet' # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 -# By default, don't use this param +# By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TBH_ACTIVATION_EPOCH: 18446744073709551615 + # Genesis diff --git a/configs/minimal.yaml b/configs/minimal.yaml index a8b30fd54..35e321e84 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -7,8 +7,10 @@ PRESET_BASE: 'minimal' # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 -# By default, don't use this param +# By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TBH_ACTIVATION_EPOCH: 18446744073709551615 + # Genesis diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index f5977db5e..12919b474 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -74,7 +74,8 @@ This patch adds transaction execution to the beacon chain as part of the Merge f | Name | Value | | - | - | | `TERMINAL_TOTAL_DIFFICULTY` | **TBD** | -| `TERMINAL_BLOCK_HASH` | `Hash32('0x0000000000000000000000000000000000000000000000000000000000000000')` | +| `TERMINAL_BLOCK_HASH` | `Hash32()` | +| `TBH_ACTIVATION_EPOCH` | `FAR_FUTURE_EPOCH` | ## Containers diff --git a/specs/merge/client-settings.md b/specs/merge/client-settings.md index 4a4b38d3e..63ddfe5ae 100644 --- a/specs/merge/client-settings.md +++ b/specs/merge/client-settings.md @@ -18,11 +18,12 @@ This document specifies configurable settings that clients must implement for th To coordinate manual overrides to [`TERMINAL_TOTAL_DIFFICULTY`](./beacon-chain.md#Transition-settings) parameter, clients must provide `--terminal-total-difficulty-override` as a configurable setting. The value provided by this setting must take precedence over pre-configured `TERMINAL_TOTAL_DIFFICULTY` parameter. Clients should accept the setting as a decimal value (i.e., *not* hexadecimal). -Except under exceptional scenarios, this setting is expected to not be used. Sufficient warning to the user about this exceptional configurable setting should be provided. +Except under exceptional scenarios, this setting is not expected to be used. Sufficient warning to the user about this exceptional configurable setting should be provided. ### Override terminal block hash -To allow for transition coordination around a specific PoW block, clients must also provide `--terminal-block-hash-override` as a configurable setting. -The value provided by this setting takes precedence over the pre-configured `TERMINAL_BLOCK_HASH` parameter. +To allow for transition coordination around a specific PoW block, clients must also provide `--terminal-block-hash-override` and `--terminal-block-hash-epoch-override` as configurable settings. +* The value provided by `--terminal-block-hash-override` takes precedence over the pre-configured `TERMINAL_BLOCK_HASH` parameter. +* The value provided by `--terminal-block-hash-epoch-override` takes precedence over the pre-configured `TBH_ACTIVATION_EPOCH` parameter. -Except under exceptional scenarios, this setting is expected to not be used. Sufficient warning to the user about this exceptional configurable setting should be provided. +Except under exceptional scenarios, these settings are not expected to be used. Sufficient warning to the user about this exceptional configurable setting should be provided. diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index ea4a5588a..f1f5e265c 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -74,8 +74,8 @@ Used by fork-choice handler, `on_block`. ```python def is_valid_terminal_pow_block(block: PowBlock, parent: PowBlock) -> bool: - if block.block_hash == TERMINAL_BLOCK_HASH: - return True + if TERMINAL_BLOCK_HASH != Hash32(): + return block.block_hash == TERMINAL_BLOCK_HASH is_total_difficulty_reached = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY is_parent_total_difficulty_valid = parent.total_difficulty < TERMINAL_TOTAL_DIFFICULTY @@ -113,6 +113,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: pow_block = get_pow_block(block.body.execution_payload.parent_hash) pow_parent = get_pow_block(pow_block.parent_hash) assert is_valid_terminal_pow_block(pow_block, pow_parent) + if TERMINAL_BLOCK_HASH != Hash32(): + assert compute_epoch_at_slot(block.slot) >= TBH_ACTIVATION_EPOCH # Add new block to the store store.blocks[hash_tree_root(block)] = block diff --git a/specs/merge/validator.md b/specs/merge/validator.md index 2de4ef6b1..f15f62f3d 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -127,6 +127,11 @@ def prepare_execution_payload(state: BeaconState, fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: if not is_merge_complete(state): + is_tbh_override_set = TERMINAL_BLOCK_HASH != Hash32() + if is_tbh_override_set and get_current_epoch(state.slot) < TBH_ACTIVATION_EPOCH: + # TBH is set but activation epoch is not yet reached, no prepare payload call is needed + return None + terminal_pow_block = get_terminal_pow_block(pow_chain) if terminal_pow_block is None: # Pre-merge, no prepare payload call is needed From d5be6b5d6868321c8928eb82586256601c6d75d2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Oct 2021 18:10:27 -0600 Subject: [PATCH 06/15] remove prepare_payload in favor of a unification with notify_forkchoice_updated --- setup.py | 9 ++- specs/merge/fork-choice.md | 21 ++++++- specs/merge/validator.md | 118 ++++++++++++++++++++++--------------- 3 files changed, 96 insertions(+), 52 deletions(-) diff --git a/setup.py b/setup.py index 863761daa..fa396ab50 100644 --- a/setup.py +++ b/setup.py @@ -497,7 +497,7 @@ class MergeSpecBuilder(AltairSpecBuilder): return super().imports(preset_name) + f''' from typing import Protocol from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes20, ByteList, ByteVector, uint256, Union +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector, uint256, Union ''' @classmethod @@ -527,7 +527,10 @@ class NoopExecutionEngine(ExecutionEngine): def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: return True - def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, finalized_block_hash: Hash32) -> None: + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> None: pass def prepare_payload(self: ExecutionEngine, @@ -683,7 +686,7 @@ def combine_dicts(old_dict: Dict[str, T], new_dict: Dict[str, T]) -> Dict[str, T ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature', - 'Bytes1', 'Bytes4', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', + 'Bytes1', 'Bytes4', 'Bytes8', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set', diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index ea4a5588a..44e729104 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -12,6 +12,7 @@ - [`ExecutionEngine`](#executionengine) - [`notify_forkchoice_updated`](#notify_forkchoice_updated) - [Helpers](#helpers) + - [`PayloadAttributes`](#payloadattributes) - [`PowBlock`](#powblock) - [`get_pow_block`](#get_pow_block) - [`is_valid_terminal_pow_block`](#is_valid_terminal_pow_block) @@ -43,8 +44,14 @@ This function performs two actions *atomically*: * Applies finality to the execution state: it irreversibly persists the chain of all execution payloads and corresponding state, up to and including `finalized_block_hash`. +Additionally, if `payload_attributes` is provided, this function sets in motion a payload build process on top of +`head_block_hash` with the result to be gathered by a followup call to `get_payload`. + ```python -def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, finalized_block_hash: Hash32) -> None: +def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> None: ... ``` @@ -52,6 +59,18 @@ def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, fi ## Helpers +### `PayloadAttributes` + +Used to signal to initiate the payload build process via `notify_forkchoice_updated`. + +```python +@dataclass +class PayloadAttributes(object): + timestamp: uint64 + random: Bytes32 + fee_recipient: ExecutionAddress +``` + ### `PowBlock` ```python diff --git a/specs/merge/validator.md b/specs/merge/validator.md index 2de4ef6b1..cbca6eeda 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -11,9 +11,12 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Custom types](#custom-types) +- [Helpers](#helpers) + - [`get_pow_block_at_terminal_total_difficulty`](#get_pow_block_at_terminal_total_difficulty) + - [`get_terminal_pow_block`](#get_terminal_pow_block) + - [`get_payload_id`](#get_payload_id) - [Protocols](#protocols) - [`ExecutionEngine`](#executionengine) - - [`prepare_payload`](#prepare_payload) - [`get_payload`](#get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) @@ -29,7 +32,7 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This document is an extension of the [Altair -- Honest Validator](../altair/validator.md) guide. +This documens is an extension of the [Altair -- Honest Validator](../altair/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [The Merge](./beacon-chain.md) are requisite for this document and used throughout. @@ -39,7 +42,59 @@ Please see related Beacon Chain doc before continuing and use them as a referenc | Name | SSZ equivalent | Description | | - | - | - | -| `PayloadId` | `uint64` | Identifier of a payload building process | +| `PayloadId` | `Bytes8` | Identifier of a payload building process | + +## Helpers + +### `get_pow_block_at_terminal_total_difficulty` + +```python +def get_pow_block_at_terminal_total_difficulty(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]: + # `pow_chain` abstractly represents all blocks in the PoW chain + for block in pow_chain: + parent = get_pow_block(block.parent_hash) + block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY + parent_reached_ttd = parent.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY + if block_reached_ttd and not parent_reached_ttd: + return block + + return None +``` + +### `get_terminal_pow_block` + +```python +def get_terminal_pow_block(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]: + if TERMINAL_BLOCK_HASH != Hash32(): + # Terminal block hash override takes precedence over terminal total difficulty + pow_block_overrides = [block for block in pow_chain if block.block_hash == TERMINAL_BLOCK_HASH] + if not any(pow_block_overrides): + return None + return pow_block_overrides[0] + + return get_pow_block_at_terminal_total_difficulty(pow_chain) +``` + +### `get_payload_id` + +Given the `head_block_hash` and the `payload_attributes` that were used to +initiate the build process via `notify_forkchoice_updated`, `get_payload_id()` +returns the `payload_id` used to retrieve the payload via `get_payload`. + +```python +def get_payload_id(parent_hash: Hash32, payload_attributes: PayloadAttributes) -> PayloadId: + return PayloadId( + hash( + parent_hash + + uint_to_bytes(payload_attributes.timestamp) + + payload_attributes.random + + payload_attributes.fee_recipient + )[0:8] + ) +``` + +*Note*: This function does *not* use simple serialize `hash_tree_root` as to +avoid requiring simple serialize hashing capabilities in the Execution Layer. ## Protocols @@ -50,27 +105,10 @@ Please see related Beacon Chain doc before continuing and use them as a referenc The body of each of these functions is implementation dependent. The Engine API may be used to implement them with an external execution engine. -#### `prepare_payload` - -Given the set of execution payload attributes, `prepare_payload` initiates a process of building an execution payload -on top of the execution chain tip identified by `parent_hash`. - -```python -def prepare_payload(self: ExecutionEngine, - parent_hash: Hash32, - timestamp: uint64, - random: Bytes32, - fee_recipient: ExecutionAddress) -> PayloadId: - """ - Return ``payload_id`` that is used to obtain the execution payload in a subsequent ``get_payload`` call. - """ - ... -``` - #### `get_payload` Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that -has been built since the corresponding call to `prepare_payload` method. +has been built since the corresponding call to `notify_forkchoice_updated` method. ```python def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: @@ -92,38 +130,17 @@ All validator responsibilities remain unchanged other than those noted below. Na To obtain an execution payload, a block proposer building a block on top of a `state` must take the following actions: -1. Set `payload_id = prepare_execution_payload(state, pow_chain, fee_recipient, execution_engine)`, where: +1. Set `payload_id = prepare_execution_payload(state, pow_chain, finalized_block_hash, fee_recipient, execution_engine)`, where: * `state` is the state object after applying `process_slots(state, slot)` transition to the resulting state of the parent block processing * `pow_chain` is a list that abstractly represents all blocks in the PoW chain + * `finalized_block_hash` is the hash of the latest finalized execution payload (`Hash32()` if none yet finalized) * `fee_recipient` is the value suggested to be used for the `coinbase` field of the execution payload ```python -def get_pow_block_at_terminal_total_difficulty(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]: - # `pow_chain` abstractly represents all blocks in the PoW chain - for block in pow_chain: - parent = get_pow_block(block.parent_hash) - block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY - parent_reached_ttd = parent.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY - if block_reached_ttd and not parent_reached_ttd: - return block - - return None - - -def get_terminal_pow_block(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]: - if TERMINAL_BLOCK_HASH != Hash32(): - # Terminal block hash override takes precedence over terminal total difficulty - pow_block_overrides = [block for block in pow_chain if block.block_hash == TERMINAL_BLOCK_HASH] - if not any(pow_block_overrides): - return None - return pow_block_overrides[0] - - return get_pow_block_at_terminal_total_difficulty(pow_chain) - - def prepare_execution_payload(state: BeaconState, pow_chain: Sequence[PowBlock], + finalized_block_hash: Hash32, fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: if not is_merge_complete(state): @@ -137,9 +154,14 @@ def prepare_execution_payload(state: BeaconState, # Post-merge, normal payload parent_hash = state.latest_execution_payload_header.block_hash - timestamp = compute_timestamp_at_slot(state, state.slot) - random = get_randao_mix(state, get_current_epoch(state)) - return execution_engine.prepare_payload(parent_hash, timestamp, random, fee_recipient) + # Set the forkchoice head and initiate the payload build process + payload_attributes = PayloadAttributes( + timestamp=compute_timestamp_at_slot(state, state.slot), + random=get_randao_mix(state, get_current_epoch(state)), + fee_recipient=fee_recipient, + ) + execution_engine.notify_forkchoice_updated(parent_hash, finalized_block_hash, payload_attributes) + return get_payload_id(parent_hash, payload_attributes) ``` 2. Set `block.body.execution_payload = get_execution_payload(payload_id, execution_engine)`, where: From ba582b3e3a1848a856281869977dc2cdacf64504 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Oct 2021 11:25:50 +0800 Subject: [PATCH 07/15] Fix setup.py parser and rename `TBH_ACTIVATION_EPOCH` -> `TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH` --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- setup.py | 2 +- specs/merge/beacon-chain.md | 2 +- specs/merge/client-settings.md | 2 +- specs/merge/fork-choice.md | 2 +- specs/merge/validator.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 74896c90f..6d23073b7 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -9,7 +9,7 @@ PRESET_BASE: 'mainnet' TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 -TBH_ACTIVATION_EPOCH: 18446744073709551615 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 35e321e84..4f99fce31 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -9,7 +9,7 @@ PRESET_BASE: 'minimal' TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 -TBH_ACTIVATION_EPOCH: 18446744073709551615 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 diff --git a/setup.py b/setup.py index 863761daa..fe990fbee 100644 --- a/setup.py +++ b/setup.py @@ -603,7 +603,7 @@ def objects_to_spec(preset_name: str, # Access global dict of config vars for runtime configurables for name in spec_object.config_vars.keys(): - functions_spec = functions_spec.replace(name, 'config.' + name) + functions_spec = re.sub(r"\b%s\b" % name, 'config.' + name, functions_spec) def format_config_var(name: str, vardef: VariableDefinition) -> str: if vardef.type_name is None: diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index 12919b474..2e1fd88fb 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -75,7 +75,7 @@ This patch adds transaction execution to the beacon chain as part of the Merge f | - | - | | `TERMINAL_TOTAL_DIFFICULTY` | **TBD** | | `TERMINAL_BLOCK_HASH` | `Hash32()` | -| `TBH_ACTIVATION_EPOCH` | `FAR_FUTURE_EPOCH` | +| `TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH` | `FAR_FUTURE_EPOCH` | ## Containers diff --git a/specs/merge/client-settings.md b/specs/merge/client-settings.md index 63ddfe5ae..64f912372 100644 --- a/specs/merge/client-settings.md +++ b/specs/merge/client-settings.md @@ -24,6 +24,6 @@ Except under exceptional scenarios, this setting is not expected to be used. Suf To allow for transition coordination around a specific PoW block, clients must also provide `--terminal-block-hash-override` and `--terminal-block-hash-epoch-override` as configurable settings. * The value provided by `--terminal-block-hash-override` takes precedence over the pre-configured `TERMINAL_BLOCK_HASH` parameter. -* The value provided by `--terminal-block-hash-epoch-override` takes precedence over the pre-configured `TBH_ACTIVATION_EPOCH` parameter. +* The value provided by `--terminal-block-hash-epoch-override` takes precedence over the pre-configured `TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH` parameter. Except under exceptional scenarios, these settings are not expected to be used. Sufficient warning to the user about this exceptional configurable setting should be provided. diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index f1f5e265c..d5fa08ee2 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -114,7 +114,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: pow_parent = get_pow_block(pow_block.parent_hash) assert is_valid_terminal_pow_block(pow_block, pow_parent) if TERMINAL_BLOCK_HASH != Hash32(): - assert compute_epoch_at_slot(block.slot) >= TBH_ACTIVATION_EPOCH + assert compute_epoch_at_slot(block.slot) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH # Add new block to the store store.blocks[hash_tree_root(block)] = block diff --git a/specs/merge/validator.md b/specs/merge/validator.md index f15f62f3d..e0066ddce 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -128,7 +128,7 @@ def prepare_execution_payload(state: BeaconState, execution_engine: ExecutionEngine) -> Optional[PayloadId]: if not is_merge_complete(state): is_tbh_override_set = TERMINAL_BLOCK_HASH != Hash32() - if is_tbh_override_set and get_current_epoch(state.slot) < TBH_ACTIVATION_EPOCH: + if is_tbh_override_set and get_current_epoch(state.slot) < TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: # TBH is set but activation epoch is not yet reached, no prepare payload call is needed return None From 34335e0334763bc0d3f7f56a8c1707add053aac7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Oct 2021 15:30:49 +0800 Subject: [PATCH 08/15] Remove `prepare_payload` leftover --- setup.py | 7 ------- specs/merge/validator.md | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/setup.py b/setup.py index fa396ab50..6ca46c636 100644 --- a/setup.py +++ b/setup.py @@ -533,13 +533,6 @@ class NoopExecutionEngine(ExecutionEngine): payload_attributes: Optional[PayloadAttributes]) -> None: pass - def prepare_payload(self: ExecutionEngine, - parent_hash: Hash32, - timestamp: uint64, - random: Bytes32, - feeRecipient: ExecutionAddress) -> PayloadId: - raise NotImplementedError("no default block production") - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: raise NotImplementedError("no default block production") diff --git a/specs/merge/validator.md b/specs/merge/validator.md index cbca6eeda..379687dc4 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -100,7 +100,7 @@ avoid requiring simple serialize hashing capabilities in the Execution Layer. ### `ExecutionEngine` -*Note*: `prepare_payload` and `get_payload` functions are added to the `ExecutionEngine` protocol for use as a validator. +*Note*: `get_payload` function is added to the `ExecutionEngine` protocol for use as a validator. The body of each of these functions is implementation dependent. The Engine API may be used to implement them with an external execution engine. From 294b60a48befc76177ca72caadbe3a77f27ca8eb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Oct 2021 09:24:07 -0600 Subject: [PATCH 09/15] pr feedback --- specs/merge/fork-choice.md | 2 ++ specs/merge/validator.md | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/merge/fork-choice.md b/specs/merge/fork-choice.md index d5fa08ee2..83bba3adf 100644 --- a/specs/merge/fork-choice.md +++ b/specs/merge/fork-choice.md @@ -110,9 +110,11 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # [New in Merge] if is_merge_block(pre_state, block.body): + # Check the parent PoW block of execution payload is a valid terminal PoW block. pow_block = get_pow_block(block.body.execution_payload.parent_hash) pow_parent = get_pow_block(pow_block.parent_hash) assert is_valid_terminal_pow_block(pow_block, pow_parent) + # If `TERMINAL_BLOCK_HASH` is used as an override, the activation epoch must be reached. if TERMINAL_BLOCK_HASH != Hash32(): assert compute_epoch_at_slot(block.slot) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH diff --git a/specs/merge/validator.md b/specs/merge/validator.md index e0066ddce..88bf96643 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -127,9 +127,10 @@ def prepare_execution_payload(state: BeaconState, fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: if not is_merge_complete(state): - is_tbh_override_set = TERMINAL_BLOCK_HASH != Hash32() - if is_tbh_override_set and get_current_epoch(state.slot) < TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: - # TBH is set but activation epoch is not yet reached, no prepare payload call is needed + is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() + is_activation_epoch_reached = get_current_epoch(state.slot) < TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH + if is_terminal_block_hash_set and is_activation_epoch_reached: + # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed return None terminal_pow_block = get_terminal_pow_block(pow_chain) From 8023edc94bd13c1f8cb396d1c6dc774dedc4d71b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Oct 2021 09:28:52 -0600 Subject: [PATCH 10/15] cleanup some copy relatedto removal of prepare_payload --- specs/merge/validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/merge/validator.md b/specs/merge/validator.md index 379687dc4..1e90b6b38 100644 --- a/specs/merge/validator.md +++ b/specs/merge/validator.md @@ -32,7 +32,7 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This documens is an extension of the [Altair -- Honest Validator](../altair/validator.md) guide. +This document is an extension of the [Altair -- Honest Validator](../altair/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [The Merge](./beacon-chain.md) are requisite for this document and used throughout. @@ -102,8 +102,8 @@ avoid requiring simple serialize hashing capabilities in the Execution Layer. *Note*: `get_payload` function is added to the `ExecutionEngine` protocol for use as a validator. -The body of each of these functions is implementation dependent. -The Engine API may be used to implement them with an external execution engine. +The body of this function is implementation dependent. +The Engine API may be used to implement it with an external execution engine. #### `get_payload` From 8a27a7cb5d3c252c7393c3facb1420c211a3bd22 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Oct 2021 15:46:18 -0600 Subject: [PATCH 11/15] remove union type for eecution-payload txs --- setup.py | 4 ++-- specs/merge/beacon-chain.md | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index e414d2721..4524df2ca 100644 --- a/setup.py +++ b/setup.py @@ -497,7 +497,7 @@ class MergeSpecBuilder(AltairSpecBuilder): return super().imports(preset_name) + f''' from typing import Protocol from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector, uint256, Union +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector, uint256 ''' @classmethod @@ -543,7 +543,7 @@ EXECUTION_ENGINE = NoopExecutionEngine()""" @classmethod def hardcoded_custom_type_dep_constants(cls) -> str: constants = { - 'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)', + 'MAX_BYTES_PER_TRANSACTION': 'uint64(2**20)', } return {**super().hardcoded_custom_type_dep_constants(), **constants} diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index 2e1fd88fb..8fafb0720 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -50,8 +50,7 @@ This patch adds transaction execution to the beacon chain as part of the Merge f | Name | SSZ equivalent | Description | | - | - | - | -| `OpaqueTransaction` | `ByteList[MAX_BYTES_PER_OPAQUE_TRANSACTION]` | a [typed transaction envelope](https://eips.ethereum.org/EIPS/eip-2718#opaque-byte-array-rather-than-an-rlp-array) structured as `TransactionType \|\| TransactionPayload` | -| `Transaction` | `Union[OpaqueTransaction]` | a transaction | +| `Transaction` | `ByteList[MAX_BYTES_PER_TRANSACTION]` | either a [typed transaction envelope](https://eips.ethereum.org/EIPS/eip-2718#opaque-byte-array-rather-than-an-rlp-array) or a legacy transaction| | `ExecutionAddress` | `Bytes20` | Address of account on the execution layer | ## Constants @@ -60,7 +59,7 @@ This patch adds transaction execution to the beacon chain as part of the Merge f | Name | Value | | - | - | -| `MAX_BYTES_PER_OPAQUE_TRANSACTION` | `uint64(2**20)` (= 1,048,576) | +| `MAX_BYTES_PER_TRANSACTION` | `uint64(2**20)` (= 1,048,576) | | `MAX_TRANSACTIONS_PER_PAYLOAD` | `uint64(2**14)` (= 16,384) | | `BYTES_PER_LOGS_BLOOM` | `uint64(2**8)` (= 256) | | `GAS_LIMIT_DENOMINATOR` | `uint64(2**10)` (= 1,024) | From c6af7b3228001b283fcc66d030d0babc5c7ad758 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Oct 2021 15:46:41 -0600 Subject: [PATCH 12/15] add merge execution values to preset yaml files --- presets/mainnet/merge.yaml | 15 ++++++++++++++- presets/minimal/merge.yaml | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/presets/mainnet/merge.yaml b/presets/mainnet/merge.yaml index 97f07c7f0..66f7abe28 100644 --- a/presets/mainnet/merge.yaml +++ b/presets/mainnet/merge.yaml @@ -1,3 +1,16 @@ # Mainnet preset - The Merge -# No presets here. +# Execution +# --------------------------------------------------------------- +# 2**20 (= 1,048,576) +MAX_BYTES_PER_TRANSACTION: 1048576 +# 2**14 (= 16,384) +MAX_TRANSACTIONS_PER_PAYLOAD: 16384 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**10 (= 1,024) +GAS_LIMIT_DENOMINATOR: 1024 +# 5,000 +MIN_GAS_LIMIT: 5000 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 diff --git a/presets/minimal/merge.yaml b/presets/minimal/merge.yaml index 88aa86c09..d9b5640dd 100644 --- a/presets/minimal/merge.yaml +++ b/presets/minimal/merge.yaml @@ -1,3 +1,16 @@ # Minimal preset - The Merge -# No presets here. +# Execution +# --------------------------------------------------------------- +# 2**20 (= 1,048,576) +MAX_BYTES_PER_TRANSACTION: 1048576 +# 2**14 (= 16,384) +MAX_TRANSACTIONS_PER_PAYLOAD: 16384 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**10 (= 1,024) +GAS_LIMIT_DENOMINATOR: 1024 +# 5,000 +MIN_GAS_LIMIT: 5000 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 From edb5a023c85fde7756fb711a2cf1d601dfe2b417 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 Oct 2021 12:24:00 -0600 Subject: [PATCH 13/15] remove extraneous p2p condition --- specs/merge/p2p-interface.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/merge/p2p-interface.md b/specs/merge/p2p-interface.md index 954c1b008..3cbba2e23 100644 --- a/specs/merge/p2p-interface.md +++ b/specs/merge/p2p-interface.md @@ -74,8 +74,6 @@ Alias `block = signed_beacon_block.message`, `execution_payload = block.body.exe -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. - _[REJECT]_ Gas used is less than the gas limit -- i.e. `execution_payload.gas_used <= execution_payload.gas_limit`. - - _[REJECT]_ The execution payload block hash is not equal to the parent hash -- - i.e. `execution_payload.block_hash != execution_payload.parent_hash`. - _[REJECT]_ The execution payload transaction list data is within expected size limits, the data MUST NOT be larger than the SSZ list-limit, and a client MAY be more strict. From cb513aa82d19feb81a85915df33b290df1be595d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 Oct 2021 11:37:02 +0800 Subject: [PATCH 14/15] --fork cli option: fix the case of unset directory path + validate fork name --- tests/core/pyspec/eth2spec/test/conftest.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 55613be0a..751d4d3f1 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -49,6 +49,15 @@ def pytest_addoption(parser): ) +def _validate_fork_name(forks): + for fork in forks: + if fork not in ALL_PHASES: + raise ValueError( + f'The given --fork argument "{fork}" is not an available fork.' + f' The available forks: {ALL_PHASES}' + ) + + @fixture(autouse=True) def preset(request): context.DEFAULT_TEST_PRESET = request.config.getoption("--preset") @@ -56,10 +65,11 @@ def preset(request): @fixture(autouse=True) def run_phases(request): - phases = request.config.getoption("--fork") - if phases: - phases = [phase.lower() for phase in phases] - context.DEFAULT_PYTEST_FORKS = set(phases) + forks = request.config.getoption("--fork", default=None) + if forks: + forks = [phase.lower() for phase in forks] + _validate_fork_name(forks) + context.DEFAULT_PYTEST_FORKS = set(forks) else: context.DEFAULT_PYTEST_FORKS = ALL_PHASES From 405f2e2c2f84f1f435105a6b86244585262706a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 Oct 2021 23:21:56 +0800 Subject: [PATCH 15/15] Update tests/core/pyspec/eth2spec/test/conftest.py Co-authored-by: Danny Ryan --- tests/core/pyspec/eth2spec/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 751d4d3f1..b3c250c11 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -67,7 +67,7 @@ def preset(request): def run_phases(request): forks = request.config.getoption("--fork", default=None) if forks: - forks = [phase.lower() for phase in forks] + forks = [fork.lower() for fork in forks] _validate_fork_name(forks) context.DEFAULT_PYTEST_FORKS = set(forks) else: