Merge branch 'dev' into terminal_blockhash_override_2

configs/mainnet.yaml

@@ -31,7 +31,7 @@ GENESIS_DELAY: 604800
 
 # Altair
 ALTAIR_FORK_VERSION: 0x01000000
-ALTAIR_FORK_EPOCH: 18446744073709551615
+ALTAIR_FORK_EPOCH: 74240  # Oct 27, 2021, 10:56:23am UTC
 # Merge
 MERGE_FORK_VERSION: 0x02000000
 MERGE_FORK_EPOCH: 18446744073709551615

setup.py

@@ -32,7 +32,7 @@ except ImportError:
 from marko.block import Heading, FencedCode, LinkRefDef, BlankLine
 from marko.inline import CodeSpan
 from marko.ext.gfm import gfm
-from marko.ext.gfm.elements import Table, Paragraph
+from marko.ext.gfm.elements import Table
 
 
 # Definitions in context.py
@@ -522,16 +522,23 @@ def get_pow_chain_head() -> PowBlock:
 
 class NoopExecutionEngine(ExecutionEngine):
 
-    def on_payload(self, execution_payload: ExecutionPayload) -> bool:
+    def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool:
         return True
 
-    def set_head(self, block_hash: Hash32) -> bool:
-        return True
+    def notify_consensus_validated(self: ExecutionEngine, block_hash: Hash32, valid: bool) -> None:
+        pass
 
-    def finalize_block(self, block_hash: Hash32) -> bool:
-        return True
+    def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, finalized_block_hash: Hash32) -> None:
+        pass
 
-    def assemble_block(self, block_hash: Hash32, timestamp: uint64, random: Bytes32) -> ExecutionPayload:
+    def prepare_payload(self: ExecutionEngine,
+                        parent_hash: Hash32,
+                        timestamp: uint64,
+                        random: Bytes32,
+                        feeRecipient: ExecutionAddress) -> PayloadId:
+        raise NotImplementedError("no default block production")
+
+    def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload:
         raise NotImplementedError("no default block production")
 
 

specs/altair/fork.md

@@ -1,7 +1,5 @@
 # Altair -- Fork Logic
 
-**Notice**: This document is a work-in-progress for researchers and implementers.
-
 ## Table of contents
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
@@ -26,13 +24,13 @@ Warning: this configuration is not definitive.
 | Name | Value |
 | - | - |
 | `ALTAIR_FORK_VERSION` | `Version('0x01000000')` |
-| `ALTAIR_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** |
+| `ALTAIR_FORK_EPOCH` | `Epoch(74240)` (Oct 27, 2021, 10:56:23am UTC) |
 
 ## Fork to Altair
 
 ### Fork trigger
 
-TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at epoch `ALTAIR_FORK_EPOCH`.
+The fork is triggered at epoch `ALTAIR_FORK_EPOCH`.
 
 Note that for the pure Altair networks, we don't apply `upgrade_to_altair` since it starts with Altair version logic.
 

specs/merge/beacon-chain.md

@@ -31,7 +31,8 @@
     - [`compute_timestamp_at_slot`](#compute_timestamp_at_slot)
 - [Beacon chain state transition function](#beacon-chain-state-transition-function)
   - [Execution engine](#execution-engine)
-    - [`on_payload`](#on_payload)
+    - [`execute_payload`](#execute_payload)
+    - [`notify_consensus_validated`](#notify_consensus_validated)
   - [Block processing](#block-processing)
   - [Execution payload processing](#execution-payload-processing)
     - [`is_valid_gas_limit`](#is_valid_gas_limit)
@@ -53,6 +54,7 @@ This patch adds transaction execution to the beacon chain as part of the Merge f
 | - | - | - |
 | `OpaqueTransaction` | `ByteList[MAX_BYTES_PER_OPAQUE_TRANSACTION]` | a [typed transaction envelope](https://eips.ethereum.org/EIPS/eip-2718#opaque-byte-array-rather-than-an-rlp-array) structured as `TransactionType \|\| TransactionPayload` |
 | `Transaction` | `Union[OpaqueTransaction]` | a transaction |
+| `ExecutionAddress` | `Bytes20` | Address of account on the execution layer |
 
 ## Constants
 
@@ -159,7 +161,7 @@ class BeaconState(Container):
 class ExecutionPayload(Container):
     # Execution block header fields
     parent_hash: Hash32
-    coinbase: Bytes20  # 'beneficiary' in the yellow paper
+    coinbase: ExecutionAddress  # 'beneficiary' in the yellow paper
     state_root: Bytes32
     receipt_root: Bytes32  # 'receipts root' in the yellow paper
     logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM]
@@ -181,7 +183,7 @@ class ExecutionPayload(Container):
 class ExecutionPayloadHeader(Container):
     # Execution block header fields
     parent_hash: Hash32
-    coinbase: Bytes20
+    coinbase: ExecutionAddress
     state_root: Bytes32
     receipt_root: Bytes32
     logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM]
@@ -241,19 +243,38 @@ def compute_timestamp_at_slot(state: BeaconState, slot: Slot) -> uint64:
 The implementation-dependent `ExecutionEngine` protocol encapsulates the execution sub-system logic via:
 
 * a state object `self.execution_state` of type `ExecutionState`
-* a state transition function `self.on_payload` which mutates `self.execution_state`
+* a state transition function `self.execute_payload` which applies changes to the `self.execution_state`
+* a function `self.notify_consensus_validated` which signals that the beacon block containing the execution payload
+is valid with respect to the consensus rule set
 
-#### `on_payload`
+*Note*: `execute_payload` and `notify_consensus_validated` are functions accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol.
+
+The body of each of these functions is implementation dependent.
+The Engine API may be used to implement them with an external execution engine.
+
+#### `execute_payload`
 
 ```python
-def on_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool:
+def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool:
     """
     Returns ``True`` iff ``execution_payload`` is valid with respect to ``self.execution_state``.
     """
     ...
 ```
 
-The above function is accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol.
+#### `notify_consensus_validated`
+
+```python
+def notify_consensus_validated(self: ExecutionEngine, block_hash: Hash32, valid: bool) -> None:
+    ...
+```
+
+The inputs to this function depend on the result of the state transition. A call to `notify_consensus_validated` must be made after the [`state_transition`](../phase0/beacon-chain.md#beacon-chain-state-transition-function) function finishes. The value of the `valid` parameter must be set as follows:
+
+* `True` if `state_transition` function call succeeds
+* `False` if `state_transition` function call fails
+
+*Note*: The call of the `notify_consensus_validated` function with `valid = True` maps on the `POS_CONSENSUS_VALIDATED` event defined in the [EIP-3675](https://eips.ethereum.org/EIPS/eip-3675#definitions).
 
 ### Block processing
 
@@ -310,7 +331,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe
     # Verify timestamp
     assert payload.timestamp == compute_timestamp_at_slot(state, state.slot)
     # Verify the execution payload is valid
-    assert execution_engine.on_payload(payload)
+    assert execution_engine.execute_payload(payload)
     # Cache execution payload header
     state.latest_execution_payload_header = ExecutionPayloadHeader(
         parent_hash=payload.parent_hash,

specs/merge/fork-choice.md

@@ -10,8 +10,7 @@
 - [Introduction](#introduction)
 - [Protocols](#protocols)
   - [`ExecutionEngine`](#executionengine)
-    - [`set_head`](#set_head)
-    - [`finalize_block`](#finalize_block)
+    - [`notify_forkchoice_updated`](#notify_forkchoice_updated)
 - [Helpers](#helpers)
   - [`PowBlock`](#powblock)
   - [`get_pow_block`](#get_pow_block)
@@ -32,38 +31,24 @@ This is the modification of the fork choice according to the executable beacon c
 
 ### `ExecutionEngine`
 
-The following methods are added to the `ExecutionEngine` protocol for use in the fork choice:
-
-#### `set_head`
-
-Re-organizes the execution payload chain and corresponding state to make `block_hash` the head.
+*Note*: The `notify_forkchoice_updated` function is added to the `ExecutionEngine` protocol to signal the fork choice updates.
 
 The body of this function is implementation dependent.
-The Consensus API may be used to implement this with an external execution engine.
+The Engine API may be used to implement it with an external execution engine.
+
+#### `notify_forkchoice_updated`
+
+This function performs two actions *atomically*:
+* Re-organizes the execution payload chain and corresponding state to make `head_block_hash` the head.
+* Applies finality to the execution state: it irreversibly persists the chain of all execution payloads
+and corresponding state, up to and including `finalized_block_hash`.
 
 ```python
-def set_head(self: ExecutionEngine, block_hash: Hash32) -> bool:
-    """
-    Returns True if the ``block_hash`` was successfully set as head of the execution payload chain.
-    """
+def notify_forkchoice_updated(self: ExecutionEngine, head_block_hash: Hash32, finalized_block_hash: Hash32) -> None:
     ...
 ```
 
-#### `finalize_block`
-
-Applies finality to the execution state: it irreversibly persists the chain of all execution payloads
-and corresponding state, up to and including `block_hash`.
-
-The body of this function is implementation dependent.
-The Consensus API may be used to implement this with an external execution engine.
-
-```python
-def finalize_block(self: ExecutionEngine, block_hash: Hash32) -> bool:
-    """
-    Returns True if the data up to and including ``block_hash`` was successfully finalized.
-    """
-    ...
-```
+*Note*: The call of the `notify_forkchoice_updated` function maps on the `POS_FORKCHOICE_UPDATED` event defined in the [EIP-3675](https://eips.ethereum.org/EIPS/eip-3675#definitions).
 
 ## Helpers
 

specs/merge/p2p-interface.md

@@ -68,14 +68,10 @@ See the Merge [state transition document](./beacon-chain.md#beaconblockbody) for
 In addition to the gossip validations for this topic from prior specifications,
 the following validations MUST pass before forwarding the `signed_beacon_block` on the network.
 Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`.
-- If the merge is complete with respect to the head state -- i.e. `is_merge_complete(state)` --
-  then validate the following:
-  - _[REJECT]_ The block's execution payload must be non-empty --
-    i.e. `execution_payload != ExecutionPayload()`
 - If the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`
   then validate the following:
   - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot
-    -- i.e. `execution_payload.timestamp == compute_time_at_slot(state, block.slot)`.
+    -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`.
   - _[REJECT]_ Gas used is less than the gas limit --
     i.e. `execution_payload.gas_used <= execution_payload.gas_limit`.
   - _[REJECT]_ The execution payload block hash is not equal to the parent hash --
@@ -89,7 +85,7 @@ Alias `block = signed_beacon_block.message`, `execution_payload = block.body.exe
 
 ### Transitioning the gossip
 
-See gossip transition details found in the [Altair document](../altair/p2p) for
+See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for
 details on how to handle transitioning gossip topics for the Merge.
 
 ## The Req/Resp domain

specs/merge/validator.md

@@ -10,13 +10,15 @@
 
 - [Introduction](#introduction)
 - [Prerequisites](#prerequisites)
+- [Custom types](#custom-types)
 - [Protocols](#protocols)
   - [`ExecutionEngine`](#executionengine)
-    - [`assemble_block`](#assemble_block)
+    - [`prepare_payload`](#prepare_payload)
+    - [`get_payload`](#get_payload)
 - [Beacon chain responsibilities](#beacon-chain-responsibilities)
   - [Block proposal](#block-proposal)
     - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody)
-      - [Execution Payload](#execution-payload)
+      - [ExecutionPayload](#executionpayload)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
@@ -33,22 +35,48 @@ All behaviors and definitions defined in this document, and documents it extends
 All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [The Merge](./beacon-chain.md) are requisite for this document and used throughout.
 Please see related Beacon Chain doc before continuing and use them as a reference throughout.
 
+## Custom types
+
+| Name | SSZ equivalent | Description |
+| - | - | - |
+| `PayloadId` | `uint64` | Identifier of a payload building process |
+
 ## Protocols
 
 ### `ExecutionEngine`
 
-The following methods are added to the `ExecutionEngine` protocol for use as a validator:
+*Note*: `prepare_payload` and `get_payload` functions are added to the `ExecutionEngine` protocol for use as a validator.
 
-#### `assemble_block`
+The body of each of these functions is implementation dependent.
+The Engine API may be used to implement them with an external execution engine.
 
-Produces a new instance of an execution payload, with the specified `timestamp`,
-on top of the execution payload chain tip identified by `block_hash`.
+#### `prepare_payload`
 
-The body of this function is implementation dependent.
-The Consensus API may be used to implement this with an external execution engine.
+Given the set of execution payload attributes, `prepare_payload` initiates a process of building an execution payload
+on top of the execution chain tip identified by `parent_hash`.
 
 ```python
-def assemble_block(self: ExecutionEngine, block_hash: Hash32, timestamp: uint64, random: Bytes32) -> ExecutionPayload:
+def prepare_payload(self: ExecutionEngine,
+                    parent_hash: Hash32,
+                    timestamp: uint64,
+                    random: Bytes32,
+                    fee_recipient: ExecutionAddress) -> PayloadId:
+    """
+    Return ``payload_id`` that is used to obtain the execution payload in a subsequent ``get_payload`` call.
+    """
+    ...
+```
+
+#### `get_payload`
+
+Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that
+has been built since the corresponding call to `prepare_payload` method.
+
+```python
+def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload:
+    """
+    Return ``execution_payload`` object.
+    """
     ...
 ```
 
@@ -60,9 +88,15 @@ All validator responsibilities remain unchanged other than those noted below. Na
 
 #### Constructing the `BeaconBlockBody`
 
-##### Execution Payload
+##### ExecutionPayload
+
+To obtain an execution payload, a block proposer building a block on top of a `state` must take the following actions:
+
+1. Set `payload_id = prepare_execution_payload(state, pow_chain, fee_recipient, execution_engine)`, where:
+    * `state` is the state object after applying `process_slots(state, slot)` transition to the resulting state of the parent block processing
+    * `pow_chain` is a list that abstractly represents all blocks in the PoW chain
+    * `fee_recipient` is the value suggested to be used for the `coinbase` field of the execution payload
 
-* Set `block.body.execution_payload = get_execution_payload(state, execution_engine, pow_chain)` where:
 
 ```python
 def get_pow_block_at_terminal_total_difficulty(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]:
@@ -81,35 +115,43 @@ def get_terminal_pow_block(pow_chain: Sequence[PowBlock]) -> Optional[PowBlock]:
     if TERMINAL_BLOCK_HASH != Hash32():
         # Terminal block hash override takes precedence over terminal total difficulty
         pow_block_overrides = [block for block in pow_chain if block.block_hash == TERMINAL_BLOCK_HASH]
-        if len(pow_block_overrides) != 0:
-            return pow_block_overrides[0]
-        else:
+        if not any(pow_block_overrides):
             return None
+        return pow_block_overrides[0]
 
     return get_pow_block_at_terminal_total_difficulty(pow_chain)
 
 
-def produce_execution_payload(state: BeaconState,
-                              parent_hash: Hash32,
-                              execution_engine: ExecutionEngine) -> ExecutionPayload:
-    timestamp = compute_timestamp_at_slot(state, state.slot)
-    randao_mix = get_randao_mix(state, get_current_epoch(state))
-    return execution_engine.assemble_block(parent_hash, timestamp, randao_mix)
-
-
-def get_execution_payload(state: BeaconState,
-                          execution_engine: ExecutionEngine,
-                          pow_chain: Sequence[PowBlock]) -> ExecutionPayload:
+def prepare_execution_payload(state: BeaconState,
+                              pow_chain: Sequence[PowBlock],
+                              fee_recipient: ExecutionAddress,
+                              execution_engine: ExecutionEngine) -> Optional[PayloadId]:
     if not is_merge_complete(state):
         terminal_pow_block = get_terminal_pow_block(pow_chain)
         if terminal_pow_block is None:
-            # Pre-merge, empty payload
-            return ExecutionPayload()
-        else:
-            # Signify merge via producing on top of the last PoW block
-            return produce_execution_payload(state, terminal_pow_block.block_hash, execution_engine)
+            # Pre-merge, no prepare payload call is needed
+            return None
+        # Signify merge via producing on top of the terminal PoW block
+        parent_hash = terminal_pow_block.block_hash
+    else:
+        # Post-merge, normal payload
+        parent_hash = state.latest_execution_payload_header.block_hash
 
-    # Post-merge, normal payload
-    parent_hash = state.latest_execution_payload_header.block_hash
-    return produce_execution_payload(state, parent_hash, execution_engine)
+    timestamp = compute_timestamp_at_slot(state, state.slot)
+    random = get_randao_mix(state, get_current_epoch(state))
+    return execution_engine.prepare_payload(parent_hash, timestamp, random, fee_recipient)
 ```
+
+2. Set `block.body.execution_payload = get_execution_payload(payload_id, execution_engine)`, where:
+
+```python
+def get_execution_payload(payload_id: Optional[PayloadId], execution_engine: ExecutionEngine) -> ExecutionPayload:
+    if payload_id is None:
+        # Pre-merge, empty payload
+        return ExecutionPayload()
+    else:
+        return execution_engine.get_payload(payload_id)
+```
+
+*Note*: It is recommended for a validator to call `prepare_execution_payload` as soon as input parameters become known,
+and make subsequent calls to this function when any of these parameters gets updated.

tests/core/pyspec/eth2spec/VERSION.txt

@@ -1 +1 @@
-1.1.0-beta.4
+1.1.0-beta.5

tests/core/pyspec/eth2spec/test/altair/merkle/test_single_proof.py

@@ -8,11 +8,14 @@ from eth2spec.test.helpers.merkle import build_proof
 
 @with_phases([ALTAIR])
 @spec_state_test
-def test_next_sync_committee_tree(spec, state):
-    state.next_sync_committee: object = spec.SyncCommittee(
-        pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)]
-    )
+def test_next_sync_committee_merkle_proof(spec, state):
+    yield "state", state
     next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX)
+    yield "proof", {
+        "leaf": "0x" + state.next_sync_committee.hash_tree_root().hex(),
+        "leaf_index": spec.NEXT_SYNC_COMMITTEE_INDEX,
+        "branch": ['0x' + root.hex() for root in next_sync_committee_branch]
+    }
     assert spec.is_valid_merkle_branch(
         leaf=state.next_sync_committee.hash_tree_root(),
         branch=next_sync_committee_branch,
@@ -24,8 +27,15 @@ def test_next_sync_committee_tree(spec, state):
 
 @with_phases([ALTAIR])
 @spec_state_test
-def test_finality_root_tree(spec, state):
+def test_finality_root_merkle_proof(spec, state):
+    yield "state", state
     finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX)
+    yield "proof", {
+        "leaf": "0x" + state.finalized_checkpoint.root.hex(),
+        "leaf_index": spec.FINALIZED_ROOT_INDEX,
+        "branch": ['0x' + root.hex() for root in finality_branch]
+    }
+
     assert spec.is_valid_merkle_branch(
         leaf=state.finalized_checkpoint.root,
         branch=finality_branch,

tests/core/pyspec/eth2spec/test/helpers/execution_payload.py

@@ -11,7 +11,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None):
 
     payload = spec.ExecutionPayload(
         parent_hash=latest.block_hash,
-        coinbase=spec.Bytes20(),
+        coinbase=spec.ExecutionAddress(),
         state_root=latest.state_root,  # no changes to the state
         receipt_root=b"no receipts here" + b"\x00" * 16,  # TODO: root of empty MPT may be better.
         logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](),  # TODO: zeroed logs bloom for empty logs ok?

tests/core/pyspec/eth2spec/test/merge/block_processing/test_process_execution_payload.py

@@ -25,7 +25,7 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True,
     called_new_block = False
 
     class TestEngine(spec.NoopExecutionEngine):
-        def on_payload(self, payload) -> bool:
+        def execute_payload(self, payload) -> bool:
             nonlocal called_new_block, execution_valid
             called_new_block = True
             assert payload == execution_payload

tests/formats/merkle/README.md

@@ -0,0 +1,8 @@
+# Merkle tests
+
+This series of tests provides reference test vectors for validating correct
+generation and verification of merkle proofs based on static data.
+
+Handlers:
+- `single_proof`: see [Single leaf proof test format](./single_proof.md)
+- Different types of merkle proofs may be supported in the future.

tests/formats/merkle/single_proof.md

@@ -0,0 +1,28 @@
+# Single leaf merkle proof tests
+
+This series of tests provides reference test vectors for validating correct
+generation and verification of merkle proofs based on static data.
+
+## Test case format
+
+### `state.ssz_snappy`
+
+An SSZ-snappy encoded `BeaconState` object from which other data is generated.
+
+### `proof.yaml`
+
+A proof of the leaf value (a merkle root) at generalized-index `leaf_index` in the given `state`.
+
+```yaml
+leaf: Bytes32            # string, hex encoded, with 0x prefix
+leaf_index: int          # integer, decimal
+branch: list of Bytes32  # list, each element is a string, hex encoded, with 0x prefix 
+```
+
+## Condition
+
+A test-runner can implement the following assertions:
+- Check that `is_valid_merkle_branch` confirms `leaf` at `leaf_index` to verify
+  against `has_tree_root(state)` and `proof`.
+- If the implementation supports generating merkle proofs, check that the
+  self-generated proof matches the `proof` provided with the test.

tests/generators/merkle/README.md

@@ -0,0 +1,6 @@
+# Merkle
+
+The purpose of this test-generator is to provide test-vectors for validating the 
+correct merkleization of objects and corresponding merkle proofs.
+
+Test-format documentation can be found [here](../../formats/merkle/README.md).

tests/generators/merkle/main.py

@@ -0,0 +1,14 @@
+from eth2spec.test.helpers.constants import ALTAIR
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+
+
+if __name__ == "__main__":
+    altair_mods = {key: 'eth2spec.test.altair.merkle.test_' + key for key in [
+        'single_proof',
+    ]}
+
+    all_mods = {
+        ALTAIR: altair_mods
+    }
+
+    run_state_test_generators(runner_name="merkle", all_mods=all_mods)

tests/generators/merkle/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]