From c99e48c60cc06a93e86cc61214012741c9b58cf5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 6 Dec 2021 10:07:54 +1100 Subject: [PATCH 001/112] Add first efforts --- sync/optimistic.md | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 sync/optimistic.md diff --git a/sync/optimistic.md b/sync/optimistic.md new file mode 100644 index 000000000..b3220b96f --- /dev/null +++ b/sync/optimistic.md @@ -0,0 +1,105 @@ +# Optimistic Sync + +## Introduction + +In order to provide a syncing execution engine with a (partially-verified) view +of the head of the chain, it may be desirable for a consensus engine to import +beacon blocks without verifying the execution payloads. This partial sync is +called an *optimistic sync*. + +## Mechanisms + +To perform an optimistic sync: + +- The `execute_payload` function MUST return `True` if the execution + engine returns SYNCING or VALID. An INVALID response MUST return `False`. +- The `validate_merge_block` function MUST NOT raise an assertion if both the + `pow_block` and `pow_parent` are unknown to the execution engine. + +In addition to these changes to validation, the consensus engine MUST be able +to ascertain, after import, which blocks returned SYNCING and which returned +VALID. This document will assume consensus engines store the following sets: + +- `valid_roots: Set[Root]`: `hash_tree_root(block)` where + `block.body.execution_payload` is known to be VALID. +- `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where + `block.body.execution_payload` is known to be SYNCING. + +Notably, `optimistic_roots` only includes blocks which have execution enabled +whilst `valid_roots` contains blocks *with or without* execution enabled (i.e., +all blocks). + +A consensus engine MUST be able to retrospectively (i.e., after import) modify +the status of SYNCING blocks to be either VALID or INVALID based upon responses +from an execution engine. I.e., perform the following transitions: + +- SYNCING -> VALID +- SYNCING -> INVALID + +When a block transitions from SYNCING -> VALID, all *ancestors* of the block MUST +also transition from SYNCING -> VALID. + +When a block transitions from SYNCING -> INVALID, all *descendants* of the +block MUST also transition from SYNCING -> INVALID. + +## Fork Choice + +Consensus engines MUST support removing blocks that transition from SYNCING to +INVALID. Specifically, an INVALID block MUST NOT be included in the canonical +chain and the weights from INVALID blocks MUST NOT be applied to any VALID or +SYNCING ancestors. + +## Validator assignments + +An entirely optimistically synced node is *not* a full node. It is unable to +produce blocks, since an execution engine cannot produce a payload upon an +unknown parent. It cannot faithfully attest to the head block of the chain, +since it has not fully verified that block. + +Let `head_block: BeaconBlock` be the result of calling of the fork choice +algorithm at the time of block production. Let `justified_block: BeaconBlock` +be the latest current justified ancestor ancestor of the `head_block`. + +```python +def is_optimistic(block: BeaconBlock) -> bool: + hash_tree_root(block) in optimistic_roots +``` + +```python +def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: + while block.parent_root != Root(): + if not is_optimistic(block): + return block + block = get_block(block.parent_root) + + return None +``` + +Let a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let a validator on an optimistic node be an *optimistic validator*. + +### Block production + +A optimistic validator MUST NOT produce a block (i.e., sign across the +`DOMAIN_BEACON_PROPOSER` domain), unless one of the follow exceptions are met: + +#### Exception 1. + +If the justified block is fully verified (i.e., `not +is_optimistic(justified_block)`, the validator MUST produce a block upon +`latest_valid_ancestor(head)`. + +### Attesting + +An optimistic validator MUST NOT participate in attestation (i.e., sign across the +`DOMAIN_BEACON_ATTESTER`, `DOMAIN_SELECTION_PROOF` or +`DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are +met: + +#### Exception + +#### Exception 1. + +If a validator *does not* have an optimistic head (i.e., `not +is_optimistic(head_block)`), the node is *fully synced*. +The validator may produce an attestation. From 38fffd3e2f5d7f32d85e0b317fc5a0202534b2e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 17:09:09 +1100 Subject: [PATCH 002/112] Tidy, finish duties --- sync/optimistic.md | 105 ++++++++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b3220b96f..5837a1349 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -2,52 +2,71 @@ ## Introduction -In order to provide a syncing execution engine with a (partially-verified) view -of the head of the chain, it may be desirable for a consensus engine to import -beacon blocks without verifying the execution payloads. This partial sync is -called an *optimistic sync*. +In order to provide a syncing execution engine with a partial view of the head +of the chain, it may be desirable for a consensus engine to import beacon +blocks without verifying the execution payloads. This partial sync is called an +*optimistic sync*. ## Mechanisms To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution - engine returns SYNCING or VALID. An INVALID response MUST return `False`. + engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able -to ascertain, after import, which blocks returned SYNCING and which returned -VALID. This document will assume consensus engines store the following sets: +to ascertain, after import, which blocks returned `SYNCING` and which returned +`VALID`. This document will assume that consensus engines store the following +sets: - `valid_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be VALID. + `block.body.execution_payload` is known to be `VALID`. - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be SYNCING. + `block.body.execution_payload` is known to be `SYNCING`. -Notably, `optimistic_roots` only includes blocks which have execution enabled -whilst `valid_roots` contains blocks *with or without* execution enabled (i.e., -all blocks). +Notably, `optimistic_roots` only includes blocks which have execution enabled. +On the other hand, `valid_roots` contains blocks *with or without* execution +enabled (i.e., all blocks). A consensus engine MUST be able to retrospectively (i.e., after import) modify -the status of SYNCING blocks to be either VALID or INVALID based upon responses +the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: -- SYNCING -> VALID -- SYNCING -> INVALID +- `SYNCING` -> `VALID` +- `SYNCING` -> `INVALID` -When a block transitions from SYNCING -> VALID, all *ancestors* of the block MUST -also transition from SYNCING -> VALID. +When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the block MUST +also transition from `SYNCING` -> `VALID`. -When a block transitions from SYNCING -> INVALID, all *descendants* of the -block MUST also transition from SYNCING -> INVALID. +When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the +block MUST also transition from `SYNCING` -> `INVALID`. + +### Execution Engine Errors + +A consensus engine MUST NOT interpret an error or failure to respond to a +message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY +queue such a message for later processing. + +## Merge Transition + +To protect against attacks during the transition from empty `ExecutionPayload` +values to those which include the terminal PoW block, a consensus engine MUST +NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head +block references a block for which +`is_execution_enabled(head_state, head_block.body) == True`. + +TODO: this restriction is very onerous, however it is the best known remedy for +the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can +do better. ## Fork Choice -Consensus engines MUST support removing blocks that transition from SYNCING to -INVALID. Specifically, an INVALID block MUST NOT be included in the canonical -chain and the weights from INVALID blocks MUST NOT be applied to any VALID or -SYNCING ancestors. +Consensus engines MUST support removing from fork choice blocks that transition +from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +point MUST NOT be included in the canonical chain and the weights from those +`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. ## Validator assignments @@ -56,6 +75,8 @@ produce blocks, since an execution engine cannot produce a payload upon an unknown parent. It cannot faithfully attest to the head block of the chain, since it has not fully verified that block. +### Helpers + Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `justified_block: BeaconBlock` be the latest current justified ancestor ancestor of the `head_block`. @@ -75,8 +96,18 @@ def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: return None ``` -Let a node which returns `is_optimistic(head) == True` be an *optimistic -node*. Let a validator on an optimistic node be an *optimistic validator*. +```python +def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: + ancestor = latest_valid_ancestor(block) + + if ancestor is None or ancestor.slot + SLOTS_PER_EPOCH >= block.slot: + return None + + return ancestor +``` + +Let only a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let only a validator on an optimistic node be an *optimistic validator*. ### Block production @@ -86,9 +117,11 @@ A optimistic validator MUST NOT produce a block (i.e., sign across the #### Exception 1. If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MUST produce a block upon +is_optimistic(justified_block)`, the validator MAY produce a block upon `latest_valid_ancestor(head)`. +If the latest valid ancestor is `None`, the validator MUST NOT produce a block. + ### Attesting An optimistic validator MUST NOT participate in attestation (i.e., sign across the @@ -96,10 +129,20 @@ An optimistic validator MUST NOT participate in attestation (i.e., sign across t `DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are met: -#### Exception - #### Exception 1. -If a validator *does not* have an optimistic head (i.e., `not -is_optimistic(head_block)`), the node is *fully synced*. -The validator may produce an attestation. +If the justified block is fully verified (i.e., `not +is_optimistic(justified_block)`, the validator MAY sign across the following +domains: + +- `DOMAIN_BEACON_ATTESTER`: where `attestation.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))`. +- `DOMAIN_AGGREGATE_AND_PROOF` and `DOMAIN_SELECTION_PROOF`: where `aggregate.message.aggregate.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))` + +If the recent valid ancestor is `None`, the validator MUST NOT participate in +attestation. + +### Participating in Sync Committees + +An optimistic validator MUST NOT participate in sync committees (i.e., sign across the +`DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or +`DOMAIN_CONTRIBUTION_AND_PROOF` domains). From 891882307120eff346a8768aab3f3ce6d2497cd3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:02:45 +1100 Subject: [PATCH 003/112] Start adding p2p components --- sync/optimistic.md | 53 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5837a1349..0089abbd3 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -68,13 +68,6 @@ from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. -## Validator assignments - -An entirely optimistically synced node is *not* a full node. It is unable to -produce blocks, since an execution engine cannot produce a payload upon an -unknown parent. It cannot faithfully attest to the head block of the chain, -since it has not fully verified that block. - ### Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice @@ -109,6 +102,17 @@ def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: Let only a node which returns `is_optimistic(head) == True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. +When this specification only defines behaviour for an optimistic +node/validator, but *not* for the non-optimistic case, assume default +behaviours without regard for optimistic sync. + +## Validator assignments + +An entirely optimistically synced node is *not* a full node. It is unable to +produce blocks, since an execution engine cannot produce a payload upon an +unknown parent. It cannot faithfully attest to the head block of the chain, +since it has not fully verified that block. + ### Block production A optimistic validator MUST NOT produce a block (i.e., sign across the @@ -146,3 +150,38 @@ attestation. An optimistic validator MUST NOT participate in sync committees (i.e., sign across the `DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or `DOMAIN_CONTRIBUTION_AND_PROOF` domains). + +## P2P Networking + +### Topics and Messages on Gossipsub + +#### `beacon_block` + +An optimistic validator MUST subscribe to the `beacon_block` topic. Propagation +validation conditions are modified as such: + +Do not apply the existing condition: + +- [REJECT] The block's parent (defined by block.parent_root) passes validation. + +Instead, apply the new condition: + +- [REJECT] The block's parent (defined by block.parent_root) passes validation, + *and* `block.parent root not in optimistic_roots`. + +#### `beacon_aggregate_and_proof` + +An optimistic validator MUST NOT subscribe to the `beacon_aggregate_and_proof` +topic. + +#### `voluntary_exit` + +An optimistic validator MUST NOT subscribe to the `voluntary_exit` topic. + +#### `proposer_slashing` + +An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. + +#### `attester_slashing` + +An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. From 7f5b7d1535fa36708472f8a2811c3a760a65dc06 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:03:35 +1100 Subject: [PATCH 004/112] Remove attesting during opt sync --- sync/optimistic.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 0089abbd3..0b8036b21 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -130,20 +130,7 @@ If the latest valid ancestor is `None`, the validator MUST NOT produce a block. An optimistic validator MUST NOT participate in attestation (i.e., sign across the `DOMAIN_BEACON_ATTESTER`, `DOMAIN_SELECTION_PROOF` or -`DOMAIN_AGGREGATE_AND_PROOF` domains), unless one of the follow exceptions are -met: - -#### Exception 1. - -If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MAY sign across the following -domains: - -- `DOMAIN_BEACON_ATTESTER`: where `attestation.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))`. -- `DOMAIN_AGGREGATE_AND_PROOF` and `DOMAIN_SELECTION_PROOF`: where `aggregate.message.aggregate.data.beacon_block_root == hash_tree_root(recent_valid_ancestor(head))` - -If the recent valid ancestor is `None`, the validator MUST NOT participate in -attestation. +`DOMAIN_AGGREGATE_AND_PROOF` domains). ### Participating in Sync Committees From 1a89d167da53de19f1f02fd95c5b49f143157c50 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:15:53 +1100 Subject: [PATCH 005/112] Remove recent valid ancestor --- sync/optimistic.md | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 0b8036b21..a59c0ef87 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -76,27 +76,15 @@ be the latest current justified ancestor ancestor of the `head_block`. ```python def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots + hash_tree_root(block) in optimistic_roots ``` ```python -def latest_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: - while block.parent_root != Root(): - if not is_optimistic(block): - return block - block = get_block(block.parent_root) - - return None -``` - -```python -def recent_valid_ancestor(block: BeaconBlock) -> Optional[BeaconBlock]: - ancestor = latest_valid_ancestor(block) - - if ancestor is None or ancestor.slot + SLOTS_PER_EPOCH >= block.slot: - return None - - return ancestor +def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: + while block.parent_root != Root(): + if not is_optimistic(block) or block.parent_root == Root(): + return block + block = get_block(block.parent_root) ``` Let only a node which returns `is_optimistic(head) == True` be an *optimistic @@ -113,19 +101,18 @@ produce blocks, since an execution engine cannot produce a payload upon an unknown parent. It cannot faithfully attest to the head block of the chain, since it has not fully verified that block. -### Block production +### Block Production A optimistic validator MUST NOT produce a block (i.e., sign across the -`DOMAIN_BEACON_PROPOSER` domain), unless one of the follow exceptions are met: +`DOMAIN_BEACON_PROPOSER` domain), unless one of the following exceptions are +met: -#### Exception 1. +#### Block Production Exception 1. If the justified block is fully verified (i.e., `not is_optimistic(justified_block)`, the validator MAY produce a block upon `latest_valid_ancestor(head)`. -If the latest valid ancestor is `None`, the validator MUST NOT produce a block. - ### Attesting An optimistic validator MUST NOT participate in attestation (i.e., sign across the @@ -140,11 +127,11 @@ An optimistic validator MUST NOT participate in sync committees (i.e., sign acro ## P2P Networking -### Topics and Messages on Gossipsub +### The Gossip Domain (gossipsub) #### `beacon_block` -An optimistic validator MUST subscribe to the `beacon_block` topic. Propagation +An optimistic validator MAY subscribe to the `beacon_block` topic. Propagation validation conditions are modified as such: Do not apply the existing condition: From 5a5f980fca98c54f3b83b84130d9c9644b7c7b41 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 13 Dec 2021 19:25:33 +1100 Subject: [PATCH 006/112] Add RPC responses --- sync/optimistic.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index a59c0ef87..149fdd391 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,3 +159,45 @@ An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. #### `attester_slashing` An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. + +#### `beacon_attestation_{subnet_id}` + +An optimistic validator MUST NOT subscribe to the +`beacon_attestation_{subnet_id}` (attestation subnets) topics. + +#### `sync_committee_contribution_and_proof` + +An optimistic validator MUST NOT subscribe to the +`sync_committee_contribution_and_proof` topic. + +#### `sync_committee_{subnet_id}` + +An optimistic validator MUST NOT subscribe to the `sync_committee_{subnet_id}` +(sync committee subnets) topics. + +### The Req/Resp Domain + +#### BeaconBlocksByRange (v1, v2) + +Consensus engines MUST NOT include any block in a response where +`is_optimistic(block) == False`. + +#### BeaconBlocksByRoot (v1, v2) + +Consensus engines MUST NOT include any block in a response where +`is_optimistic(block) == False`. + +#### Status + +An optimistic node MUST use the `latest_valid_ancestor(head)` block to form +responses, rather than the head block. Specifically, an optimistic node must +form a `Status` message as so: + +The fields are, as seen by the client at the time of sending the message: + +- `fork_digest`: As previously defined. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the latest valid ancestor block + (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). +- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. +- `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). +- `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. From 2c62ed3b77bf08e71f8345d35657c99a114e8c88 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 14:53:04 +1100 Subject: [PATCH 007/112] Flip bool --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 149fdd391..be5626bf1 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -185,7 +185,7 @@ Consensus engines MUST NOT include any block in a response where #### BeaconBlocksByRoot (v1, v2) Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == False`. +`is_optimistic(block) == True`. #### Status From 497b5f8cbb1552f694f1ccf133e68beb87837b42 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:44:15 +1100 Subject: [PATCH 008/112] Add EE assumptions --- sync/optimistic.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index be5626bf1..63c060009 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -49,6 +49,15 @@ A consensus engine MUST NOT interpret an error or failure to respond to a message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY queue such a message for later processing. +### Assumptions about Execution Engine Behaviour + +This specification assumes execution engines will only return `SYNCING` when +there is insufficient information available to make a `VALID` or `INVALID` +determination on the given `ExecutionPayload` (e.g., the parent payload is +unknown). Specifically, `SYNCING` responses should be fork-specific; the search +for a block on one chain MUST NOT trigger a `SYNCING` response for another +chain. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From 0ad6025b488ff690e87c270a061cd3d736de76e6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:02 +1100 Subject: [PATCH 009/112] Remove valid roots set --- sync/optimistic.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 63c060009..7092a69ee 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -13,23 +13,16 @@ To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the - `pow_block` and `pow_parent` are unknown to the execution engine. +- The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned `VALID`. This document will assume that consensus engines store the following -sets: +set: -- `valid_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be `VALID`. - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where `block.body.execution_payload` is known to be `SYNCING`. -Notably, `optimistic_roots` only includes blocks which have execution enabled. -On the other hand, `valid_roots` contains blocks *with or without* execution -enabled (i.e., all blocks). - A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: From 3b67c334e4903092bdaa68bee62479f5eb000dd1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:15 +1100 Subject: [PATCH 010/112] Add note about removal from optimistic_roots --- sync/optimistic.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 7092a69ee..3abc5c1c6 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -36,8 +36,10 @@ also transition from `SYNCING` -> `VALID`. When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -### Execution Engine Errors +When a node transitions from the `SYNCING` state is is removed from the set of +`optimistic_roots`. +### Execution Engine Errors A consensus engine MUST NOT interpret an error or failure to respond to a message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY queue such a message for later processing. From fb520a83563570f2dddfb2cf0f9d0a1a00d424aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:51:26 +1100 Subject: [PATCH 011/112] Condense gossip topics --- sync/optimistic.md | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3abc5c1c6..70319503d 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -147,37 +147,20 @@ Instead, apply the new condition: - [REJECT] The block's parent (defined by block.parent_root) passes validation, *and* `block.parent root not in optimistic_roots`. -#### `beacon_aggregate_and_proof` +#### Other Topics -An optimistic validator MUST NOT subscribe to the `beacon_aggregate_and_proof` -topic. +An optimistic node MUST NOT subscribe to the following topics: -#### `voluntary_exit` +- `beacon_aggregate_and_proof` +- `voluntary_exit` +- `proposer_slashing` +- `attester_slashing` +- `beacon_attestation_{subnet_id}` +- `sync_committee_contribution_and_proof` +- `sync_committee_{subnet_id}` -An optimistic validator MUST NOT subscribe to the `voluntary_exit` topic. - -#### `proposer_slashing` - -An optimistic validator MUST NOT subscribe to the `proposer_slashing` topic. - -#### `attester_slashing` - -An optimistic validator MUST NOT subscribe to the `attester_slashing` topic. - -#### `beacon_attestation_{subnet_id}` - -An optimistic validator MUST NOT subscribe to the -`beacon_attestation_{subnet_id}` (attestation subnets) topics. - -#### `sync_committee_contribution_and_proof` - -An optimistic validator MUST NOT subscribe to the -`sync_committee_contribution_and_proof` topic. - -#### `sync_committee_{subnet_id}` - -An optimistic validator MUST NOT subscribe to the `sync_committee_{subnet_id}` -(sync committee subnets) topics. +Once the node ceases to be optimistic, it MAY re-subscribe to the aformentioned +topics. ### The Req/Resp Domain From d1851dce21f7fb89eb65f248eca477d44f41598e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:53:03 +1100 Subject: [PATCH 012/112] Tidy tab formatting --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 70319503d..edb54ab27 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -86,8 +86,8 @@ def is_optimistic(block: BeaconBlock) -> bool: ```python def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: while block.parent_root != Root(): - if not is_optimistic(block) or block.parent_root == Root(): - return block + if not is_optimistic(block) or block.parent_root == Root(): + return block block = get_block(block.parent_root) ``` From 4c4ffe71b4c20634ddeb238fe50e304fb1b239f1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 15:54:33 +1100 Subject: [PATCH 013/112] Fix latest_valid_ancestor --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index edb54ab27..ccae7d093 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -85,7 +85,7 @@ def is_optimistic(block: BeaconBlock) -> bool: ```python def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: - while block.parent_root != Root(): + while True: if not is_optimistic(block) or block.parent_root == Root(): return block block = get_block(block.parent_root) From 3f6e5b9c3ad5c453bcb7ef19f342801751f92c58 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:10:19 +1100 Subject: [PATCH 014/112] Add checkpoint sync --- sync/optimistic.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index ccae7d093..07293bde0 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -72,6 +72,11 @@ from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. +## Checkpoint Sync (Weak Subjectivity Sync) + +A consensus engine MAY assume that the `ExecutionPayload` of a block used for +checkpoint sync is `VALID`. + ### Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice From ffc2c405c41396528eac9b8fa6ed528773655ff6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:11:40 +1100 Subject: [PATCH 015/112] Add section about API --- sync/optimistic.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 07293bde0..c67855710 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -193,3 +193,20 @@ The fields are, as seen by the client at the time of sending the message: - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. - `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). - `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. + +## Ethereum Beacon APIs + +Consensus engines which provide an implementation of the [Ethereum Beacon +APIs](https://github.com/ethereum/beacon-APIs) must take care to avoid +presenting optimistic blocks as fully-verified blocks. + +When information about an optimistic block is requested, the consensus engine: + +- MUST NOT return a "success"-type response (e.g., 2xx). +- MAY return an "empty"-type response (e.g., 404). +- MAY return a "beacon node is currently syncing"-type response (e.g., 503). + +When `is_optimistic(head) == True`, the consensus engine: + +- MAY substitute the head block with `latest_valid_ancestor(block)`. +- MAY return a "beacon node is currently syncing"-type response (e.g., 503). From 9d7d4d086939ccc68ef8619a3cd66e2df4fbe891 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:21:23 +1100 Subject: [PATCH 016/112] Remove validate_merge_block check --- sync/optimistic.md | 1 - 1 file changed, 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c67855710..4b0c381a7 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -13,7 +13,6 @@ To perform an optimistic sync: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. In addition to these changes to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned From 4f1d8152d5b0d5a3eed26435bbe036c153ad5bdb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:21:30 +1100 Subject: [PATCH 017/112] Add note about full verification --- sync/optimistic.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 4b0c381a7..696a954ac 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -22,6 +22,11 @@ set: - `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where `block.body.execution_payload` is known to be `SYNCING`. +Notably, blocks included in `optimistic_roots` have passed all verifications +included in `process_block` (noting the modifications to the +`execute_payload`). I.e., the blocks are fully verified but awaiting execution +of the `ExecutionPayload`. + A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses from an execution engine. I.e., perform the following transitions: From eb32e141d29a18208b0e0f9b2447f17416934f27 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:22:52 +1100 Subject: [PATCH 018/112] Fix typo --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 696a954ac..37e7d4447 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -40,7 +40,7 @@ also transition from `SYNCING` -> `VALID`. When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a node transitions from the `SYNCING` state is is removed from the set of +When a node transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors From 084255465da0c96e7cb94e3c2d1ac3a4a9d02652 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:42:35 +1100 Subject: [PATCH 019/112] Add section about re-orgs --- sync/optimistic.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 37e7d4447..49a5b3c88 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -57,6 +57,16 @@ unknown). Specifically, `SYNCING` responses should be fork-specific; the search for a block on one chain MUST NOT trigger a `SYNCING` response for another chain. +### Re-Orgs + +The consensus engine MUST support any chain reorganisation which does *not* +affect the justified checkpoint. The consensus engine MAY support re-orgs +beyond the justified checkpoint. + +If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a +consensus engine MAY choose to alert the user and force the application to +exit. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From d9a0d16cc682e88e052c50be2bf712c54ee0b643 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 14 Dec 2021 16:46:31 +1100 Subject: [PATCH 020/112] Tidy --- sync/optimistic.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 49a5b3c88..fa04d5da7 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -75,9 +75,9 @@ NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the hea block references a block for which `is_execution_enabled(head_state, head_block.body) == True`. -TODO: this restriction is very onerous, however it is the best known remedy for -the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can -do better. +> TODO: this restriction is very onerous, however it is the best known remedy for +> the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can +> do better. ## Fork Choice From 538cc816814f1c95b88c7fed0078467101797578 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 15 Dec 2021 09:02:39 +1100 Subject: [PATCH 021/112] Flip bool --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index fa04d5da7..298d0c276 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -186,7 +186,7 @@ topics. #### BeaconBlocksByRange (v1, v2) Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == False`. +`is_optimistic(block) == True`. #### BeaconBlocksByRoot (v1, v2) From 5c1fcaf3e7bb535113753e78f1a6c738be9dd7b3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:34:43 +1100 Subject: [PATCH 022/112] Add qualification for errors --- sync/optimistic.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 298d0c276..e92647007 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -44,9 +44,11 @@ When a node transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors + A consensus engine MUST NOT interpret an error or failure to respond to a -message as a `SYNCING`, `VALID` or `INVALID` response. A consensus engine MAY -queue such a message for later processing. +message as a `SYNCING`, `VALID` or `INVALID` response. A message which receives +and error or no response MUST NOT be permitted to modify the fork choice +`Store`. A consensus engine MAY queue such a message for later processing. ### Assumptions about Execution Engine Behaviour From 2aa4edf933a4be8178db0a49bb6a750e68223486 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:44:21 +1100 Subject: [PATCH 023/112] Move helpers --- sync/optimistic.md | 75 ++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e92647007..d633f25ac 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -7,14 +7,57 @@ of the chain, it may be desirable for a consensus engine to import beacon blocks without verifying the execution payloads. This partial sync is called an *optimistic sync*. +## Constants + +|Name|Value|Unit +|---|---|---| +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `64` | slots + +## Helpers + +Let `head_block: BeaconBlock` be the result of calling of the fork choice +algorithm at the time of block production. Let `justified_block: BeaconBlock` +be the latest current justified ancestor ancestor of the `head_block`. + +```python +def is_optimistic(block: BeaconBlock) -> bool: + hash_tree_root(block) in optimistic_roots +``` + +```python +def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: + while True: + if not is_optimistic(block) or block.parent_root == Root(): + return block + block = get_block(block.parent_root) +``` + +```python +def is_execution_block(block: BeaconBlock) -> BeaconBlock: + block.execution_payload != ExecutionPayload() +``` + +Let only a node which returns `is_optimistic(head) == True` be an *optimistic +node*. Let only a validator on an optimistic node be an *optimistic validator*. + +When this specification only defines behaviour for an optimistic +node/validator, but *not* for the non-optimistic case, assume default +behaviours without regard for optimistic sync. + ## Mechanisms -To perform an optimistic sync: +## When to optimistically import blocks + +TODO + +## How to optimistically import blocks + +To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -In addition to these changes to validation, the consensus engine MUST be able +In addition to this change to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned `VALID`. This document will assume that consensus engines store the following set: @@ -75,7 +118,7 @@ To protect against attacks during the transition from empty `ExecutionPayload` values to those which include the terminal PoW block, a consensus engine MUST NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head block references a block for which -`is_execution_enabled(head_state, head_block.body) == True`. +`is_execution_block(head_block) == True`. > TODO: this restriction is very onerous, however it is the best known remedy for > the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can @@ -93,32 +136,6 @@ point MUST NOT be included in the canonical chain and the weights from those A consensus engine MAY assume that the `ExecutionPayload` of a block used for checkpoint sync is `VALID`. -### Helpers - -Let `head_block: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. Let `justified_block: BeaconBlock` -be the latest current justified ancestor ancestor of the `head_block`. - -```python -def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots -``` - -```python -def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: - while True: - if not is_optimistic(block) or block.parent_root == Root(): - return block - block = get_block(block.parent_root) -``` - -Let only a node which returns `is_optimistic(head) == True` be an *optimistic -node*. Let only a validator on an optimistic node be an *optimistic validator*. - -When this specification only defines behaviour for an optimistic -node/validator, but *not* for the non-optimistic case, assume default -behaviours without regard for optimistic sync. - ## Validator assignments An entirely optimistically synced node is *not* a full node. It is unable to From e49685eed5cc2c4f95b83798a6e90650f60d837b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Dec 2021 14:56:46 +1100 Subject: [PATCH 024/112] Add section for enabling opt sync --- sync/optimistic.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d633f25ac..ff249dcbf 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -37,6 +37,11 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: block.execution_payload != ExecutionPayload() ``` +```python +def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: + block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot +``` + Let only a node which returns `is_optimistic(head) == True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. @@ -48,7 +53,13 @@ behaviours without regard for optimistic sync. ## When to optimistically import blocks -TODO +A block MUST NOT be optimistically imported, unless either of the following +conditions are met: + +1. The justified checkpoint has execution enabled. I.e., + `is_execution_block(get_block(get_state(head_block).finalized_checkpoint.root))` +1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of + the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. ## How to optimistically import blocks From ffba24f03edf3a7a896fd96adb5d398696e17d9f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:15:51 +1100 Subject: [PATCH 025/112] Add failure recovery --- sync/optimistic.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ff249dcbf..65391332c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -11,7 +11,10 @@ blocks without verifying the execution payloads. This partial sync is called an |Name|Value|Unit |---|---|---| -|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `64` | slots +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots + +*Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See +[Failure Recovery](#failure-recovery). ## Helpers @@ -123,6 +126,38 @@ If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to exit. +## Failure Recovery + +During the merge transition it is possible for an attacker to craft a +`BeaconBlock` with an execution payload that references an +eternally-unavailable `body.execution_payload.parent_hash` value. In some rare +circumstances, it is possible that an attacker can build atop such a block to +trigger justification. If an optimistic node imports this malicious chain, that +node will have a "poisoned" fork choice store, such that the node is unable to +produce a child of the head (due to the invalid chain of payloads) and the node +is unable to fork around the head (due to the justification of the malicious +chain). + +The fork choice poisoning attack is temporary for an individual node, assuming +there exists an honest chain. An honest chain which justifies a higher epoch +than the malicious chain will take precedence and revive any poisoned store +once imported. + +The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network +will justify a honest chain within some number of slots. With this assumption, +it is therefore "safe" to optimistically import transition blocks during the +sync process. Since there is an assumption that an honest chain with a higher +justified checkpoint exists, any fork choice poisoning will be short-lived and +resolved before that node is required to produce a block. + +However, the assumption that the honest, canonical chain will always justify +within `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` slots is dubious. Therefore, +clients MUST provide the following command line flag to assist with manual +disaster recovery: + +- `--safe_slots_to_import_optimistically`: modifies the + `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. + ## Merge Transition To protect against attacks during the transition from empty `ExecutionPayload` From 26e934b1e1d3fd2e3f4d92cd273626eabf344d14 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:16:09 +1100 Subject: [PATCH 026/112] Remove merge transition section --- sync/optimistic.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 65391332c..4a89e5118 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -158,18 +158,6 @@ disaster recovery: - `--safe_slots_to_import_optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. -## Merge Transition - -To protect against attacks during the transition from empty `ExecutionPayload` -values to those which include the terminal PoW block, a consensus engine MUST -NOT perform an optimistic sync unless the `finalized_checkpoint.root` of the head -block references a block for which -`is_execution_block(head_block) == True`. - -> TODO: this restriction is very onerous, however it is the best known remedy for -> the attack described in https://hackmd.io/S5ZEVhsNTqqfJirTAkBPlg I hope we can -> do better. - ## Fork Choice Consensus engines MUST support removing from fork choice blocks that transition From da6cad8767c43b72ee3380b97b4ae3f3356f1255 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:32:45 +1100 Subject: [PATCH 027/112] Tidy --- sync/optimistic.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 4a89e5118..15f94f636 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -14,7 +14,7 @@ blocks without verifying the execution payloads. This partial sync is called an |`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots *Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See -[Failure Recovery](#failure-recovery). +[Fork Choice Poisoning](#fork-choice-poisoning).* ## Helpers @@ -64,6 +64,9 @@ conditions are met: 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. +*See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind +these conditions.* + ## How to optimistically import blocks To optimistically import a block: @@ -126,7 +129,14 @@ If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to exit. -## Failure Recovery +## Fork Choice + +Consensus engines MUST support removing from fork choice blocks that transition +from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +point MUST NOT be included in the canonical chain and the weights from those +`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. + +### Fork Choice Poisoning During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an @@ -158,13 +168,6 @@ disaster recovery: - `--safe_slots_to_import_optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. -## Fork Choice - -Consensus engines MUST support removing from fork choice blocks that transition -from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any -point MUST NOT be included in the canonical chain and the weights from those -`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. - ## Checkpoint Sync (Weak Subjectivity Sync) A consensus engine MAY assume that the `ExecutionPayload` of a block used for From 9901cb38f7f5b562d4a46dc1da78d2138a177c8b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 10:41:50 +1100 Subject: [PATCH 028/112] Move optimsitic_roots definition --- sync/optimistic.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 15f94f636..6b52b98a2 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -22,6 +22,10 @@ Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `justified_block: BeaconBlock` be the latest current justified ancestor ancestor of the `head_block`. +Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all +optimistically imported blocks which have yet to receive an `INVALID` or +`VALID` designation from an execution engine. + ```python def is_optimistic(block: BeaconBlock) -> bool: hash_tree_root(block) in optimistic_roots @@ -74,15 +78,11 @@ To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -In addition to this change to validation, the consensus engine MUST be able -to ascertain, after import, which blocks returned `SYNCING` and which returned -`VALID`. This document will assume that consensus engines store the following -set: +In addition to this change to validation, the consensus engine MUST be able to +ascertain, after import, which blocks returned `SYNCING` (`optimistic_roots`) +and which returned `VALID`. -- `optimistic_roots: Set[Root]`: `hash_tree_root(block)` where - `block.body.execution_payload` is known to be `SYNCING`. - -Notably, blocks included in `optimistic_roots` have passed all verifications +Notably, optimistically imported blocks MUST have passed all verifications included in `process_block` (noting the modifications to the `execute_payload`). I.e., the blocks are fully verified but awaiting execution of the `ExecutionPayload`. From 26431b762d92f7e4e813179d0cb11b2386f8e324 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 11:12:11 +1100 Subject: [PATCH 029/112] Tidy --- sync/optimistic.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6b52b98a2..553c3e487 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -64,7 +64,7 @@ A block MUST NOT be optimistically imported, unless either of the following conditions are met: 1. The justified checkpoint has execution enabled. I.e., - `is_execution_block(get_block(get_state(head_block).finalized_checkpoint.root))` + `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. @@ -79,13 +79,11 @@ To optimistically import a block: engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. In addition to this change to validation, the consensus engine MUST be able to -ascertain, after import, which blocks returned `SYNCING` (`optimistic_roots`) -and which returned `VALID`. +ascertain, after import, which blocks returned `SYNCING` and which returned +`VALID`. -Notably, optimistically imported blocks MUST have passed all verifications -included in `process_block` (noting the modifications to the -`execute_payload`). I.e., the blocks are fully verified but awaiting execution -of the `ExecutionPayload`. +Optimistically imported blocks MUST pass all verifications included in +`process_block` (withstanding the modifications to `execute_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses @@ -94,8 +92,9 @@ from an execution engine. I.e., perform the following transitions: - `SYNCING` -> `VALID` - `SYNCING` -> `INVALID` -When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the block MUST -also transition from `SYNCING` -> `VALID`. +When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the +block MUST also transition from `SYNCING` -> `VALID`. Such a block is no longer +considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. @@ -105,10 +104,11 @@ When a node transitions from the `SYNCING` state it is removed from the set of ### Execution Engine Errors -A consensus engine MUST NOT interpret an error or failure to respond to a -message as a `SYNCING`, `VALID` or `INVALID` response. A message which receives -and error or no response MUST NOT be permitted to modify the fork choice -`Store`. A consensus engine MAY queue such a message for later processing. +When an execution engine returns an error or fails to respond to a payload +validity request some block, a consensus engine: + +- MUST NOT optimistically import the block. +- MAY queue the block for later processing. ### Assumptions about Execution Engine Behaviour @@ -140,7 +140,7 @@ point MUST NOT be included in the canonical chain and the weights from those During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an -eternally-unavailable `body.execution_payload.parent_hash` value. In some rare +eternally-unavailable `body.execution_payload.parent_hash` value. In rare circumstances, it is possible that an attacker can build atop such a block to trigger justification. If an optimistic node imports this malicious chain, that node will have a "poisoned" fork choice store, such that the node is unable to @@ -149,14 +149,13 @@ is unable to fork around the head (due to the justification of the malicious chain). The fork choice poisoning attack is temporary for an individual node, assuming -there exists an honest chain. An honest chain which justifies a higher epoch -than the malicious chain will take precedence and revive any poisoned store -once imported. +there exists an honest chain which justifies a higher epoch than the malicious +chain. Such an honest chain will take precedence and revive any poisoned store. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, -it is therefore "safe" to optimistically import transition blocks during the -sync process. Since there is an assumption that an honest chain with a higher +it is acceptable to optimistically import transition blocks during the sync +process. Since there is an assumption that an honest chain with a higher justified checkpoint exists, any fork choice poisoning will be short-lived and resolved before that node is required to produce a block. @@ -165,13 +164,14 @@ within `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` slots is dubious. Therefore, clients MUST provide the following command line flag to assist with manual disaster recovery: -- `--safe_slots_to_import_optimistically`: modifies the +- `--safe-slots-to-import-optimistically`: modifies the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. ## Checkpoint Sync (Weak Subjectivity Sync) A consensus engine MAY assume that the `ExecutionPayload` of a block used for -checkpoint sync is `VALID`. +checkpoint sync is `VALID` without providing that payload to an execution +engine. ## Validator assignments From a797ae406bbdc29f01fc9533ef230e7f613e21e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 14:04:33 +1100 Subject: [PATCH 030/112] Add qualification about fc store --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 553c3e487..c1841d2dd 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -108,6 +108,7 @@ When an execution engine returns an error or fails to respond to a payload validity request some block, a consensus engine: - MUST NOT optimistically import the block. +- MUST NOT apply the block to the fork choice store. - MAY queue the block for later processing. ### Assumptions about Execution Engine Behaviour From b287f65dc9e9cc4e8988825f0fe6d7776589bab5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 16:21:11 +1100 Subject: [PATCH 031/112] Allow RPC blocks --- sync/optimistic.md | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c1841d2dd..b59c5b99b 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -240,30 +240,10 @@ topics. ### The Req/Resp Domain -#### BeaconBlocksByRange (v1, v2) - -Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == True`. - -#### BeaconBlocksByRoot (v1, v2) - -Consensus engines MUST NOT include any block in a response where -`is_optimistic(block) == True`. - -#### Status - -An optimistic node MUST use the `latest_valid_ancestor(head)` block to form -responses, rather than the head block. Specifically, an optimistic node must -form a `Status` message as so: - -The fields are, as seen by the client at the time of sending the message: - -- `fork_digest`: As previously defined. -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the latest valid ancestor block - (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). -- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the latest valid ancestor block. -- `head_root`: The `hash_tree_root` root of the current latest valid ancestor block (`BeaconBlock`). -- `head_slot`: The slot of the block corresponding to `latest_valid_ancestor(head)`. +Non-faulty, optimistic nodes may send blocks which result in an INVALID +response from an execution engine. To prevent network segregation between +optimistic and non-optimistic nodes, transmission of an INVALID payload SHOULD +NOT cause a node to be down-scored or disconnected. ## Ethereum Beacon APIs @@ -271,13 +251,32 @@ Consensus engines which provide an implementation of the [Ethereum Beacon APIs](https://github.com/ethereum/beacon-APIs) must take care to avoid presenting optimistic blocks as fully-verified blocks. +### Helpers + +Let the following response types be defined as any response with the +corresponding HTTP status code: + +- "Success" Response: Status Codes 200-299. +- "Not Found" Response: Status Code 404. +- "Syncing" Response: Status Code 503. + +### Requests for Optimistic Blocks + When information about an optimistic block is requested, the consensus engine: -- MUST NOT return a "success"-type response (e.g., 2xx). -- MAY return an "empty"-type response (e.g., 404). -- MAY return a "beacon node is currently syncing"-type response (e.g., 503). +- MUST NOT respond with success. +- MAY respond with not found. +- MAY respond with syncing. + +### Requests for the Head When `is_optimistic(head) == True`, the consensus engine: - MAY substitute the head block with `latest_valid_ancestor(block)`. -- MAY return a "beacon node is currently syncing"-type response (e.g., 503). +- MAY return syncing. + +### Requests to Validators Endpoints + +When `is_optimistic(head) == True`, the consensus engine: + +MUST respon From 91cad9b19b873b66a260ae47bb20aba903d1a2aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:19:32 +1100 Subject: [PATCH 032/112] Improve gossip wording --- sync/optimistic.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b59c5b99b..cf9bc2d54 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -220,8 +220,14 @@ Do not apply the existing condition: Instead, apply the new condition: -- [REJECT] The block's parent (defined by block.parent_root) passes validation, - *and* `block.parent root not in optimistic_roots`. +- [IGNORE] The block's parent (defined by block.parent_root) passes validation + except the block.body.execution_payload was deemed INVALID. +- [REJECT] The block's parent (defined by block.parent_root) passes all + validation, excluding verification of the block.body.execution_payload. + +The effect of these modifications is that invalid payloads may be propagated +across the network, but only when contained inside a block that is valid in *all +other aspects*. #### Other Topics From aa9a2967aa398e9c9bddac542644462907c395cf Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:27:02 +1100 Subject: [PATCH 033/112] Clarify API head condition --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index cf9bc2d54..e36378119 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -278,6 +278,7 @@ When information about an optimistic block is requested, the consensus engine: When `is_optimistic(head) == True`, the consensus engine: +- MUST NOT return `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. - MAY return syncing. From 7837dc74bff92576c868d3c4f97bb921102d5dd6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:46:42 +1100 Subject: [PATCH 034/112] Tidy, add validator endpoints --- sync/optimistic.md | 53 ++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e36378119..fcb6d981a 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -19,8 +19,10 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers Let `head_block: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. Let `justified_block: BeaconBlock` -be the latest current justified ancestor ancestor of the `head_block`. +algorithm at the time of block production. + +Let `justified_block: BeaconBlock` be the latest current justified ancestor +ancestor of the `head_block`. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or @@ -99,7 +101,7 @@ considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a node transitions from the `SYNCING` state it is removed from the set of +When a block transitions from the `SYNCING` state it is removed from the set of `optimistic_roots`. ### Execution Engine Errors @@ -116,9 +118,9 @@ validity request some block, a consensus engine: This specification assumes execution engines will only return `SYNCING` when there is insufficient information available to make a `VALID` or `INVALID` determination on the given `ExecutionPayload` (e.g., the parent payload is -unknown). Specifically, `SYNCING` responses should be fork-specific; the search -for a block on one chain MUST NOT trigger a `SYNCING` response for another -chain. +unknown). Specifically, `SYNCING` responses should be fork-specific, in that +the search for a block on one chain MUST NOT trigger a `SYNCING` response for +another chain. ### Re-Orgs @@ -132,7 +134,7 @@ exit. ## Fork Choice -Consensus engines MUST support removing from fork choice blocks that transition +Consensus engines MUST support removing blocks from fork choice that transition from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any point MUST NOT be included in the canonical chain and the weights from those `INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. @@ -141,17 +143,18 @@ point MUST NOT be included in the canonical chain and the weights from those During the merge transition it is possible for an attacker to craft a `BeaconBlock` with an execution payload that references an -eternally-unavailable `body.execution_payload.parent_hash` value. In rare -circumstances, it is possible that an attacker can build atop such a block to -trigger justification. If an optimistic node imports this malicious chain, that -node will have a "poisoned" fork choice store, such that the node is unable to -produce a child of the head (due to the invalid chain of payloads) and the node -is unable to fork around the head (due to the justification of the malicious +eternally-unavailable `body.execution_payload.parent_hash` (i.e., the parent +hash is random bytes). In rare circumstances, it is possible that an attacker +can build atop such a block to trigger justification. If an optimistic node +imports this malicious chain, that node will have a "poisoned" fork choice +store, such that the node is unable to produce a block that descends from the +head (due to the invalid chain of payloads) and the node is unable to produce a +block that forks around the head (due to the justification of the malicious chain). -The fork choice poisoning attack is temporary for an individual node, assuming -there exists an honest chain which justifies a higher epoch than the malicious -chain. Such an honest chain will take precedence and revive any poisoned store. +The fork choice poisoning attack is temporary for an individual node when that +an honest chain exists which justifies a higher epoch than the malicious chain. +Such an honest chain will take precedence and revive any poisoned store. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, @@ -170,9 +173,9 @@ disaster recovery: ## Checkpoint Sync (Weak Subjectivity Sync) -A consensus engine MAY assume that the `ExecutionPayload` of a block used for -checkpoint sync is `VALID` without providing that payload to an execution -engine. +A consensus engine MAY assume that the `ExecutionPayload` of a block used as an +anchor for checkpoint sync is `VALID` without necessarily providing that +payload to an execution engine. ## Validator assignments @@ -220,8 +223,7 @@ Do not apply the existing condition: Instead, apply the new condition: -- [IGNORE] The block's parent (defined by block.parent_root) passes validation - except the block.body.execution_payload was deemed INVALID. +- [IGNORE] The block's parent was imported optimistically. - [REJECT] The block's parent (defined by block.parent_root) passes all validation, excluding verification of the block.body.execution_payload. @@ -274,16 +276,17 @@ When information about an optimistic block is requested, the consensus engine: - MAY respond with not found. - MAY respond with syncing. -### Requests for the Head +### Requests for an Optimistic Head When `is_optimistic(head) == True`, the consensus engine: -- MUST NOT return `head`. +- MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. - MAY return syncing. ### Requests to Validators Endpoints -When `is_optimistic(head) == True`, the consensus engine: +When `is_optimistic(head) == True`, the consensus engine MUST return syncing to +all endpoints which match the following pattern: -MUST respon +- `eth/*/validator/*` From 451ae293ce7cc9ff4ee9230dfb932698d979e884 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 17:55:10 +1100 Subject: [PATCH 035/112] Specify no invalid parents --- sync/optimistic.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index fcb6d981a..515e340cc 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -79,6 +79,9 @@ To optimistically import a block: - The `execute_payload` function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The `validate_merge_block` function MUST NOT raise an assertion if both the +`pow_block` and `pow_parent` are unknown to the execution engine. +- The parent of the block MUST NOT have an INVALID execution payload. In addition to this change to validation, the consensus engine MUST be able to ascertain, after import, which blocks returned `SYNCING` and which returned From 9421bf3dfd7ea5a4658e419eca66c63f75a388fc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:32:47 +1100 Subject: [PATCH 036/112] Tidy --- sync/optimistic.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 515e340cc..665bea63d 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -43,7 +43,7 @@ def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: ```python def is_execution_block(block: BeaconBlock) -> BeaconBlock: - block.execution_payload != ExecutionPayload() + block.body.execution_payload != ExecutionPayload() ``` ```python @@ -110,7 +110,7 @@ When a block transitions from the `SYNCING` state it is removed from the set of ### Execution Engine Errors When an execution engine returns an error or fails to respond to a payload -validity request some block, a consensus engine: +validity request for some block, a consensus engine: - MUST NOT optimistically import the block. - MUST NOT apply the block to the fork choice store. @@ -155,9 +155,10 @@ head (due to the invalid chain of payloads) and the node is unable to produce a block that forks around the head (due to the justification of the malicious chain). -The fork choice poisoning attack is temporary for an individual node when that -an honest chain exists which justifies a higher epoch than the malicious chain. -Such an honest chain will take precedence and revive any poisoned store. +If honest chain exists which justifies a higher epoch than the malicious chain, +that chain will take precedence and revive any poisoned store. Therefore, the +poisoning attack is temporary if >= 2/3rds of the network is honest and +non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From ff50bfe6e8695a6fbe92b6964ba454b6e225e562 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:32:54 +1100 Subject: [PATCH 037/112] Remove block production exception --- sync/optimistic.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 665bea63d..ac805be0e 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -183,22 +183,15 @@ payload to an execution engine. ## Validator assignments -An entirely optimistically synced node is *not* a full node. It is unable to -produce blocks, since an execution engine cannot produce a payload upon an -unknown parent. It cannot faithfully attest to the head block of the chain, -since it has not fully verified that block. +An optimistic node is *not* a full node. It is unable to produce blocks, since +an execution engine cannot produce a payload upon an unknown parent. It cannot +faithfully attest to the head block of the chain, since it has not fully +verified that block. ### Block Production A optimistic validator MUST NOT produce a block (i.e., sign across the -`DOMAIN_BEACON_PROPOSER` domain), unless one of the following exceptions are -met: - -#### Block Production Exception 1. - -If the justified block is fully verified (i.e., `not -is_optimistic(justified_block)`, the validator MAY produce a block upon -`latest_valid_ancestor(head)`. +`DOMAIN_BEACON_PROPOSER` domain). ### Attesting From e696d1103c4efec9d6d8432953fcee0c06ae5cac Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 20 Dec 2021 18:36:03 +1100 Subject: [PATCH 038/112] Update gossip conditions --- sync/optimistic.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ac805be0e..e45d6bbb2 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -220,9 +220,10 @@ Do not apply the existing condition: Instead, apply the new condition: -- [IGNORE] The block's parent was imported optimistically. - [REJECT] The block's parent (defined by block.parent_root) passes all validation, excluding verification of the block.body.execution_payload. +- [IGNORE] The block's parent (defined by block.parent_root) passes all + validation, including verification of the block.body.execution_payload. The effect of these modifications is that invalid payloads may be propagated across the network, but only when contained inside a block that is valid in *all From 941531c8784a960c5bbb56a417658fcbc11150d7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 22 Dec 2021 09:24:14 +1100 Subject: [PATCH 039/112] Update sync/optimistic.md Co-authored-by: terence tsao --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index e45d6bbb2..abf352d07 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -68,7 +68,7 @@ conditions are met: 1. The justified checkpoint has execution enabled. I.e., `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` 1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of - the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot) == True`. + the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot, block) == True`. *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* From 12293c999aa3ead27934d01e0887c6d48cd6693f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 Dec 2021 16:49:43 +1100 Subject: [PATCH 040/112] Bump safe slots --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index abf352d07..b67cf85d4 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -11,7 +11,7 @@ blocks without verifying the execution payloads. This partial sync is called an |Name|Value|Unit |---|---|---| -|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `96` | slots +|`SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`| `128` | slots *Note: the `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` must be user-configurable. See [Fork Choice Poisoning](#fork-choice-poisoning).* From 50f526e418cb4115937271b49371d09502fbb820 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 22 Dec 2021 09:32:33 +1100 Subject: [PATCH 041/112] Fix typo --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b67cf85d4..33ae71e9a 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -241,8 +241,8 @@ An optimistic node MUST NOT subscribe to the following topics: - `sync_committee_contribution_and_proof` - `sync_committee_{subnet_id}` -Once the node ceases to be optimistic, it MAY re-subscribe to the aformentioned -topics. +Once the node ceases to be optimistic, it MAY re-subscribe to the +aforementioned topics. ### The Req/Resp Domain From 46bc2067402057590285f6253729ca0275d6a90f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 29 Dec 2021 12:40:16 +0100 Subject: [PATCH 042/112] Add `CONFIG_NAME` to configs Runtime configurations apply to a certain network and the name of that network is useful for humans such that they can talk about it. Some of the existing configs already include a `CONFIG_NAME` toggle - might as well add it here as well and avoid some confusion - this name above all becomes useful in the beacon API. By extension, the `CONFIG_NAME` config will appear in the beacon api as a result of being defined here. --- configs/mainnet.yaml | 7 +++++++ configs/minimal.yaml | 7 +++++++ setup.py | 2 +- tests/core/pyspec/eth2spec/config/config_util.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6c6af6282..f123f4f81 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -3,6 +3,13 @@ # Extends the mainnet preset PRESET_BASE: 'mainnet' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'mainnet' + # Transition # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 4e48c470b..61deb0be4 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -3,6 +3,13 @@ # Extends the minimal preset PRESET_BASE: 'minimal' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'minimal' + # Transition # --------------------------------------------------------------- # TBD, 2**256-2**10 is a placeholder diff --git a/setup.py b/setup.py index 81a629531..aa4157b29 100644 --- a/setup.py +++ b/setup.py @@ -752,7 +752,7 @@ def parse_config_vars(conf: Dict[str, str]) -> Dict[str, str]: """ out: Dict[str, str] = dict() for k, v in conf.items(): - if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE'): + if isinstance(v, str) and (v.startswith("0x") or k == 'PRESET_BASE' or k == 'CONFIG_NAME'): # Represent byte data with string, to avoid misinterpretation as big-endian int. # Everything is either byte data or an integer, with PRESET_BASE as one exception. out[k] = f"'{v}'" diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 0d06428ea..8fe2d8344 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -14,7 +14,7 @@ def parse_config_vars(conf: Dict[str, Any]) -> Dict[str, Any]: out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) - elif k != 'PRESET_BASE': + elif k != 'PRESET_BASE' and k != 'CONFIG_NAME': out[k] = int(v) else: out[k] = v From af4725d08a2036767d6e3f208fddcc793338fc52 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 3 Jan 2022 07:51:35 -0700 Subject: [PATCH 043/112] add some non-empty extra_data tests --- .../test_process_execution_payload.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b8c5f55c3..16c759839 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -214,3 +214,35 @@ def test_bad_timestamp_regular_payload(spec, state): execution_payload.timestamp = execution_payload.timestamp + 1 yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) + + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_extra_data_first_payload(spec, state): + # pre-state + state = build_state_with_incomplete_transition(spec, state) + next_slot(spec, state) + + # execution payload + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.extra_data = b'\x45' * 12 + + yield from run_execution_payload_processing(spec, state, execution_payload) + + assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data + + +@with_bellatrix_and_later +@spec_state_test +def test_non_empty_extra_data_regular_payload(spec, state): + # pre-state + state = build_state_with_complete_transition(spec, state) + next_slot(spec, state) + + # execution payload + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.extra_data = b'\x45' * 12 + + yield from run_execution_payload_processing(spec, state, execution_payload) + + assert state.latest_execution_payload_header.extra_data == execution_payload.extra_data From b220655d243eb36cac0d01a05324e0268b15d0e2 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:08:47 +0100 Subject: [PATCH 044/112] Add 3 new invalid test cases --- tests/generators/ssz_generic/ssz_container.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8f3a59ce3..9c1d9d0cb 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -105,7 +105,7 @@ def invalid_cases(): RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: if len(offsets) != 0: - for offset_index in offsets: + for index,offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), @@ -118,3 +118,20 @@ def invalid_cases(): offset_index=offset_index, change=lambda x: 0 )) + if index == 0: + yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x - 1 + )) + if mode == RandomizationMode.mode_max_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[0:2] + yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ + invalid_test_case(lambda:serialized) + if mode == RandomizationMode.mode_one_count: + serialized = serialize(container_case_fn(rng, mode, typ)) + serialized = serialized + serialized[0:1] + yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ + invalid_test_case(lambda:serialized) From 7ccd528cd15a7310e95aa07fdbc0c7971f163e89 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:26:00 +0100 Subject: [PATCH 045/112] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 9c1d9d0cb..8c8ed0b46 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -105,7 +105,7 @@ def invalid_cases(): RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: if len(offsets) != 0: - for index,offset_index in enumerate(offsets): + for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), @@ -120,7 +120,7 @@ def invalid_cases(): )) if index == 0: yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ - invalid_test_case(lambda: mod_offset( + invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 @@ -129,9 +129,9 @@ def invalid_cases(): serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ - invalid_test_case(lambda:serialized) + invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:1] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ - invalid_test_case(lambda:serialized) + invalid_test_case(lambda: serialized) From b5908cf2948146c15b327a329c9303b4a0cd0d3c Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:28:19 +0100 Subject: [PATCH 046/112] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8c8ed0b46..4cff3ed4a 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -124,7 +124,7 @@ def invalid_cases(): b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 - )) + )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] From db16f590cbd32f15b1531a29541503ab5fbb1a76 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 13:29:57 +0100 Subject: [PATCH 047/112] fixing lint --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 4cff3ed4a..59f53e66f 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -124,7 +124,7 @@ def invalid_cases(): b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, change=lambda x: x - 1 - )) + )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[0:2] From 6f64d46bc470fbe8ba32d44b3c0fa3b3d45079cd Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:06:42 +0100 Subject: [PATCH 048/112] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 59f53e66f..2498d392b 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -119,7 +119,7 @@ def invalid_cases(): change=lambda x: 0 )) if index == 0: - yield f'{name}_{mode.to_name()}_first offset_{offset_index}_minus_one', \ + yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), offset_index=offset_index, From 22719f2748f932e3d37a05026a088f26ae18c22c Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:06:50 +0100 Subject: [PATCH 049/112] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 2498d392b..818361fda 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -127,7 +127,7 @@ def invalid_cases(): )) if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) - serialized = serialized + serialized[0:2] + serialized = serialized + serialized[:2] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: From 7df0d4d70e60ff10d214578ec555b341d3b78f9d Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:23 +0100 Subject: [PATCH 050/112] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 818361fda..70e765c0c 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -128,7 +128,7 @@ def invalid_cases(): if mode == RandomizationMode.mode_max_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[:2] - yield f'{name}_{mode.to_name()}_last offset_{offset_index}_overflow', \ + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_overflow', \ invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) From 69a2fb4af09cb2c97d3a33c713917fa2512de4f7 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:31 +0100 Subject: [PATCH 051/112] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 70e765c0c..72b1729a4 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -132,6 +132,6 @@ def invalid_cases(): invalid_test_case(lambda: serialized) if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) - serialized = serialized + serialized[0:1] + serialized = serialized + serialized[:1] yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ invalid_test_case(lambda: serialized) From e56dddd6ecd3b507721195f06782916b4dd8a5ae Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 6 Jan 2022 19:07:48 +0100 Subject: [PATCH 052/112] Update tests/generators/ssz_generic/ssz_container.py Co-authored-by: Alex Stokes --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 72b1729a4..18fb3c09f 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -133,5 +133,5 @@ def invalid_cases(): if mode == RandomizationMode.mode_one_count: serialized = serialize(container_case_fn(rng, mode, typ)) serialized = serialized + serialized[:1] - yield f'{name}_{mode.to_name()}_last offset_{offset_index}_wrong_byte_length', \ + yield f'{name}_{mode.to_name()}_last_offset_{offset_index}_wrong_byte_length', \ invalid_test_case(lambda: serialized) From d74cb5c9ec416ac03222f11cf27f9a72285dc0c7 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 6 Jan 2022 15:34:59 -0800 Subject: [PATCH 053/112] remove unnecessary conditional --- tests/generators/ssz_generic/ssz_container.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index 8f3a59ce3..088a8ca54 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -104,17 +104,16 @@ def invalid_cases(): RandomizationMode.mode_nil_count, RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: - if len(offsets) != 0: - for offset_index in offsets: - yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: x + 1 - )) - yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ - invalid_test_case(lambda: mod_offset( - b=serialize(container_case_fn(rng, mode, typ)), - offset_index=offset_index, - change=lambda x: 0 - )) + for offset_index in offsets: + yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: x + 1 + )) + yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \ + invalid_test_case(lambda: mod_offset( + b=serialize(container_case_fn(rng, mode, typ)), + offset_index=offset_index, + change=lambda x: 0 + )) From fe605894e87e85ded911d62cb67ca1de88340d18 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jan 2022 17:07:13 +1100 Subject: [PATCH 054/112] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 738d43a05..86079a4c5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The current features are: * [Honest Validator guide changes](specs/altair/validator.md) * [P2P Networking](specs/altair/p2p-interface.md) -### Bellatrix (as known as The Merge) +### Bellatrix (also known as The Merge) The Bellatrix protocol upgrade is still actively in development. The exact specification has not been formally accepted as final and details are still subject to change. From 9e619f88b81041cb2b38887789b0fa830a5e71e6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 11 Jan 2022 16:42:20 +1100 Subject: [PATCH 055/112] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang Co-authored-by: Mikhail Kalinin Co-authored-by: terence tsao --- sync/optimistic.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 33ae71e9a..964413e25 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -29,29 +29,29 @@ optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. ```python -def is_optimistic(block: BeaconBlock) -> bool: - hash_tree_root(block) in optimistic_roots +def is_optimistic(block: BeaconBlock, optimistic_roots: Set[Root]) -> bool: + return hash_tree_root(block) in optimistic_roots ``` ```python -def latest_valid_ancestor(block: BeaconBlock) -> BeaconBlock: +def latest_verified_ancestor(block: BeaconBlock) -> BeaconBlock: while True: if not is_optimistic(block) or block.parent_root == Root(): - return block + return block block = get_block(block.parent_root) ``` ```python def is_execution_block(block: BeaconBlock) -> BeaconBlock: - block.body.execution_payload != ExecutionPayload() + return block.body.execution_payload != ExecutionPayload() ``` ```python def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: - block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot + return block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot ``` -Let only a node which returns `is_optimistic(head) == True` be an *optimistic +Let only a node which returns `is_optimistic(head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -60,7 +60,7 @@ behaviours without regard for optimistic sync. ## Mechanisms -## When to optimistically import blocks +### When to optimistically import blocks A block MUST NOT be optimistically imported, unless either of the following conditions are met: @@ -73,13 +73,13 @@ conditions are met: *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* -## How to optimistically import blocks +### How to optimistically import blocks To optimistically import a block: -- The `execute_payload` function MUST return `True` if the execution +- The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The `validate_merge_block` function MUST NOT raise an assertion if both the +- The [`validate_merge_block`](../specs/merge/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. @@ -190,7 +190,7 @@ verified that block. ### Block Production -A optimistic validator MUST NOT produce a block (i.e., sign across the +An optimistic validator MUST NOT produce a block (i.e., sign across the `DOMAIN_BEACON_PROPOSER` domain). ### Attesting @@ -216,14 +216,14 @@ validation conditions are modified as such: Do not apply the existing condition: -- [REJECT] The block's parent (defined by block.parent_root) passes validation. +- [REJECT] The block's parent (defined by `block.parent_root`) passes validation. Instead, apply the new condition: -- [REJECT] The block's parent (defined by block.parent_root) passes all - validation, excluding verification of the block.body.execution_payload. -- [IGNORE] The block's parent (defined by block.parent_root) passes all - validation, including verification of the block.body.execution_payload. +- [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation, excluding verification of the `block.body.execution_payload`. +- [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation, including verification of the `block.body.execution_payload`. The effect of these modifications is that invalid payloads may be propagated across the network, but only when contained inside a block that is valid in *all @@ -276,7 +276,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(head) == True`, the consensus engine: +When `is_optimistic(head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_valid_ancestor(block)`. @@ -284,7 +284,7 @@ When `is_optimistic(head) == True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(head) == True`, the consensus engine MUST return syncing to +When `is_optimistic(head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 6eba269e9ba772f40451716290e9e030fdd6a34f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 11 Jan 2022 19:02:40 +1100 Subject: [PATCH 056/112] Update sync/optimistic.md Co-authored-by: Raul Jordan --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 964413e25..3b96ae9a8 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -62,7 +62,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks -A block MUST NOT be optimistically imported, unless either of the following +A block MAY be optimistically imported when either of the following conditions are met: 1. The justified checkpoint has execution enabled. I.e., From 890a020f25a39a1da972cc6e56ab70e9c25c7c02 Mon Sep 17 00:00:00 2001 From: Fredrik Svantes Date: Tue, 11 Jan 2022 10:31:57 +0100 Subject: [PATCH 057/112] Updating URL for eth-clients Was renamed from https://github.com/eth2-clients to https://github.com/eth-clients --- configs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/README.md b/configs/README.md index f470b932d..6ef081e4c 100644 --- a/configs/README.md +++ b/configs/README.md @@ -9,7 +9,7 @@ Standard configs: - [`minimal.yaml`](./minimal.yaml): Minimal configuration, used in spec-testing along with the [`minimal`](../presets/minimal) preset. Not all network configurations are in scope for the specification, -see [`github.com/eth2-clients/eth2-networks`](https://github.com/eth2-clients/eth2-networks) for common networks, +see [`github.com/eth-clients/eth2-networks`](https://github.com/eth-clients/eth2-networks) for common networks, and additional testnet assets. ## Forking From 48e19b15ae92428734a671b1f7d8b902b8562e77 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 11:22:39 +0100 Subject: [PATCH 058/112] Rename `sync_committee_aggregate` > `sync_aggregate` This renames the `sync_committee_aggregate` field of `LightClientUpdate` to `sync_aggregate` for consistency with the terminology in the rest of the spec. --- specs/altair/sync-protocol.md | 10 +++++----- .../test/altair/unittests/test_sync_protocol.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4d..38102e00f 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -69,7 +69,7 @@ class LightClientUpdate(Container): finalized_header: BeaconBlockHeader finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature - sync_committee_aggregate: SyncAggregate + sync_aggregate: SyncAggregate # Fork version for the aggregate signature fork_version: Version ``` @@ -187,8 +187,8 @@ def validate_light_client_update(store: LightClientStore, index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=active_header.state_root, ) - - sync_aggregate = update.sync_committee_aggregate + + sync_aggregate = update.sync_aggregate # Verify sync committee has sufficient participants assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS @@ -225,12 +225,12 @@ def process_light_client_update(store: LightClientStore, genesis_validators_root: Root) -> None: validate_light_client_update(store, update, current_slot, genesis_validators_root) - sync_committee_bits = update.sync_committee_aggregate.sync_committee_bits + sync_committee_bits = update.sync_aggregate.sync_committee_bits # Update the best update in case we have to force-update to it if the timeout elapses if ( store.best_valid_update is None - or sum(sync_committee_bits) > sum(store.best_valid_update.sync_committee_aggregate.sync_committee_bits) + or sum(sync_committee_bits) > sum(store.best_valid_update.sync_aggregate.sync_committee_bits) ): store.best_valid_update = update diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 30444c4ce..120004654 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -58,7 +58,7 @@ def test_process_light_client_update_not_timeout(spec, state): block_header.slot, committee, ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -76,7 +76,7 @@ def test_process_light_client_update_not_timeout(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) @@ -123,7 +123,7 @@ def test_process_light_client_update_timeout(spec, state): committee, block_root=spec.Root(block_header.hash_tree_root()), ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -140,7 +140,7 @@ def test_process_light_client_update_timeout(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finality_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) @@ -201,7 +201,7 @@ def test_process_light_client_update_finality_updated(spec, state): committee, block_root=spec.Root(block_header.hash_tree_root()), ) - sync_committee_aggregate = spec.SyncAggregate( + sync_aggregate = spec.SyncAggregate( sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, ) @@ -212,7 +212,7 @@ def test_process_light_client_update_finality_updated(spec, state): next_sync_committee_branch=next_sync_committee_branch, finalized_header=finalized_block_header, finality_branch=finality_branch, - sync_committee_aggregate=sync_committee_aggregate, + sync_aggregate=sync_aggregate, fork_version=state.fork.current_version, ) From 212eb00fc15736d58d0b7a5ada44060e56a8093b Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 12:07:49 +0100 Subject: [PATCH 059/112] Document light client constants This adds documentation about the unit and actual value of light client specific constants, consistently with the rest of the spec. --- specs/altair/beacon-chain.md | 2 +- specs/altair/sync-protocol.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 3c7177a1b..2e897deb7 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -126,7 +126,7 @@ This patch updates a few configuration values to move penalty parameters closer | Name | Value | Unit | Duration | | - | - | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**9)` (= 512) | Validators | | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**9)` (= 512) | validators | | | `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours | ## Configuration diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4d..1baa830f8 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -42,17 +42,17 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | -| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` | -| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` (= 105) | +| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` (= 55) | ## Preset ### Misc -| Name | Value | Notes | -| - | - | - | -| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | | -| `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | ~27.3 hours | +| Name | Value | Unit | Duration | +| - | - | - | - | +| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | validators | +| `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | epochs | ~27.3 hours | ## Containers From 79c456b9f400d029af0883774c6e7feef5188f72 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 11 Jan 2022 11:48:22 +0100 Subject: [PATCH 060/112] Fix light client docs to match function signature In the light client docs a mentioning of a function trigger is lacking the `genesis_validators_root` argument. This patch adds that argument to the documentation to match the real function signature. It also slightly improves the grammar. --- specs/altair/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index c8c7c3d4d..0cabf51b8 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -127,7 +127,7 @@ def get_safety_threshold(store: LightClientStore) -> uint64: ## Light client state updates -A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. `process_slot_for_light_client_store` is processed every time the current slot increments. +A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot, genesis_validators_root)` where `current_slot` is the current slot based on a local clock. `process_slot_for_light_client_store` is triggered every time the current slot increments. #### `process_slot_for_light_client_store` From 7e2461e8056a709f0c20177f1af768cf6ca94376 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 11 Jan 2022 20:34:06 -0800 Subject: [PATCH 061/112] Remove client setting --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 86079a4c5..fb58edf8d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ The Bellatrix protocol upgrade is still actively in development. The exact speci * [Bellatrix fork](specs/bellatrix/fork.md) * [Fork Choice changes](specs/bellatrix/fork-choice.md) * [Validator additions](specs/bellatrix/validator.md) - * [Client settings](specs/bellatrix/client-settings.md) * [P2P Networking](specs/bellatrix/p2p-interface.md) ### Sharding From 6c13e2ecdbe2fbb2262e5570788dec1664d90803 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 15:54:11 +1100 Subject: [PATCH 062/112] Expand `should_optimistically_import_block` --- sync/optimistic.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3b96ae9a8..6137579cb 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -29,14 +29,23 @@ optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. ```python -def is_optimistic(block: BeaconBlock, optimistic_roots: Set[Root]) -> bool: - return hash_tree_root(block) in optimistic_roots +@dataclass +class Store(object): + optimistic_roots: Set[Root] + head_block_root: Root + blocks: Dict[Root, BeaconBlock] + block_states: Dict[Root, BeaconState] ``` ```python -def latest_verified_ancestor(block: BeaconBlock) -> BeaconBlock: +def is_optimistic(store: Store, block: BeaconBlock) -> bool: + return hash_tree_root(block) in store.optimistic_roots +``` + +```python +def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: while True: - if not is_optimistic(block) or block.parent_root == Root(): + if not is_optimistic(store, block) or block.parent_root == Root(): return block block = get_block(block.parent_root) ``` @@ -47,8 +56,11 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(current_slot: Slot, block: BeaconBlock) -> bool: - return block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot +def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: + justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(store.blocks[justified_root]) + block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot + return justified_is_verified or block_is_deep ``` Let only a node which returns `is_optimistic(head) is True` be an *optimistic @@ -105,7 +117,7 @@ When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state it is removed from the set of -`optimistic_roots`. +`store.optimistic_roots`. ### Execution Engine Errors From 2ce2aac0c89eac7ff6098cc2d0a4ad7f700f1d06 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 15:57:27 +1100 Subject: [PATCH 063/112] Address "yet to" paragraph --- sync/optimistic.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6137579cb..f3f70ee2f 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -28,6 +28,10 @@ Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. +Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all +optimistically imported blocks which have only received a `SYNCING` designation +from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). + ```python @dataclass class Store(object): From 736f3cec3f1d5c5a167c8d74b33a4d73b3f6b07b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:09:24 +1100 Subject: [PATCH 064/112] Remove `justified_block` --- sync/optimistic.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index f3f70ee2f..c511e9781 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -21,9 +21,6 @@ blocks without verifying the execution payloads. This partial sync is called an Let `head_block: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. -Let `justified_block: BeaconBlock` be the latest current justified ancestor -ancestor of the `head_block`. - Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have yet to receive an `INVALID` or `VALID` designation from an execution engine. From 55d92cee34842be718d4580619fdd16812a5f9d2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:10:59 +1100 Subject: [PATCH 065/112] Fix typo --- sync/optimistic.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index c511e9781..959fa9ffb 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -168,10 +168,10 @@ head (due to the invalid chain of payloads) and the node is unable to produce a block that forks around the head (due to the justification of the malicious chain). -If honest chain exists which justifies a higher epoch than the malicious chain, -that chain will take precedence and revive any poisoned store. Therefore, the -poisoning attack is temporary if >= 2/3rds of the network is honest and -non-faulty. +If an honest chain exists which justifies a higher epoch than the malicious +chain, that chain will take precedence and revive any poisoned store. +Therefore, the poisoning attack is temporary if >= 2/3rds of the network is +honest and non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From 1228e01883685ae90aeef8b05676879ed3d8bb13 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:18:53 +1100 Subject: [PATCH 066/112] Update p2p-networking --- specs/merge/p2p-interface.md | 17 +++++++++++++ sync/optimistic.md | 46 ------------------------------------ 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/specs/merge/p2p-interface.md b/specs/merge/p2p-interface.md index 0ab3d0825..9646eae4d 100644 --- a/specs/merge/p2p-interface.md +++ b/specs/merge/p2p-interface.md @@ -85,6 +85,10 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB Specifically, this type changes with the addition of `execution_payload` to the inner `BeaconBlockBody`. See the Merge [state transition document](./beacon-chain.md#beaconblockbody) for further details. +Blocks with execution enabled will be permitted to propagate regardless of the +validity of the execution payload. This prevents network segregation between +[optimistic](/sync/optimistic.md) and non-optimistic nodes. + In addition to the gossip validations for this topic from prior specifications, the following validations MUST pass before forwarding the `signed_beacon_block` on the network. Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. @@ -92,6 +96,13 @@ Alias `block = signed_beacon_block.message`, `execution_payload = block.body.exe then validate the following: - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. + - [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation, excluding verification of the `block.body.execution_payload`. + - [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation, including verification of the `block.body.execution_payload`. + +The following gossip validation from prior specifications MUST NOT be applied if the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`: + - [REJECT] The block's parent (defined by `block.parent_root`) passes validation. ### Transitioning the gossip @@ -100,6 +111,12 @@ details on how to handle transitioning gossip topics for the Merge. ## The Req/Resp domain +Non-faulty, [optimistic](/sync/optimistic.md) nodes may send blocks which +result in an INVALID response from an execution engine. To prevent network +segregation between optimistic and non-optimistic nodes, transmission of an +INVALID payload via the Req/Resp domain SHOULD NOT cause a node to be +down-scored or disconnected. + ### Messages #### BeaconBlocksByRange v2 diff --git a/sync/optimistic.md b/sync/optimistic.md index 959fa9ffb..84175d06c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -218,52 +218,6 @@ An optimistic validator MUST NOT participate in sync committees (i.e., sign acro `DOMAIN_SYNC_COMMITTEE`, `DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF` or `DOMAIN_CONTRIBUTION_AND_PROOF` domains). -## P2P Networking - -### The Gossip Domain (gossipsub) - -#### `beacon_block` - -An optimistic validator MAY subscribe to the `beacon_block` topic. Propagation -validation conditions are modified as such: - -Do not apply the existing condition: - -- [REJECT] The block's parent (defined by `block.parent_root`) passes validation. - -Instead, apply the new condition: - -- [REJECT] The block's parent (defined by `block.parent_root`) passes all - validation, excluding verification of the `block.body.execution_payload`. -- [IGNORE] The block's parent (defined by `block.parent_root`) passes all - validation, including verification of the `block.body.execution_payload`. - -The effect of these modifications is that invalid payloads may be propagated -across the network, but only when contained inside a block that is valid in *all -other aspects*. - -#### Other Topics - -An optimistic node MUST NOT subscribe to the following topics: - -- `beacon_aggregate_and_proof` -- `voluntary_exit` -- `proposer_slashing` -- `attester_slashing` -- `beacon_attestation_{subnet_id}` -- `sync_committee_contribution_and_proof` -- `sync_committee_{subnet_id}` - -Once the node ceases to be optimistic, it MAY re-subscribe to the -aforementioned topics. - -### The Req/Resp Domain - -Non-faulty, optimistic nodes may send blocks which result in an INVALID -response from an execution engine. To prevent network segregation between -optimistic and non-optimistic nodes, transmission of an INVALID payload SHOULD -NOT cause a node to be down-scored or disconnected. - ## Ethereum Beacon APIs Consensus engines which provide an implementation of the [Ethereum Beacon From 60eab25774e6282c2531b02921b4172780e8fb04 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:39:43 +1100 Subject: [PATCH 067/112] Add section about merge block --- sync/optimistic.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 84175d06c..d90880310 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,5 +1,7 @@ # Optimistic Sync +[`validate_merge_block`]: ../specs/merge/fork-choice.md#validate_merge_block + ## Introduction In order to provide a syncing execution engine with a partial view of the head @@ -92,7 +94,7 @@ To optimistically import a block: - The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The [`validate_merge_block`](../specs/merge/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the +- The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. @@ -120,6 +122,13 @@ block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state it is removed from the set of `store.optimistic_roots`. +When a "merge block" (i.e. a block which enables execution) is declared to be +`VALID` by an execution engine (either directly or indirectly), the full +[`validate_merge_block`][] MUST be run against the merge block. If the block +fails [`validate_merge_block`][], the merge block MUST be treated the same as +an `INVALID` block (i.e., it and all its descendants are invalidated and +removed from the block tree). + ### Execution Engine Errors When an execution engine returns an error or fails to respond to a payload From 0ae80d93826c1dcf0051ebd7afc49febe46d4bb5 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:41:24 +1100 Subject: [PATCH 068/112] Fix typo --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d90880310..05ab5076f 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -255,7 +255,7 @@ When information about an optimistic block is requested, the consensus engine: When `is_optimistic(head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. -- MAY substitute the head block with `latest_valid_ancestor(block)`. +- MAY substitute the head block with `latest_verified_ancestor(block)`. - MAY return syncing. ### Requests to Validators Endpoints From de1a6caa5dec08fa6358f2e2506e5c4e50548814 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:43:36 +1100 Subject: [PATCH 069/112] Add comment about INVALID block --- sync/optimistic.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 05ab5076f..72037dc3a 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -47,6 +47,7 @@ def is_optimistic(store: Store, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: + # It is assumed that the `block` parameter is never an INVALID block. while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block From ad7e92433a05b0fd6c2b74fddae3c5cf154b1533 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 16:49:08 +1100 Subject: [PATCH 070/112] Update links to bellatrix --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 72037dc3a..656a328b1 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,6 +1,6 @@ # Optimistic Sync -[`validate_merge_block`]: ../specs/merge/fork-choice.md#validate_merge_block +[`validate_merge_block`]: ../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction @@ -93,7 +93,7 @@ these conditions.* To optimistically import a block: -- The [`execute_payload`](../specs/merge/beacon-chain.md#execute_payload) function MUST return `True` if the execution +- The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. From 90fb7f68719b9bae2889c505e892114ee22540e8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jan 2022 19:04:43 +1100 Subject: [PATCH 071/112] Add rationale --- specs/bellatrix/p2p-interface.md | 23 +++++++++++ sync/optimistic.md | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 75c70a18a..5a260d982 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -198,3 +198,26 @@ valid block sizes in the range of gas limits expected in the medium term. As with both gossip and req/rsp maximum values, type-specific limits should always by simultaneously respected. + +### Why allow invalid payloads on the P2P network? + +The specification allows blocks with invalid payloads to propagate across +gossip and via RPC calls. The reasoning for this is as follows: + +1. Optimistic nodes must listen to block gossip to obtain a view of the head of + the chain. +2. Therefore, optimistic nodes must propagate gossip blocks. Otherwise, they'd + be censoring. +3. If optimistic nodes will propose blocks via gossip, then they must respond + to requests for the parent via RPC. +4. Therefore, optimistic nodes must send optimistic blocks via RPC. + +So, to prevent network segregation from optimistic nodes accidentally sending +invalid payloads, nodes should never downscore/disconnect nodes due to invalid +payloads. This does open the network to some DoS attacks from invalid execution +payloads, but the scope of actors is limited to validators who can put those +payloads in valid (and slashable) beacon blocks. Therefore, it is argued that +the DoS risk introduced in tolerable. + +More complicated schemes are possible that could restrict invalid payloads from +RPC. However, it's not clear that complexity is warranted. diff --git a/sync/optimistic.md b/sync/optimistic.md index 656a328b1..6793b972d 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -265,3 +265,73 @@ When `is_optimistic(head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` + +## Design Decision Rationale + +### Why `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`? + +Nodes can only import an optimistic block if their justified checkpoint is +verified or the block is older than `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY`. + +These restraints are applied in order to mitigate an attack where a block which +enables execution (a *transition block*) can reference a junk parent hash. This +makes it impossible for honest nodes to build atop that block. If an attacker +exploits a nuance in fork choice `filter_block_tree`, they can, in some rare +cases, produce a junk block that out-competes all locally produced blocks for +the head. This prevents a node from producing a chain of blocks, therefore +breaking liveness. + +Thankfully, if 2/3rds of validators are not poisoned, they can justify an +honest chain which will un-poison all other nodes. + +Notably, this attack only exists for optimistic nodes. Nodes which fully verify +the transition block will reject a block with a junk parent hash. + +Given all of this, we can say two things: + +1. **BNs which are following the head during the transition shouldn't + optimistically import the transition block.** If 1/3rd of validators + optimistically import the poison block, there will be no remaining nodes to + justify an honest chain. +2. **BNs which are syncing can optimistically import transition blocks.** In + this case a justified chain already exists blocks. The poison block would be + quickly reverted and would have no affect on liveness. + +Astute readers will notice that (2) contains a glaring assumption about network +liveness. This is necessary because a node cannot feasibly ascertain that the +transition block is justified without importing that block and risking +poisoning. Therefore, we use `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` to say +something along the lines of: *"if the transition block is sufficiently old +enough, then we can just assume that block is honest or there exists an honest +justified chain to out-compete it."* + +Note the use of "feasibly" in the previous paragraph. One can imagine +mechanisms to check that a block is justified before importing it. For example, +just keep processing blocks without adding them to fork choice. However, there +are still edge-cases here (e.g., when to halt and declare there was no +justification?) and how to mitigate implemenation complexity. At this point, +it's important to reflect on the attack and how likely it is to happen. It +requires some rather contrived circumstances and it seems very unlikley to +occur. Therefore, we need to consider if adding complexity to avoid an +unlikely attack increases or decreases our total risk. Presently, it appears +that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this +trade-off. + +### Transitioning from VALID -> INVALID or INVALID -> VALID + +These operations are purposefully omitted. It is outside of the scope of the +specification since it's only possible with a faulty EE. + +Such a scenario requires manual intervention. + +## What about Light Clients? + +An alternative to optimistic sync is to run a light client inside/alongside +beacon nodes that mitigates the need for optimistic sync by providing +tip-of-chain blocks to the execution engine. However, light clients comes with +their own set of complexities. Relying on light clients may also restrict nodes +from syncing from genesis, if they so desire. + +A notable thing about optimistic sync is that it's *optional*. Should an +implementation decide to go the light-client route, then they can just ignore +optimistic sync all together. From 6d73b0a4ac30a8fae1ba909d1ebf628234cb41ca Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:52:07 +1100 Subject: [PATCH 072/112] Add poisoning prevention --- sync/optimistic.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 6793b972d..217b8bfd1 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -179,9 +179,10 @@ block that forks around the head (due to the justification of the malicious chain). If an honest chain exists which justifies a higher epoch than the malicious -chain, that chain will take precedence and revive any poisoned store. -Therefore, the poisoning attack is temporary if >= 2/3rds of the network is -honest and non-faulty. +chain, that chain will take precedence and revive any poisoned store. Such a +chain, if imported before the malicious chain, will prevent the store from +being poisoned. Therefore, the poisoning attack is temporary if >= 2/3rds of +the network is honest and non-faulty. The `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` parameter assumes that the network will justify a honest chain within some number of slots. With this assumption, From 0c2e416a6e9cc9a7fb8c8e30b07ea0ece77832ff Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:56:32 +1100 Subject: [PATCH 073/112] Run doctoc --- specs/bellatrix/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 5a260d982..964a77464 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -29,6 +29,7 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas - [Why was the max gossip message size increased at Bellatrix?](#why-was-the-max-gossip-message-size-increased-at-bellatrix) - [Req/Resp](#reqresp) - [Why was the max chunk response size increased at Bellatrix?](#why-was-the-max-chunk-response-size-increased-at-bellatrix) + - [Why allow invalid payloads on the P2P network?](#why-allow-invalid-payloads-on-the-p2p-network) From 6d72038f12583de772e6972f4f9b834b2508353e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 06:56:38 +1100 Subject: [PATCH 074/112] Fix spelling mistakes --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 217b8bfd1..1b413ea71 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -310,9 +310,9 @@ Note the use of "feasibly" in the previous paragraph. One can imagine mechanisms to check that a block is justified before importing it. For example, just keep processing blocks without adding them to fork choice. However, there are still edge-cases here (e.g., when to halt and declare there was no -justification?) and how to mitigate implemenation complexity. At this point, +justification?) and how to mitigate implementation complexity. At this point, it's important to reflect on the attack and how likely it is to happen. It -requires some rather contrived circumstances and it seems very unlikley to +requires some rather contrived circumstances and it seems very unlikely to occur. Therefore, we need to consider if adding complexity to avoid an unlikely attack increases or decreases our total risk. Presently, it appears that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this From 18c32e0a5e10e4b2c7170139ea1a7ba1acdf08a3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:00:28 +1100 Subject: [PATCH 075/112] Fix indents --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 1b413ea71..826396a41 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -61,8 +61,8 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ```python def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: - justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(store.blocks[justified_root]) + justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot return justified_is_verified or block_is_deep ``` From 6af3d4cbfc5b2e075e0441172277d08736e45b72 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:13:10 +1100 Subject: [PATCH 076/112] Fix comment indents --- sync/optimistic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 826396a41..ab539a64d 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -47,7 +47,7 @@ def is_optimistic(store: Store, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: - # It is assumed that the `block` parameter is never an INVALID block. + # It is assumed that the `block` parameter is never an INVALID block. while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block From b7c332f0ee836da702c7a9e74afa8b8c285f8a20 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:17:43 +1100 Subject: [PATCH 077/112] Tidy --- sync/optimistic.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index ab539a64d..31f182ac4 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -20,7 +20,7 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers -Let `head_block: BeaconBlock` be the result of calling of the fork choice +Let `head: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all @@ -51,7 +51,7 @@ def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: while True: if not is_optimistic(store, block) or block.parent_root == Root(): return block - block = get_block(block.parent_root) + block = store.blocks[block.parent_root] ``` ```python @@ -67,7 +67,7 @@ def should_optimistically_import_block(store: Store, current_slot: Slot, block: return justified_is_verified or block_is_deep ``` -Let only a node which returns `is_optimistic(head) is True` be an *optimistic +Let only a node which returns `is_optimistic(store, head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -254,7 +254,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(head) is True`, the consensus engine: +When `is_optimistic(store, head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_verified_ancestor(block)`. @@ -262,7 +262,7 @@ When `is_optimistic(head) is True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(head) is True`, the consensus engine MUST return syncing to +When `is_optimistic(store, head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 8522f27b8ecfcffd067f873c957f824808ab53e9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:17:57 +1100 Subject: [PATCH 078/112] Modify "when" section --- sync/optimistic.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 31f182ac4..5d7f0d1f4 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -78,13 +78,14 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks -A block MAY be optimistically imported when either of the following -conditions are met: +A block MAY be optimistically imported when +`should_optimistically_import_block(store, current_slot, block)` returns +`True`. This ensures that blocks are only optimistically imported if either: -1. The justified checkpoint has execution enabled. I.e., - `is_execution_block(get_block(get_state(head_block).current_justified_checkpoint.root))` -1. The current slot (as per the system clock) is at least `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of - the slot of the block being imported. I.e., `should_optimistically_import_block(current_slot, block) == True`. +1. The justified checkpoint has execution enabled. +1. The current slot (as per the system clock) is at least + `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` ahead of the slot of the block being + imported. *See [Fork Choice Poisoning](#fork-choice-poisoning) for the motivations behind these conditions.* From 856eea4986f9b20d98132fc0d91898d68b31aac3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 13 Jan 2022 07:23:12 +1100 Subject: [PATCH 079/112] Describe all fields in store --- sync/optimistic.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5d7f0d1f4..cb45b7036 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -21,11 +21,12 @@ blocks without verifying the execution payloads. This partial sync is called an ## Helpers Let `head: BeaconBlock` be the result of calling of the fork choice -algorithm at the time of block production. +algorithm at the time of block production. Let `head_block_root: Root` be the +root of that block. -Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all -optimistically imported blocks which have yet to receive an `INVALID` or -`VALID` designation from an execution engine. +Let `blocks: Dict[Root, BeaconBlock]` and `block_states: Dict[Root, +BeaconState]` be the blocks (and accompanying states) that have been verified +either completely or optimistically. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have only received a `SYNCING` designation From 7255faf41b0e1aa3dc1bd12a823a98b459a45a70 Mon Sep 17 00:00:00 2001 From: Antonio Sanso Date: Thu, 13 Jan 2022 13:40:48 +0100 Subject: [PATCH 080/112] Update ssz_container.py --- tests/generators/ssz_generic/ssz_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py index b93399522..1b30d687a 100644 --- a/tests/generators/ssz_generic/ssz_container.py +++ b/tests/generators/ssz_generic/ssz_container.py @@ -104,7 +104,7 @@ def invalid_cases(): RandomizationMode.mode_nil_count, RandomizationMode.mode_one_count, RandomizationMode.mode_max_count]: - for offset_index in offsets: + for index, offset_index in enumerate(offsets): yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \ invalid_test_case(lambda: mod_offset( b=serialize(container_case_fn(rng, mode, typ)), From fd27d938986ed79017c62f9ddb842f147c56234e Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:20:30 -0800 Subject: [PATCH 081/112] add `pylint` to catch unused args --- Makefile | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ca0096fb9..ec3302e27 100644 --- a/Makefile +++ b/Makefile @@ -130,9 +130,11 @@ codespell: codespell . --skip ./.git -I .codespell-whitelist # TODO: add future protocol upgrade patch packages to linting. +# NOTE: we use `pylint` just for catching unused arguments in spec code lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ + && pylint --disable=all --enable unused-argument ./eth2spec/phase0 ./eth2spec/altair ./eth2spec/bellatrix \ && mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.bellatrix lint_generators: pyspec diff --git a/setup.py b/setup.py index aa4157b29..625b6f572 100644 --- a/setup.py +++ b/setup.py @@ -1008,7 +1008,7 @@ setup( python_requires=">=3.8, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==3.7.7", "mypy==0.812"], + "lint": ["flake8==3.7.7", "mypy==0.812", "pylint==2.12.2"], "generator": ["python-snappy==0.5.4"], }, install_requires=[ From 2b45496fe48fa75450ad29a05bdd48866f86528a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:31:27 -0800 Subject: [PATCH 082/112] clean up unused argument from `phase0` --- specs/altair/beacon-chain.md | 2 +- specs/phase0/beacon-chain.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 2e897deb7..a14ddb4f6 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -521,7 +521,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: signing_root = compute_signing_root(deposit_message, domain) # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, deposit.data.signature): - state.validators.append(get_validator_from_deposit(state, deposit)) + state.validators.append(get_validator_from_deposit(deposit)) state.balances.append(amount) state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7172307c6..618f39637 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1832,7 +1832,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits ```python -def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: +def get_validator_from_deposit(deposit: Deposit) -> Validator: amount = deposit.data.amount effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) @@ -1877,7 +1877,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add validator and balance entries - state.validators.append(get_validator_from_deposit(state, deposit)) + state.validators.append(get_validator_from_deposit(deposit)) state.balances.append(amount) else: # Increase balance by deposit amount From 10603b71c073c43b6aa85ff0983eca0bc315708c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 13 Jan 2022 11:31:45 -0800 Subject: [PATCH 083/112] indicate unused argument in utility function --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 625b6f572..f11951415 100644 --- a/setup.py +++ b/setup.py @@ -514,7 +514,7 @@ def get_pow_block(hash: Bytes32) -> Optional[PowBlock]: return PowBlock(block_hash=hash, parent_hash=Bytes32(), total_difficulty=uint256(0)) -def get_execution_state(execution_state_root: Bytes32) -> ExecutionState: +def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: pass From fd89b72c6b53e7191870ea870550a90a63dc371f Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 17 Jan 2022 12:56:51 +0100 Subject: [PATCH 084/112] `doctoc` on `custody_game` specs This updates the TOC in `custody_game/beacon-chain.md` using `make doctoc specs`. --- specs/custody_game/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/custody_game/beacon-chain.md b/specs/custody_game/beacon-chain.md index 6f9f61cf9..fd00dedda 100644 --- a/specs/custody_game/beacon-chain.md +++ b/specs/custody_game/beacon-chain.md @@ -11,7 +11,8 @@ - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) -- [Configuration](#configuration) + - [Domain types](#domain-types) +- [Preset](#preset) - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Size parameters](#size-parameters) From 8319d070848c55bda44ddbbfe72020cbcaf39b8d Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 17 Jan 2022 17:38:24 +0100 Subject: [PATCH 085/112] Consistently use `compute_sync_committee_period` There were a couple instances where a division was used on an epoch to derive the corresponding sync committee period instead of calling the `compute_sync_committee_period` function. These instances were changed to also use the function. --- specs/altair/sync-protocol.md | 8 ++++---- .../test_process_sync_committee_updates.py | 2 +- .../eth2spec/test/altair/unittests/test_sync_protocol.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index fe7882ff9..09d75129c 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -157,8 +157,8 @@ def validate_light_client_update(store: LightClientStore, assert current_slot >= active_header.slot > store.finalized_header.slot # Verify update does not skip a sync committee period - finalized_period = compute_epoch_at_slot(store.finalized_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - update_period = compute_epoch_at_slot(active_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + finalized_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) + update_period = compute_sync_committee_period(compute_epoch_at_slot(active_header.slot)) assert update_period in (finalized_period, finalized_period + 1) # Verify that the `finalized_header`, if present, actually is the finalized header saved in the @@ -208,8 +208,8 @@ def validate_light_client_update(store: LightClientStore, ```python def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None: active_header = get_active_header(update) - finalized_period = compute_epoch_at_slot(store.finalized_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD - update_period = compute_epoch_at_slot(active_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + finalized_period = compute_sync_committee_period(compute_epoch_at_slot(store.finalized_header.slot)) + update_period = compute_sync_committee_period(compute_epoch_at_slot(active_header.slot)) if update_period == finalized_period + 1: store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index 1667a12b2..4b0cb70d5 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -24,7 +24,7 @@ def run_sync_committees_progress_test(spec, state): first_sync_committee = state.current_sync_committee.copy() second_sync_committee = state.next_sync_committee.copy() - current_period = spec.get_current_epoch(state) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + current_period = spec.compute_sync_committee_period(spec.get_current_epoch(state)) next_period = current_period + 1 next_period_start_epoch = next_period * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD next_period_start_slot = next_period_start_epoch * spec.SLOTS_PER_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index 120004654..df0b33421 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -98,8 +98,8 @@ def test_process_light_client_update_timeout(spec, state): # Forward to next sync committee period next_slots(spec, state, spec.UPDATE_TIMEOUT) - snapshot_period = spec.compute_epoch_at_slot(store.optimistic_header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD - update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) assert snapshot_period + 1 == update_period block = build_empty_block_for_next_slot(spec, state) @@ -169,8 +169,8 @@ def test_process_light_client_update_finality_updated(spec, state): # Ensure that finality checkpoint has changed assert state.finalized_checkpoint.epoch == 3 # Ensure that it's same period - snapshot_period = spec.compute_epoch_at_slot(store.optimistic_header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD - update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + snapshot_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(store.optimistic_header.slot)) + update_period = spec.compute_sync_committee_period(spec.compute_epoch_at_slot(state.slot)) assert snapshot_period == update_period # Updated sync_committee and finality From 671c4f0e3a87cb52e90d57699df1f54942378f4d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 17 Jan 2022 11:05:01 -0700 Subject: [PATCH 086/112] receipt_root -> receipts_root --- specs/bellatrix/beacon-chain.md | 6 +++--- .../core/pyspec/eth2spec/test/helpers/execution_payload.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 738f03556..ea737e233 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -169,7 +169,7 @@ class ExecutionPayload(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper state_root: Bytes32 - receipt_root: Bytes32 # 'receipts root' in the yellow paper + receipts_root: Bytes32 # 'receipts root' in the yellow paper logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 # 'difficulty' in the yellow paper block_number: uint64 # 'number' in the yellow paper @@ -191,7 +191,7 @@ class ExecutionPayloadHeader(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress state_root: Bytes32 - receipt_root: Bytes32 + receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 block_number: uint64 @@ -359,7 +359,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe parent_hash=payload.parent_hash, fee_recipient=payload.fee_recipient, state_root=payload.state_root, - receipt_root=payload.receipt_root, + receipts_root=payload.receipts_root, logs_bloom=payload.logs_bloom, random=payload.random, block_number=payload.block_number, diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 9c9663584..49cec884a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -13,7 +13,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): parent_hash=latest.block_hash, fee_recipient=spec.ExecutionAddress(), state_root=latest.state_root, # no changes to the state - receipt_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. + receipts_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? block_number=latest.block_number + 1, random=randao_mix, @@ -36,7 +36,7 @@ def get_execution_payload_header(spec, execution_payload): parent_hash=execution_payload.parent_hash, fee_recipient=execution_payload.fee_recipient, state_root=execution_payload.state_root, - receipt_root=execution_payload.receipt_root, + receipts_root=execution_payload.receipts_root, logs_bloom=execution_payload.logs_bloom, random=execution_payload.random, block_number=execution_payload.block_number, diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 5ebe46c0e..0539a5d74 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -28,7 +28,7 @@ def get_sample_genesis_execution_payload_header(spec, parent_hash=b'\x30' * 32, fee_recipient=b'\x42' * 20, state_root=b'\x20' * 32, - receipt_root=b'\x20' * 32, + receipts_root=b'\x20' * 32, logs_bloom=b'\x35' * spec.BYTES_PER_LOGS_BLOOM, random=eth1_block_hash, block_number=0, From 15ef2f30d2dfd7a19f30d99df76c646ce07085ac Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 09:30:01 +1100 Subject: [PATCH 087/112] Apply suggestions from @djrtwo review Co-authored-by: Danny Ryan --- specs/bellatrix/p2p-interface.md | 6 +++--- sync/optimistic.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 964a77464..2f07068fe 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -202,7 +202,7 @@ always by simultaneously respected. ### Why allow invalid payloads on the P2P network? -The specification allows blocks with invalid payloads to propagate across +The specification allows blocks with invalid execution payloads to propagate across gossip and via RPC calls. The reasoning for this is as follows: 1. Optimistic nodes must listen to block gossip to obtain a view of the head of @@ -213,8 +213,8 @@ gossip and via RPC calls. The reasoning for this is as follows: to requests for the parent via RPC. 4. Therefore, optimistic nodes must send optimistic blocks via RPC. -So, to prevent network segregation from optimistic nodes accidentally sending -invalid payloads, nodes should never downscore/disconnect nodes due to invalid +So, to prevent network segregation from optimistic nodes inadvertently sending +invalid execution payloads, nodes should never downscore/disconnect nodes due to such invalid payloads. This does open the network to some DoS attacks from invalid execution payloads, but the scope of actors is limited to validators who can put those payloads in valid (and slashable) beacon blocks. Therefore, it is argued that diff --git a/sync/optimistic.md b/sync/optimistic.md index cb45b7036..88dc28444 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -116,16 +116,16 @@ from an execution engine. I.e., perform the following transitions: - `SYNCING` -> `INVALID` When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the -block MUST also transition from `SYNCING` -> `VALID`. Such a block is no longer +block MUST also transition from `SYNCING` -> `VALID`. Such a block and any previously `SYNCING` ancestors are no longer considered "optimistically imported". When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. -When a block transitions from the `SYNCING` state it is removed from the set of +When a block transitions from the `SYNCING` state, it is removed from the set of `store.optimistic_roots`. -When a "merge block" (i.e. a block which enables execution) is declared to be +When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full [`validate_merge_block`][] MUST be run against the merge block. If the block fails [`validate_merge_block`][], the merge block MUST be treated the same as From b1ec9bcfbc395549f3871a4d80960a5ddd27322d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:12:55 +1100 Subject: [PATCH 088/112] Update gossip conditions --- specs/bellatrix/p2p-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 2f07068fe..1d790bce7 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -95,12 +95,14 @@ the following validations MUST pass before forwarding the `signed_beacon_block` Alias `block = signed_beacon_block.message`, `execution_payload = block.body.execution_payload`. - If the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)` then validate the following: - - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot - -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. - - [REJECT] The block's parent (defined by `block.parent_root`) passes all - validation, excluding verification of the `block.body.execution_payload`. - - [IGNORE] The block's parent (defined by `block.parent_root`) passes all - validation, including verification of the `block.body.execution_payload`. + - _[REJECT]_ The block's execution payload timestamp is correct with respect to the slot + -- i.e. `execution_payload.timestamp == compute_timestamp_at_slot(state, block.slot)`. + - If `exection_payload` verification of block's parent by an execution node is *not* complete: + - [REJECT] The block's parent (defined by `block.parent_root`) passes all + validation (excluding execution node verification of the `block.body.execution_payload`). + - otherwise: + - [IGNORE] The block's parent (defined by `block.parent_root`) passes all + validation (including execution node verification of the `block.body.execution_payload`). The following gossip validation from prior specifications MUST NOT be applied if the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`: - [REJECT] The block's parent (defined by `block.parent_root`) passes validation. From 6225236a52df6017f139127deec806a069cbabad Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:20:52 +1100 Subject: [PATCH 089/112] Specify about EL/CL scoring rules --- specs/bellatrix/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 1d790bce7..4f3a70990 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -117,8 +117,10 @@ details on how to handle transitioning gossip topics for Bellatrix. Non-faulty, [optimistic](/sync/optimistic.md) nodes may send blocks which result in an INVALID response from an execution engine. To prevent network segregation between optimistic and non-optimistic nodes, transmission of an -INVALID payload via the Req/Resp domain SHOULD NOT cause a node to be -down-scored or disconnected. +INVALID execution payload via the Req/Resp domain SHOULD NOT cause a node to be +down-scored or disconnected. Transmission of a block which is invalid due to +any consensus layer rules (i.e., *not* execution layer rules) MAY result in +down-scoring or disconnection. ### Messages From 092f3e0b167d6ad65d7e83ea75956c276206700a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:22:07 +1100 Subject: [PATCH 090/112] Propose -> Propagate --- specs/bellatrix/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 4f3a70990..60a9be774 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -213,7 +213,7 @@ gossip and via RPC calls. The reasoning for this is as follows: the chain. 2. Therefore, optimistic nodes must propagate gossip blocks. Otherwise, they'd be censoring. -3. If optimistic nodes will propose blocks via gossip, then they must respond +3. If optimistic nodes will propagate blocks via gossip, then they must respond to requests for the parent via RPC. 4. Therefore, optimistic nodes must send optimistic blocks via RPC. From 52caba6eae98169549b78b3e5f5219f79c0db740 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:28:08 +1100 Subject: [PATCH 091/112] Add section about backwards compat --- sync/optimistic.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 88dc28444..d794269a2 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -9,6 +9,13 @@ of the chain, it may be desirable for a consensus engine to import beacon blocks without verifying the execution payloads. This partial sync is called an *optimistic sync*. +Optimistic sync is designed to be opt-in and backwards compatible (i.e., +non-optimistic nodes can tolerate optimistic nodes on the network and vice +versa). Optimistic sync is not a fundamental requirement for consensus nodes. +Rather, it's a stop-gap measure to allow execution nodes to sync via +established methods until future Ethereum roadmap items are implemented (e.g., +statelessness). + ## Constants |Name|Value|Unit From b50bee16f6ca45ed535c608f556983622cb2c567 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:30:28 +1100 Subject: [PATCH 092/112] Rename Store -> OptimisticStore --- sync/optimistic.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index d794269a2..3aaa2688b 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -41,7 +41,7 @@ from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). ```python @dataclass -class Store(object): +class OptimisticStore(object): optimistic_roots: Set[Root] head_block_root: Root blocks: Dict[Root, BeaconBlock] @@ -49,17 +49,17 @@ class Store(object): ``` ```python -def is_optimistic(store: Store, block: BeaconBlock) -> bool: - return hash_tree_root(block) in store.optimistic_roots +def is_optimistic(opt_store: OptimisticStore, block: BeaconBlock) -> bool: + return hash_tree_root(block) in opt_store.optimistic_roots ``` ```python -def latest_verified_ancestor(store: Store, block: BeaconBlock) -> BeaconBlock: +def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> BeaconBlock: # It is assumed that the `block` parameter is never an INVALID block. while True: - if not is_optimistic(store, block) or block.parent_root == Root(): + if not is_optimistic(opt_store, block) or block.parent_root == Root(): return block - block = store.blocks[block.parent_root] + block = opt_store.blocks[block.parent_root] ``` ```python @@ -68,14 +68,14 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(store: Store, current_slot: Slot, block: BeaconBlock) -> bool: - justified_root = store.block_states[store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(store.blocks[justified_root]) +def should_optimistically_import_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: + justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root + justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot return justified_is_verified or block_is_deep ``` -Let only a node which returns `is_optimistic(store, head) is True` be an *optimistic +Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic node*. Let only a validator on an optimistic node be an *optimistic validator*. When this specification only defines behaviour for an optimistic @@ -87,7 +87,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks A block MAY be optimistically imported when -`should_optimistically_import_block(store, current_slot, block)` returns +`should_optimistically_import_block(opt_store, current_slot, block)` returns `True`. This ensures that blocks are only optimistically imported if either: 1. The justified checkpoint has execution enabled. @@ -130,7 +130,7 @@ When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the block MUST also transition from `SYNCING` -> `INVALID`. When a block transitions from the `SYNCING` state, it is removed from the set of -`store.optimistic_roots`. +`opt_store.optimistic_roots`. When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full @@ -263,7 +263,7 @@ When information about an optimistic block is requested, the consensus engine: ### Requests for an Optimistic Head -When `is_optimistic(store, head) is True`, the consensus engine: +When `is_optimistic(opt_store, head) is True`, the consensus engine: - MUST NOT return an optimistic `head`. - MAY substitute the head block with `latest_verified_ancestor(block)`. @@ -271,7 +271,7 @@ When `is_optimistic(store, head) is True`, the consensus engine: ### Requests to Validators Endpoints -When `is_optimistic(store, head) is True`, the consensus engine MUST return syncing to +When `is_optimistic(opt_store, head) is True`, the consensus engine MUST return syncing to all endpoints which match the following pattern: - `eth/*/validator/*` From 4ccd38b54e3a2a19b159ff4c1383bd2f820156c9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:32:47 +1100 Subject: [PATCH 093/112] Tidy tracking note --- sync/optimistic.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 3aaa2688b..dd9d428d4 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -108,9 +108,8 @@ To optimistically import a block: `pow_block` and `pow_parent` are unknown to the execution engine. - The parent of the block MUST NOT have an INVALID execution payload. -In addition to this change to validation, the consensus engine MUST be able to -ascertain, after import, which blocks returned `SYNCING` and which returned -`VALID`. +In addition to this change in validation, the consensus engine MUST track which +blocks returned `SYNCING` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in `process_block` (withstanding the modifications to `execute_payload`). From f4a125c89aaf1523322ff175ba32c169e0dbf493 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 11:36:36 +1100 Subject: [PATCH 094/112] Remove reorg section --- sync/optimistic.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index dd9d428d4..67f2e3ee5 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,8 +159,7 @@ another chain. ### Re-Orgs The consensus engine MUST support any chain reorganisation which does *not* -affect the justified checkpoint. The consensus engine MAY support re-orgs -beyond the justified checkpoint. +affect the justified checkpoint. If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a consensus engine MAY choose to alert the user and force the application to From 0ec61bd195a308a81e2bf221346394284801c42a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:21:34 +1100 Subject: [PATCH 095/112] Clarify liveness --- sync/optimistic.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 67f2e3ee5..66ec8c312 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -293,7 +293,9 @@ Thankfully, if 2/3rds of validators are not poisoned, they can justify an honest chain which will un-poison all other nodes. Notably, this attack only exists for optimistic nodes. Nodes which fully verify -the transition block will reject a block with a junk parent hash. +the transition block will reject a block with a junk parent hash. Therefore, +liveness is unaffected if a vast majority of nodes have fully synced execution +and consensus clients before and during the transition. Given all of this, we can say two things: From bfe4172584b35a881020b01e7f781ad0f91e8011 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:26:30 +1100 Subject: [PATCH 096/112] Define `current_slot` --- sync/optimistic.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 66ec8c312..a66408b93 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -39,6 +39,9 @@ Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all optimistically imported blocks which have only received a `SYNCING` designation from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). +Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where +`time` is the UNIX time according to the local system clock. + ```python @dataclass class OptimisticStore(object): From 24947be7c3446c142aa767fdbbb57b412a008b86 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:29:20 +1100 Subject: [PATCH 097/112] Rename should_optimistically_import... --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index a66408b93..257574a62 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -71,7 +71,7 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ``` ```python -def should_optimistically_import_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: +def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot @@ -90,7 +90,7 @@ behaviours without regard for optimistic sync. ### When to optimistically import blocks A block MAY be optimistically imported when -`should_optimistically_import_block(opt_store, current_slot, block)` returns +`is_optimistic_candidate_block(opt_store, current_slot, block)` returns `True`. This ensures that blocks are only optimistically imported if either: 1. The justified checkpoint has execution enabled. From be4319a048926aeecb53d707488612b971bccde7 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 12:31:36 +1100 Subject: [PATCH 098/112] Rename `justified_is_verified` --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 257574a62..44d1fb616 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -73,9 +73,9 @@ def is_execution_block(block: BeaconBlock) -> BeaconBlock: ```python def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root - justifed_is_verified = is_execution_block(opt_store.blocks[justified_root]) + justifed_is_execution_block = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot - return justified_is_verified or block_is_deep + return justifed_is_execution_block or block_is_deep ``` Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic From 50b236e5ebbae2eecf60b867442a3df83a88d056 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 18 Jan 2022 14:55:10 +1100 Subject: [PATCH 099/112] Address TERMINAL_BLOCK_HASH --- sync/optimistic.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sync/optimistic.md b/sync/optimistic.md index 44d1fb616..01765112c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -109,6 +109,7 @@ To optimistically import a block: engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. - The [`validate_merge_block`][] function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. + - All other assertions in [`validate_merge_block`][] (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. - The parent of the block MUST NOT have an INVALID execution payload. In addition to this change in validation, the consensus engine MUST track which @@ -348,3 +349,13 @@ from syncing from genesis, if they so desire. A notable thing about optimistic sync is that it's *optional*. Should an implementation decide to go the light-client route, then they can just ignore optimistic sync all together. + +## What if `TERMINAL_BLOCK_HASH` is used? + +If the terminal block hash override is used (i.e., `TERMINAL_BLOCK_HASH != +Hash32()`), the [`validate_merge_block`][] function will deterministically +return `True` or `False`. Whilst it's not *technically* required +retrospectively call [`validate_merge_block`][] on a transition block that +matches `TERMINAL_BLOCK_HASH` after an optimistic sync, doing so will have no +effect. For simplicity, the optimistic sync specification does not define +edge-case behaviour for when `TERMINAL_BLOCK_HASH` is used. From f77fc055f8a34e645d9ad183245952bc2c37540d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 18 Jan 2022 07:19:00 -0700 Subject: [PATCH 100/112] Update specs/bellatrix/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/bellatrix/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index ea737e233..c7bd7af69 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -169,7 +169,7 @@ class ExecutionPayload(Container): parent_hash: Hash32 fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper state_root: Bytes32 - receipts_root: Bytes32 # 'receipts root' in the yellow paper + receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] random: Bytes32 # 'difficulty' in the yellow paper block_number: uint64 # 'number' in the yellow paper From a5b3c91f250a8d6a057c3f4021c41199d41e4c21 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jan 2022 09:25:02 -0700 Subject: [PATCH 101/112] build opimistic sync file and fix a minor lint/typing issue --- setup.py | 1 + sync/optimistic.md | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index aa4157b29..888c16142 100644 --- a/setup.py +++ b/setup.py @@ -868,6 +868,7 @@ class PySpecCommand(Command): specs/bellatrix/fork.md specs/bellatrix/fork-choice.md specs/bellatrix/validator.md + sync/optimistic.md """ if len(self.md_doc_paths) == 0: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/sync/optimistic.md b/sync/optimistic.md index 01765112c..80e9de3e9 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,7 +1,8 @@ # Optimistic Sync -[`validate_merge_block`]: ../specs/bellatrix/fork-choice.md#validate_merge_block + +../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction In order to provide a syncing execution engine with a partial view of the head @@ -66,7 +67,7 @@ def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> ``` ```python -def is_execution_block(block: BeaconBlock) -> BeaconBlock: +def is_execution_block(block: BeaconBlock) -> bool: return block.body.execution_payload != ExecutionPayload() ``` @@ -106,10 +107,12 @@ these conditions.* To optimistically import a block: - The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution - engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. -- The [`validate_merge_block`][] function MUST NOT raise an assertion if both the + engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) + function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - - All other assertions in [`validate_merge_block`][] (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. + - All other assertions in [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) + (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. - The parent of the block MUST NOT have an INVALID execution payload. In addition to this change in validation, the consensus engine MUST track which @@ -137,8 +140,10 @@ When a block transitions from the `SYNCING` state, it is removed from the set of When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be `VALID` by an execution engine (either directly or indirectly), the full -[`validate_merge_block`][] MUST be run against the merge block. If the block -fails [`validate_merge_block`][], the merge block MUST be treated the same as +[`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +MUST be run against the merge block. If the block +fails [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block), +the merge block MUST be treated the same as an `INVALID` block (i.e., it and all its descendants are invalidated and removed from the block tree). @@ -353,9 +358,11 @@ optimistic sync all together. ## What if `TERMINAL_BLOCK_HASH` is used? If the terminal block hash override is used (i.e., `TERMINAL_BLOCK_HASH != -Hash32()`), the [`validate_merge_block`][] function will deterministically +Hash32()`), the [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +function will deterministically return `True` or `False`. Whilst it's not *technically* required -retrospectively call [`validate_merge_block`][] on a transition block that +retrospectively call [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) +on a transition block that matches `TERMINAL_BLOCK_HASH` after an optimistic sync, doing so will have no effect. For simplicity, the optimistic sync specification does not define edge-case behaviour for when `TERMINAL_BLOCK_HASH` is used. From 182e4496c3f4a05e20bd422d0f02df7bcbefea4a Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Fri, 21 Jan 2022 19:01:56 +0000 Subject: [PATCH 102/112] use INTERVALS_PER_SLOT in sync committee specs --- specs/altair/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/validator.md b/specs/altair/validator.md index 461c9a70d..c59aa29f7 100644 --- a/specs/altair/validator.md +++ b/specs/altair/validator.md @@ -268,7 +268,7 @@ This process occurs each slot. If a validator is in the current sync committee (i.e. `is_assigned_to_sync_committee()` above returns `True`), then for every `slot` in the current sync committee period, the validator should prepare a `SyncCommitteeMessage` for the previous slot (`slot - 1`) according to the logic in `get_sync_committee_message` as soon as they have determined the head block of `slot - 1`. This means that when assigned to `slot` a `SyncCommitteeMessage` is prepared and broadcast in `slot-1 ` instead of `slot`. This logic is triggered upon the same conditions as when producing an attestation. -Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of the slot) -- whichever comes first. +Meaning, a sync committee member should produce and broadcast a `SyncCommitteeMessage` either when (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / INTERVALS_PER_SLOT` seconds after the start of the slot) -- whichever comes first. `get_sync_committee_message(state, block_root, validator_index, privkey)` assumes the parameter `state` is the head state corresponding to processing the block up to the current slot as determined by the fork choice (including any empty slots up to the current slot processed with `process_slots` on top of the latest block), `block_root` is the root of the head block, `validator_index` is the index of the validator in the registry `state.validators` controlled by `privkey`, and `privkey` is the BLS private key for the validator. @@ -385,7 +385,7 @@ The collection of input signatures should include one signature per validator wh ##### Broadcast sync committee contribution -If the validator is selected to aggregate (`is_sync_committee_aggregator()`), then they broadcast their best aggregate as a `SignedContributionAndProof` to the global aggregate channel (`sync_committee_contribution_and_proof` topic) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. +If the validator is selected to aggregate (`is_sync_committee_aggregator()`), then they broadcast their best aggregate as a `SignedContributionAndProof` to the global aggregate channel (`sync_committee_contribution_and_proof` topic) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / INTERVALS_PER_SLOT` seconds after the start of `slot`. Selection proofs are provided in `ContributionAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. From 0e9460b8ddb86d65f92a1a8b54ccc68906bbd77e Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Tue, 25 Jan 2022 10:09:10 +0100 Subject: [PATCH 103/112] Ensure light client `optimistic_header` to be at head When a light client updates its `finalized_header` using a forced update because of the timeout, and the new header was not signed by enough sync committee participants to pass `get_safety_threshold(store)`, it may occur that `store.finalized_header.slot > store.optimistic_header.slot`. This patch ensures that the `optimistic_header` is updated to the latest `finalized_header` if that happens, so that it always indicates the latest known and accepted head. --- specs/altair/sync-protocol.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/altair/sync-protocol.md b/specs/altair/sync-protocol.md index fe7882ff9..a4a57a91b 100644 --- a/specs/altair/sync-protocol.md +++ b/specs/altair/sync-protocol.md @@ -214,6 +214,8 @@ def apply_light_client_update(store: LightClientStore, update: LightClientUpdate store.current_sync_committee = store.next_sync_committee store.next_sync_committee = update.next_sync_committee store.finalized_header = active_header + if store.finalized_header.slot > store.optimistic_header.slot: + store.optimistic_header = store.finalized_header ``` #### `process_light_client_update` From eb4497fae71ea350530a8370601e54e603a150b0 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 27 Jan 2022 15:24:53 +0600 Subject: [PATCH 104/112] Bellatrix: Rename execute_payload to notify_new_payload --- setup.py | 2 +- specs/bellatrix/beacon-chain.md | 12 ++++++------ sync/optimistic.md | 6 +++--- .../test_process_execution_payload.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index ac60eb3b7..f826968f2 100644 --- a/setup.py +++ b/setup.py @@ -524,7 +524,7 @@ def get_pow_chain_head() -> PowBlock: class NoopExecutionEngine(ExecutionEngine): - def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index c7bd7af69..63bad7f08 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -35,7 +35,7 @@ - [Modified `slash_validator`](#modified-slash_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Execution engine](#execution-engine) - - [`execute_payload`](#execute_payload) + - [`notify_new_payload`](#notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) @@ -307,17 +307,17 @@ def slash_validator(state: BeaconState, The implementation-dependent `ExecutionEngine` protocol encapsulates the execution sub-system logic via: * a state object `self.execution_state` of type `ExecutionState` -* a state transition function `self.execute_payload` which applies changes to the `self.execution_state` +* a notification function `self.notify_new_payload` which may apply changes to the `self.execution_state` -*Note*: `execute_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. +*Note*: `notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. The body of this function is implementation dependent. The Engine API may be used to implement this and similarly defined functions via an external execution engine. -#### `execute_payload` +#### `notify_new_payload` ```python -def execute_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: +def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ @@ -353,7 +353,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.execute_payload(payload) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/sync/optimistic.md b/sync/optimistic.md index 80e9de3e9..abd345b5c 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -106,8 +106,8 @@ these conditions.* To optimistically import a block: -- The [`execute_payload`](../specs/bellatrix/beacon-chain.md#execute_payload) function MUST return `True` if the execution - engine returns `SYNCING` or `VALID`. An `INVALID` response MUST return `False`. +- The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution + engine returns `SYNCING`, `VALID`, or `ACCEPTED`. An `INVALID` or `INVALID_BLOCK_HASH` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. @@ -119,7 +119,7 @@ In addition to this change in validation, the consensus engine MUST track which blocks returned `SYNCING` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in -`process_block` (withstanding the modifications to `execute_payload`). +`process_block` (withstanding the modifications to `notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 16c759839..a24e3e199 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -25,7 +25,7 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def execute_payload(self, payload) -> bool: + def notify_new_payload(self, payload) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert payload == execution_payload From 020d72c7564bae26b7127ef9dc45bb7ace2c4682 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Jan 2022 08:43:18 +1100 Subject: [PATCH 105/112] Remove link fragment --- sync/optimistic.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index abd345b5c..5f07548c3 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -1,8 +1,5 @@ # Optimistic Sync - - -../specs/bellatrix/fork-choice.md#validate_merge_block ## Introduction In order to provide a syncing execution engine with a partial view of the head From cbbe6e744a626a9f4948ef2f8e8df54a03b289d6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 28 Jan 2022 09:45:08 +1100 Subject: [PATCH 106/112] Add aliases --- sync/optimistic.md | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index 5f07548c3..b44384d34 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -25,6 +25,17 @@ statelessness). ## Helpers +For brevity, we define two aliases for values of the `status` field on +`PayloadStatusV1`: + +- Alias `NOT_VALIDATED` to: + - `SYNCING` + - `ACCEPTED` +- Alias `INVALIDATED` to: + - `INVALID` + - `INVALID_BLOCK_HASH` + - `INVALID_TERMINAL_BLOCK` + Let `head: BeaconBlock` be the result of calling of the fork choice algorithm at the time of block production. Let `head_block_root: Root` be the root of that block. @@ -34,8 +45,8 @@ BeaconState]` be the blocks (and accompanying states) that have been verified either completely or optimistically. Let `optimistic_roots: Set[Root]` be the set of `hash_tree_root(block)` for all -optimistically imported blocks which have only received a `SYNCING` designation -from an execution engine (i.e., they are not known to be `INVALID` or `VALID`). +optimistically imported blocks which have only received a `NOT_VALIDATED` designation +from an execution engine (i.e., they are not known to be `INVALIDATED` or `VALID`). Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where `time` is the UNIX time according to the local system clock. @@ -56,7 +67,7 @@ def is_optimistic(opt_store: OptimisticStore, block: BeaconBlock) -> bool: ```python def latest_verified_ancestor(opt_store: OptimisticStore, block: BeaconBlock) -> BeaconBlock: - # It is assumed that the `block` parameter is never an INVALID block. + # It is assumed that the `block` parameter is never an INVALIDATED block. while True: if not is_optimistic(opt_store, block) or block.parent_root == Root(): return block @@ -104,35 +115,35 @@ these conditions.* To optimistically import a block: - The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution - engine returns `SYNCING`, `VALID`, or `ACCEPTED`. An `INVALID` or `INVALID_BLOCK_HASH` response MUST return `False`. + engine returns `NOT_VALIDATED` or `VALID`. An `INVALIDATED` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the `pow_block` and `pow_parent` are unknown to the execution engine. - All other assertions in [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) (e.g., `TERMINAL_BLOCK_HASH`) MUST prevent an optimistic import. -- The parent of the block MUST NOT have an INVALID execution payload. +- The parent of the block MUST NOT have an `INVALIDATED` execution payload. In addition to this change in validation, the consensus engine MUST track which -blocks returned `SYNCING` and which returned `VALID` for subsequent processing. +blocks returned `NOT_VALIDATED` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in `process_block` (withstanding the modifications to `notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify -the status of `SYNCING` blocks to be either `VALID` or `INVALID` based upon responses +the status of `NOT_VALIDATED` blocks to be either `VALID` or `INVALIDATED` based upon responses from an execution engine. I.e., perform the following transitions: -- `SYNCING` -> `VALID` -- `SYNCING` -> `INVALID` +- `NOT_VALIDATED` -> `VALID` +- `NOT_VALIDATED` -> `INVALIDATED` -When a block transitions from `SYNCING` -> `VALID`, all *ancestors* of the -block MUST also transition from `SYNCING` -> `VALID`. Such a block and any previously `SYNCING` ancestors are no longer +When a block transitions from `NOT_VALIDATED` -> `VALID`, all *ancestors* of the +block MUST also transition from `NOT_VALIDATED` -> `VALID`. Such a block and any previously `NOT_VALIDATED` ancestors are no longer considered "optimistically imported". -When a block transitions from `SYNCING` -> `INVALID`, all *descendants* of the -block MUST also transition from `SYNCING` -> `INVALID`. +When a block transitions from `NOT_VALIDATED` -> `INVALIDATED`, all *descendants* of the +block MUST also transition from `NOT_VALIDATED` -> `INVALIDATED`. -When a block transitions from the `SYNCING` state, it is removed from the set of +When a block transitions from the `NOT_VALIDATED` state, it is removed from the set of `opt_store.optimistic_roots`. When a "merge block" (i.e. the first block which enables execution in a chain) is declared to be @@ -141,7 +152,7 @@ When a "merge block" (i.e. the first block which enables execution in a chain) i MUST be run against the merge block. If the block fails [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block), the merge block MUST be treated the same as -an `INVALID` block (i.e., it and all its descendants are invalidated and +an `INVALIDATED` block (i.e., it and all its descendants are invalidated and removed from the block tree). ### Execution Engine Errors @@ -155,11 +166,11 @@ validity request for some block, a consensus engine: ### Assumptions about Execution Engine Behaviour -This specification assumes execution engines will only return `SYNCING` when -there is insufficient information available to make a `VALID` or `INVALID` +This specification assumes execution engines will only return `NOT_VALIDATED` when +there is insufficient information available to make a `VALID` or `INVALIDATED` determination on the given `ExecutionPayload` (e.g., the parent payload is -unknown). Specifically, `SYNCING` responses should be fork-specific, in that -the search for a block on one chain MUST NOT trigger a `SYNCING` response for +unknown). Specifically, `NOT_VALIDATED` responses should be fork-specific, in that +the search for a block on one chain MUST NOT trigger a `NOT_VALIDATED` response for another chain. ### Re-Orgs @@ -167,16 +178,16 @@ another chain. The consensus engine MUST support any chain reorganisation which does *not* affect the justified checkpoint. -If the justified checkpoint transitions from `SYNCING` -> `INVALID`, a +If the justified checkpoint transitions from `NOT_VALIDATED` -> `INVALIDATED`, a consensus engine MAY choose to alert the user and force the application to exit. ## Fork Choice Consensus engines MUST support removing blocks from fork choice that transition -from `SYNCING` to `INVALID`. Specifically, a block deemed `INVALID` at any +from `NOT_VALIDATED` to `INVALIDATED`. Specifically, a block deemed `INVALIDATED` at any point MUST NOT be included in the canonical chain and the weights from those -`INVALID` blocks MUST NOT be applied to any `VALID` or `SYNCING` ancestors. +`INVALIDATED` blocks MUST NOT be applied to any `VALID` or `NOT_VALIDATED` ancestors. ### Fork Choice Poisoning @@ -333,7 +344,7 @@ unlikely attack increases or decreases our total risk. Presently, it appears that `SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY` sits in a sweet spot for this trade-off. -### Transitioning from VALID -> INVALID or INVALID -> VALID +### Transitioning from VALID -> INVALIDATED or INVALIDATED -> VALID These operations are purposefully omitted. It is outside of the scope of the specification since it's only possible with a faulty EE. From 20c8d0d641eb62f869d03f6a8560db15ea55c397 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Thu, 27 Jan 2022 23:57:52 +0100 Subject: [PATCH 107/112] Update reserved libp2p error documentation The spec reserves the libp2p error code range `[3, 127]` for future use but actually defines error code `3` as `ResourceUnavailable`. This patch updates the reserved range to `[4, 127]`. --- specs/das/p2p-interface.md | 5 ----- specs/phase0/p2p-interface.md | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/specs/das/p2p-interface.md b/specs/das/p2p-interface.md index b4a0a9d2c..b1d96c718 100644 --- a/specs/das/p2p-interface.md +++ b/specs/das/p2p-interface.md @@ -196,11 +196,6 @@ This builds on top of the protocol identification and encoding spec which was in Note that DAS networking uses a different protocol prefix: `/eth2/das/req` -The result codes are extended with: -- 3: **ResourceUnavailable** -- when the request was valid but cannot be served at this point in time. - -TODO: unify with phase0? Lighthoue already defined this in their response codes enum. - ### Messages #### DASQuery diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f5c138f99..4532feac8 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -569,11 +569,11 @@ The response code can have one of the following values, encoded as a single unsi The response payload adheres to the `ErrorMessage` schema (described below). - 3: **ResourceUnavailable** -- the responder does not have requested resource. The response payload adheres to the `ErrorMessage` schema (described below). - *Note*: This response code is only valid as a response to `BlocksByRange`. + *Note*: This response code is only valid as a response where specified. Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. -The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognized expressly. +The range `[4, 127]` is RESERVED for future usages, and should be treated as error if not recognized expressly. The `ErrorMessage` schema is: From 129d9e28b475d1624ed979432bcacb781494eba3 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 27 Jan 2022 19:08:59 -0800 Subject: [PATCH 108/112] add randomized tests for bellatrix --- .../test/bellatrix/random/__init__.py | 0 .../test/bellatrix/random/test_random.py | 438 ++++++++++++++++++ .../test/utils/randomized_block_tests.py | 20 +- tests/generators/random/Makefile | 2 + tests/generators/random/generate.py | 11 +- tests/generators/random/main.py | 6 +- 6 files changed, 473 insertions(+), 4 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py b/tests/core/pyspec/eth2spec/test/bellatrix/random/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py b/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py new file mode 100644 index 000000000..2dccaf949 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/bellatrix/random/test_random.py @@ -0,0 +1,438 @@ +""" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +""" + +from eth2spec.test.helpers.constants import BELLATRIX +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_0(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_1(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_2(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_3(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_4(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_5(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_6(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_7(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_8(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_9(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_10(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_11(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_12(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_13(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_14(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([BELLATRIX]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_15(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_bellatrix + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_bellatrix', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_bellatrix'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 02a5464f7..386b4c8b2 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -55,12 +55,22 @@ def randomize_state(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): return scenario_state -def randomize_state_altair(spec, state, stats): - scenario_state = randomize_state(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1) +def randomize_state_altair(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): + scenario_state = randomize_state(spec, state, stats, exit_fraction=exit_fraction, slash_fraction=slash_fraction) randomize_inactivity_scores(spec, state) return scenario_state +def randomize_state_bellatrix(spec, state, stats, exit_fraction=0.1, slash_fraction=0.1): + scenario_state = randomize_state_altair(spec, + state, + stats, + exit_fraction=exit_fraction, + slash_fraction=slash_fraction) + # TODO: randomize execution payload, merge status, etc. + return scenario_state + + # epochs def epochs_until_leak(spec): @@ -179,6 +189,12 @@ def random_block_altair_with_cycling_sync_committee_participation(spec, return block +def random_block_bellatrix(spec, state, signed_blocks, scenario_state): + block = random_block_altair_with_cycling_sync_committee_participation(spec, state, signed_blocks, scenario_state) + # TODO: return randomized execution payload + return block + + # validations def no_op_validation(_spec, _state): diff --git a/tests/generators/random/Makefile b/tests/generators/random/Makefile index 1b518bfde..607d69a47 100644 --- a/tests/generators/random/Makefile +++ b/tests/generators/random/Makefile @@ -4,5 +4,7 @@ all: pip3 install -r requirements.txt rm -f ../../core/pyspec/eth2spec/test/phase0/random/test_random.py rm -f ../../core/pyspec/eth2spec/test/altair/random/test_random.py + rm -f ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py python3 generate.py phase0 > ../../core/pyspec/eth2spec/test/phase0/random/test_random.py python3 generate.py altair > ../../core/pyspec/eth2spec/test/altair/random/test_random.py + python3 generate.py bellatrix > ../../core/pyspec/eth2spec/test/bellatrix/random/test_random.py diff --git a/tests/generators/random/generate.py b/tests/generators/random/generate.py index b880cb12b..f96f05a75 100644 --- a/tests/generators/random/generate.py +++ b/tests/generators/random/generate.py @@ -19,8 +19,10 @@ from eth2spec.test.utils.randomized_block_tests import ( no_op_validation, randomize_state, randomize_state_altair, + randomize_state_bellatrix, random_block, random_block_altair_with_cycling_sync_committee_participation, + random_block_bellatrix, last_slot_in_epoch, random_slot_in_epoch, penultimate_slot_in_epoch, @@ -30,7 +32,7 @@ from eth2spec.test.utils.randomized_block_tests import ( transition_to_leaking, transition_without_leak, ) -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX # Ensure this many blocks are present in *each* randomized scenario @@ -254,5 +256,12 @@ if __name__ == "__main__": state_randomizer=randomize_state_altair, block_randomizer=random_block_altair_with_cycling_sync_committee_participation, ) + if BELLATRIX in sys.argv: + did_generate = True + run_generate_tests_to_std_out( + BELLATRIX, + state_randomizer=randomize_state_bellatrix, + block_randomizer=random_block_bellatrix, + ) if not did_generate: warnings.warn("no phase given for test generation") diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py index f6f1b1847..1791da83b 100644 --- a/tests/generators/random/main.py +++ b/tests/generators/random/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators @@ -9,10 +9,14 @@ if __name__ == "__main__": altair_mods = {key: 'eth2spec.test.altair.random.test_' + key for key in [ 'random', ]} + bellatrix_mods = {key: 'eth2spec.test.bellatrix.random.test_' + key for key in [ + 'random', + ]} all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, + BELLATRIX: bellatrix_mods, } run_state_test_generators(runner_name="random", all_mods=all_mods) From 67fcbf22ec7b478312d590f9b620b2126343a4a1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jan 2022 06:44:04 -0700 Subject: [PATCH 109/112] bump version --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index db1527897..a5e428299 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.1.8 \ No newline at end of file +1.1.9 \ No newline at end of file From 3d7f3070208f98d6e9e8356d3acbe31474501046 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 4 Feb 2022 17:46:37 -0800 Subject: [PATCH 110/112] Refactor test_sync_protocol.py a bit --- .../altair/unittests/test_sync_protocol.py | 75 ++++--------------- .../eth2spec/test/helpers/light_client.py | 35 +++++++++ 2 files changed, 50 insertions(+), 60 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/light_client.py diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index df0b33421..04553b3f8 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -5,38 +5,29 @@ from eth2spec.test.context import ( with_presets, with_altair_and_later, ) -from eth2spec.test.helpers.attestations import next_epoch_with_attestations +from eth2spec.test.helpers.attestations import ( + next_epoch_with_attestations, +) from eth2spec.test.helpers.block import ( build_empty_block, build_empty_block_for_next_slot, ) from eth2spec.test.helpers.constants import MINIMAL +from eth2spec.test.helpers.light_client import ( + get_sync_aggregate, + initialize_light_client_store, +) from eth2spec.test.helpers.state import ( next_slots, state_transition_and_sign_block, ) -from eth2spec.test.helpers.sync_committee import ( - compute_aggregate_sync_committee_signature, -) from eth2spec.test.helpers.merkle import build_proof -def _initialize_light_client_store(spec, state): - return spec.LightClientStore( - finalized_header=spec.BeaconBlockHeader(), - current_sync_committee=state.current_sync_committee, - next_sync_committee=state.next_sync_committee, - best_valid_update=None, - optimistic_header=spec.BeaconBlockHeader(), - previous_max_active_participants=0, - current_max_active_participants=0, - ) - - @with_altair_and_later @spec_state_test def test_process_light_client_update_not_timeout(spec, state): - store = _initialize_light_client_store(spec, state) + store = initialize_light_client_store(spec, state) # Block at slot 1 doesn't increase sync committee period, so it won't force update store.finalized_header block = build_empty_block_for_next_slot(spec, state) @@ -49,19 +40,7 @@ def test_process_light_client_update_not_timeout(spec, state): body_root=signed_block.message.body.hash_tree_root(), ) # Sync committee signing the header - all_pubkeys = [v.pubkey for v in state.validators] - committee = [all_pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - sync_committee_bits = [True] * len(committee) - sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block_header.slot, - committee, - ) - sync_aggregate = spec.SyncAggregate( - sync_committee_bits=sync_committee_bits, - sync_committee_signature=sync_committee_signature, - ) + sync_aggregate = get_sync_aggregate(spec, state, block_header, block_root=None) next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] # Ensure that finality checkpoint is genesis @@ -94,7 +73,7 @@ def test_process_light_client_update_not_timeout(spec, state): @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): - store = _initialize_light_client_store(spec, state) + store = initialize_light_client_store(spec, state) # Forward to next sync committee period next_slots(spec, state, spec.UPDATE_TIMEOUT) @@ -113,20 +92,8 @@ def test_process_light_client_update_timeout(spec, state): ) # Sync committee signing the finalized_block_header - all_pubkeys = [v.pubkey for v in state.validators] - committee = [all_pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - sync_committee_bits = [True] * len(committee) - sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block_header.slot, - committee, - block_root=spec.Root(block_header.hash_tree_root()), - ) - sync_aggregate = spec.SyncAggregate( - sync_committee_bits=sync_committee_bits, - sync_committee_signature=sync_committee_signature, - ) + sync_aggregate = get_sync_aggregate( + spec, state, block_header, block_root=spec.Root(block_header.hash_tree_root())) # Sync committee is updated next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) @@ -158,7 +125,7 @@ def test_process_light_client_update_timeout(spec, state): @spec_state_test @with_presets([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): - store = _initialize_light_client_store(spec, state) + store = initialize_light_client_store(spec, state) # Change finality blocks = [] @@ -191,20 +158,8 @@ def test_process_light_client_update_finality_updated(spec, state): ) # Sync committee signing the finalized_block_header - all_pubkeys = [v.pubkey for v in state.validators] - committee = [all_pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - sync_committee_bits = [True] * len(committee) - sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - block_header.slot, - committee, - block_root=spec.Root(block_header.hash_tree_root()), - ) - sync_aggregate = spec.SyncAggregate( - sync_committee_bits=sync_committee_bits, - sync_committee_signature=sync_committee_signature, - ) + sync_aggregate = get_sync_aggregate( + spec, state, block_header, block_root=spec.Root(block_header.hash_tree_root())) update = spec.LightClientUpdate( attested_header=block_header, diff --git a/tests/core/pyspec/eth2spec/test/helpers/light_client.py b/tests/core/pyspec/eth2spec/test/helpers/light_client.py new file mode 100644 index 000000000..15c764fc4 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/light_client.py @@ -0,0 +1,35 @@ +from eth2spec.test.helpers.sync_committee import ( + compute_aggregate_sync_committee_signature, +) + + +def initialize_light_client_store(spec, state): + return spec.LightClientStore( + finalized_header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + best_valid_update=None, + optimistic_header=spec.BeaconBlockHeader(), + previous_max_active_participants=0, + current_max_active_participants=0, + ) + + +def get_sync_aggregate(spec, state, block_header, block_root=None, signature_slot=None): + if signature_slot is None: + signature_slot = block_header.slot + + all_pubkeys = [v.pubkey for v in state.validators] + committee = [all_pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block_header.slot, + committee, + block_root=block_root, + ) + return spec.SyncAggregate( + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + ) From 7c492ce4ce5043f1d8f32d8d02ab76642774fe87 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 6 Feb 2022 13:32:21 -0800 Subject: [PATCH 111/112] Fix typo --- sync/optimistic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/optimistic.md b/sync/optimistic.md index b44384d34..8e6acc442 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -82,9 +82,9 @@ def is_execution_block(block: BeaconBlock) -> bool: ```python def is_optimistic_candidate_block(opt_store: OptimisticStore, current_slot: Slot, block: BeaconBlock) -> bool: justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root - justifed_is_execution_block = is_execution_block(opt_store.blocks[justified_root]) + justified_is_execution_block = is_execution_block(opt_store.blocks[justified_root]) block_is_deep = block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot - return justifed_is_execution_block or block_is_deep + return justified_is_execution_block or block_is_deep ``` Let only a node which returns `is_optimistic(opt_store, head) is True` be an *optimistic From 1280fe2a02d60b87dab108a8bfa5b3338528b7c4 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 22 Feb 2022 17:16:33 +0600 Subject: [PATCH 112/112] Bellatrix: random -> prev_randao --- specs/bellatrix/beacon-chain.md | 10 +++++----- specs/bellatrix/fork-choice.md | 2 +- specs/bellatrix/validator.md | 2 +- .../block_processing/test_process_execution_payload.py | 6 +++--- .../bellatrix/unittests/validator/test_validator.py | 2 +- .../pyspec/eth2spec/test/helpers/execution_payload.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 63bad7f08..b37a8ab71 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -171,7 +171,7 @@ class ExecutionPayload(Container): state_root: Bytes32 receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] - random: Bytes32 # 'difficulty' in the yellow paper + prev_randao: Bytes32 # 'difficulty' in the yellow paper block_number: uint64 # 'number' in the yellow paper gas_limit: uint64 gas_used: uint64 @@ -193,7 +193,7 @@ class ExecutionPayloadHeader(Container): state_root: Bytes32 receipts_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] - random: Bytes32 + prev_randao: Bytes32 block_number: uint64 gas_limit: uint64 gas_used: uint64 @@ -348,8 +348,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash - # Verify random - assert payload.random == get_randao_mix(state, get_current_epoch(state)) + # Verify prev_randao + assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid @@ -361,7 +361,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe state_root=payload.state_root, receipts_root=payload.receipts_root, logs_bloom=payload.logs_bloom, - random=payload.random, + prev_randao=payload.prev_randao, block_number=payload.block_number, gas_limit=payload.gas_limit, gas_used=payload.gas_used, diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index 91fda88bc..0f5a7b646 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -80,7 +80,7 @@ Used to signal to initiate the payload build process via `notify_forkchoice_upda @dataclass class PayloadAttributes(object): timestamp: uint64 - random: Bytes32 + prev_randao: Bytes32 suggested_fee_recipient: ExecutionAddress ``` diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index 7e0369a39..47de49ba6 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -143,7 +143,7 @@ def prepare_execution_payload(state: BeaconState, # Set the forkchoice head and initiate the payload build process payload_attributes = PayloadAttributes( timestamp=compute_timestamp_at_slot(state, state.slot), - random=get_randao_mix(state, get_current_epoch(state)), + prev_randao=get_randao_mix(state, get_current_epoch(state)), suggested_fee_recipient=suggested_fee_recipient, ) return execution_engine.notify_forkchoice_updated(parent_hash, finalized_block_hash, payload_attributes) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index a24e3e199..cd7a74259 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -153,7 +153,7 @@ def test_bad_random_first_payload(spec, state): # execution payload execution_payload = build_empty_execution_payload(spec, state) - execution_payload.random = b'\x42' * 32 + execution_payload.prev_randao = b'\x42' * 32 yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) @@ -167,7 +167,7 @@ def test_bad_random_regular_payload(spec, state): # execution payload execution_payload = build_empty_execution_payload(spec, state) - execution_payload.random = b'\x04' * 32 + execution_payload.prev_randao = b'\x04' * 32 yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) @@ -182,7 +182,7 @@ def test_bad_everything_regular_payload(spec, state): # execution payload execution_payload = build_empty_execution_payload(spec, state) execution_payload.parent_hash = spec.Hash32() - execution_payload.random = spec.Bytes32() + execution_payload.prev_randao = spec.Bytes32() execution_payload.timestamp = 0 yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py index 5e767d2cd..05a941dfe 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py @@ -103,7 +103,7 @@ def test_prepare_execution_payload(spec, state): # 1. Handle `is_merge_complete` if is_merge_complete: - state.latest_execution_payload_header = spec.ExecutionPayloadHeader(random=b'\x12' * 32) + state.latest_execution_payload_header = spec.ExecutionPayloadHeader(prev_randao=b'\x12' * 32) else: state.latest_execution_payload_header = spec.ExecutionPayloadHeader() diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 49cec884a..14602a2cd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -16,7 +16,7 @@ def build_empty_execution_payload(spec, state, randao_mix=None): receipts_root=b"no receipts here" + b"\x00" * 16, # TODO: root of empty MPT may be better. logs_bloom=spec.ByteVector[spec.BYTES_PER_LOGS_BLOOM](), # TODO: zeroed logs bloom for empty logs ok? block_number=latest.block_number + 1, - random=randao_mix, + prev_randao=randao_mix, gas_limit=latest.gas_limit, # retain same limit gas_used=0, # empty block, 0 gas timestamp=timestamp, @@ -38,7 +38,7 @@ def get_execution_payload_header(spec, execution_payload): state_root=execution_payload.state_root, receipts_root=execution_payload.receipts_root, logs_bloom=execution_payload.logs_bloom, - random=execution_payload.random, + prev_randao=execution_payload.prev_randao, block_number=execution_payload.block_number, gas_limit=execution_payload.gas_limit, gas_used=execution_payload.gas_used, diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 0539a5d74..d0e470893 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -30,7 +30,7 @@ def get_sample_genesis_execution_payload_header(spec, state_root=b'\x20' * 32, receipts_root=b'\x20' * 32, logs_bloom=b'\x35' * spec.BYTES_PER_LOGS_BLOOM, - random=eth1_block_hash, + prev_randao=eth1_block_hash, block_number=0, gas_limit=30000000, base_fee_per_gas=1000000000,