Merge pull request #2999 from ethereum/dev

release v1.2.0

setup.py

@@ -586,6 +586,7 @@ class EIP4844SpecBuilder(BellatrixSpecBuilder):
         return super().imports(preset_name) + f'''
 from eth2spec.utils import kzg
 from eth2spec.bellatrix import {preset_name} as bellatrix
+from eth2spec.utils.ssz.ssz_impl import serialize as ssz_serialize
 '''
 
     @classmethod

specs/altair/light-client/sync-protocol.md

@@ -47,7 +47,7 @@ Such environments include resource-constrained devices (e.g. phones for trust-mi
 and metered VMs (e.g. blockchain VMs for cross-chain bridges).
 
 This document suggests a minimal light client design for the beacon chain that
-uses sync committees introduced in [this beacon chain extension](./beacon-chain.md).
+uses sync committees introduced in [this beacon chain extension](../beacon-chain.md).
 
 Additional documents describe how the light client sync protocol can be used:
 - [Full node](./full-node.md)

specs/capella/beacon-chain.md

@@ -29,7 +29,7 @@
     - [`BeaconState`](#beaconstate)
 - [Helpers](#helpers)
   - [Beacon state mutators](#beacon-state-mutators)
-    - [`withdraw`](#withdraw)
+    - [`withdraw_balance`](#withdraw_balance)
   - [Predicates](#predicates)
     - [`has_eth1_withdrawal_credential`](#has_eth1_withdrawal_credential)
     - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator)
@@ -52,11 +52,11 @@
 
 Capella is a consensus-layer upgrade containing a number of features related
 to validator withdrawals. Including:
-* Automatic withdrawals of `withdrawable` validators
+* Automatic withdrawals of `withdrawable` validators.
 * Partial withdrawals sweep for validators with 0x01 withdrawal
-  credentials and balances in exceess of `MAX_EFFECTIVE_BALANCE`
+  credentials and balances in excess of `MAX_EFFECTIVE_BALANCE`.
 * Operation to change from `BLS_WITHDRAWAL_PREFIX` to
-  `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator
+  `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator.
 
 ## Custom types
 
@@ -64,7 +64,7 @@ We define the following Python custom types for type hinting and readability:
 
 | Name | SSZ equivalent | Description |
 | - | - | - |
-| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal`|
+| `WithdrawalIndex` | `uint64` | an index of a `Withdrawal` |
 
 ## Constants
 
@@ -84,9 +84,9 @@ We define the following Python custom types for type hinting and readability:
 
 ### State list lengths
 
-| Name | Value | Unit | Duration |
+| Name | Value | Unit |
-| - | - | :-: | :-: |
+| - | - | :-: |
-| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state|
+| `WITHDRAWAL_QUEUE_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawals enqueued in state |
 
 ### Max operations per block
 
@@ -266,7 +266,7 @@ class BeaconState(Container):
 
 ### Beacon state mutators
 
-#### `withdraw`
+#### `withdraw_balance`
 
 ```python
 def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount: Gwei) -> None:
@@ -289,7 +289,7 @@ def withdraw_balance(state: BeaconState, validator_index: ValidatorIndex, amount
 ```python
 def has_eth1_withdrawal_credential(validator: Validator) -> bool:
     """
-    Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential
+    Check if ``validator`` has an 0x01 prefixed "eth1" withdrawal credential.
     """
     return validator.withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX
 ```

specs/capella/fork-choice.md

@@ -58,5 +58,5 @@ class PayloadAttributes(object):
     timestamp: uint64
     prev_randao: Bytes32
     suggested_fee_recipient: ExecutionAddress
-    withdrawals: Sequence[Withdrawal]  # new in Capella
+    withdrawals: Sequence[Withdrawal]  # [New in Capella]
 ```

specs/capella/fork.md

@@ -65,7 +65,7 @@ an irregular state change is made to upgrade to Capella.
 
 The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `CAPELLA_FORK_EPOCH * SLOTS_PER_EPOCH`.
 Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document.
-In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead the logic must be within `process_slots`.
+In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead, the logic must be within `process_slots`.
 
 ```python
 def upgrade_to_capella(pre: bellatrix.BeaconState) -> BeaconState:

specs/eip4844/p2p-interface.md

@@ -108,7 +108,7 @@ Alias `sidecar = signed_blobs_sidecar.message`.
 - _[REJECT]_ the beacon proposer signature, `signed_blobs_sidecar.signature`, is valid -- i.e.
     - Let `domain = get_domain(state, DOMAIN_BLOBS_SIDECAR, sidecar.beacon_block_slot // SLOTS_PER_EPOCH)`
     - Let `signing_root = compute_signing_root(sidecar, domain)`
-    - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blob_header.signature) is True`,   
+    - Verify `bls.Verify(proposer_pubkey, signing_root, signed_blobs_sidecar.signature) is True`,   
       where `proposer_pubkey` is the pubkey of the beacon block proposer of `sidecar.beacon_block_slot`
 - _[IGNORE]_ The sidecar is the first sidecar with valid signature received for the `(proposer_index, sidecar.beacon_block_slot)` combination,
   where `proposer_index` is the validator index of the beacon block proposer of `sidecar.beacon_block_slot`

specs/eip4844/polynomial-commitments.md

@@ -15,8 +15,8 @@
   - [BLS12-381 helpers](#bls12-381-helpers)
     - [`bls_modular_inverse`](#bls_modular_inverse)
     - [`div`](#div)
-    - [`lincomb`](#lincomb)
+    - [`g1_lincomb`](#g1_lincomb)
-    - [`matrix_lincomb`](#matrix_lincomb)
+    - [`vector_lincomb`](#vector_lincomb)
   - [KZG](#kzg)
     - [`blob_to_kzg_commitment`](#blob_to_kzg_commitment)
     - [`verify_kzg_proof`](#verify_kzg_proof)
@@ -85,10 +85,10 @@ def div(x: BLSFieldElement, y: BLSFieldElement) -> BLSFieldElement:
     return (int(x) * int(bls_modular_inverse(y))) % BLS_MODULUS
 ```
 
-#### `lincomb`
+#### `g1_lincomb`
 
 ```python
-def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment:
+def g1_lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement]) -> KZGCommitment:
     """
     BLS multiscalar multiplication. This function can be optimized using Pippenger's algorithm and variants.
     """
@@ -99,10 +99,10 @@ def lincomb(points: Sequence[KZGCommitment], scalars: Sequence[BLSFieldElement])
     return KZGCommitment(bls.G1_to_bytes48(result))
 ```
 
-#### `matrix_lincomb`
+#### `vector_lincomb`
 
 ```python
-def matrix_lincomb(vectors: Sequence[Sequence[BLSFieldElement]],
+def vector_lincomb(vectors: Sequence[Sequence[BLSFieldElement]],
                    scalars: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]:
     """
     Given a list of ``vectors``, interpret it as a 2D matrix and compute the linear combination
@@ -123,7 +123,7 @@ KZG core functions. These are also defined in EIP-4844 execution specs.
 
 ```python
 def blob_to_kzg_commitment(blob: Blob) -> KZGCommitment:
-    return lincomb(KZG_SETUP_LAGRANGE, blob)
+    return g1_lincomb(KZG_SETUP_LAGRANGE, blob)
 ```
 
 #### `verify_kzg_proof`
@@ -165,7 +165,7 @@ def compute_kzg_proof(polynomial: Sequence[BLSFieldElement], z: BLSFieldElement)
 
     # Calculate quotient polynomial by doing point-by-point division
     quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)]
-    return KZGProof(lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial))
+    return KZGProof(g1_lincomb(KZG_SETUP_LAGRANGE, quotient_polynomial))
 ```
 
 ### Polynomials

specs/eip4844/validator.md

@@ -93,9 +93,10 @@ def is_data_available(slot: Slot, beacon_block_root: Root, blob_kzg_commitments:
 ```python
 def hash_to_bls_field(x: Container) -> BLSFieldElement:
     """
-    This function is used to generate Fiat-Shamir challenges. The output is not uniform over the BLS field.
+    Compute 32-byte hash of serialized container and convert it to BLS field.
+    The output is not uniform over the BLS field.
     """
-    return int.from_bytes(hash_tree_root(x), "little") % BLS_MODULUS
+    return int.from_bytes(hash(ssz_serialize(x)), "little") % BLS_MODULUS
 ```
 
 ### `compute_powers`
@@ -116,7 +117,7 @@ def compute_powers(x: BLSFieldElement, n: uint64) -> Sequence[BLSFieldElement]:
 
 ```python
 def compute_aggregated_poly_and_commitment(
-        blobs: Sequence[BLSFieldElement],
+        blobs: Sequence[Sequence[BLSFieldElement]],
         kzg_commitments: Sequence[KZGCommitment]) -> Tuple[Polynomial, KZGCommitment]:
     """
     Return the aggregated polynomial and aggregated KZG commitment.
@@ -126,10 +127,10 @@ def compute_aggregated_poly_and_commitment(
     r_powers = compute_powers(r, len(kzg_commitments))
 
     # Create aggregated polynomial in evaluation form
-    aggregated_poly = Polynomial(matrix_lincomb(blobs, r_powers))
+    aggregated_poly = Polynomial(vector_lincomb(blobs, r_powers))
 
     # Compute commitment to aggregated polynomial
-    aggregated_poly_commitment = KZGCommitment(lincomb(kzg_commitments, r_powers))
+    aggregated_poly_commitment = KZGCommitment(g1_lincomb(kzg_commitments, r_powers))
 
     return aggregated_poly, aggregated_poly_commitment
 ```

sync/optimistic.md

@@ -88,8 +88,8 @@ Let `current_slot: Slot` be `(time - genesis_time) // SECONDS_PER_SLOT` where
 class OptimisticStore(object):
     optimistic_roots: Set[Root]
     head_block_root: Root
-    blocks: Dict[Root, BeaconBlock]
+    blocks: Dict[Root, BeaconBlock] = field(default_factory=dict)
-    block_states: Dict[Root, BeaconState]
+    block_states: Dict[Root, BeaconState] = field(default_factory=dict)
 ```
 
 ```python

tests/core/pyspec/eth2spec/VERSION.txt

@@ -1 +1 @@
-1.2.0-rc.3
+1.2.0

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -9,7 +9,6 @@ import sys
 import json
 from typing import Iterable, AnyStr, Any, Callable
 import traceback
-
 from ruamel.yaml import (
     YAML,
 )
@@ -98,6 +97,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     yaml = YAML(pure=True)
     yaml.default_flow_style = None
 
+    def _represent_none(self, _):
+        return self.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+    yaml.representer.add_representer(type(None), _represent_none)
+
     # Spec config is using a YAML subset
     cfg_yaml = YAML(pure=True)
     cfg_yaml.default_flow_style = False  # Emit separate line for each key

tests/core/pyspec/eth2spec/test/bellatrix/sync/test_optimistic.py

@@ -0,0 +1,99 @@
+from eth2spec.test.context import (
+    spec_state_test,
+    with_bellatrix_and_later,
+)
+from eth2spec.test.helpers.attestations import (
+    state_transition_with_full_block,
+)
+from eth2spec.test.helpers.block import (
+    build_empty_block_for_next_slot,
+)
+from eth2spec.test.helpers.fork_choice import (
+    get_genesis_forkchoice_store_and_block,
+    on_tick_and_append_step,
+)
+from eth2spec.test.helpers.optimistic_sync import (
+    PayloadStatusV1,
+    PayloadStatusV1Status,
+    MegaStore,
+    add_optimistic_block,
+    get_optimistic_store,
+)
+from eth2spec.test.helpers.state import (
+    next_epoch,
+    state_transition_and_sign_block,
+)
+
+
+@with_bellatrix_and_later
+@spec_state_test
+def test_from_syncing_to_invalid(spec, state):
+    test_steps = []
+    # Initialization
+    fc_store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    op_store = get_optimistic_store(spec, state, anchor_block)
+    mega_store = MegaStore(spec, fc_store, op_store)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+
+    next_epoch(spec, state)
+
+    current_time = (
+        (spec.SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY * 10 + state.slot) * spec.config.SECONDS_PER_SLOT
+        + fc_store.genesis_time
+    )
+    on_tick_and_append_step(spec, fc_store, current_time, test_steps)
+
+    # Block 0
+    block_0 = build_empty_block_for_next_slot(spec, state)
+    block_0.body.execution_payload.block_hash = spec.hash(bytes(f'block_0', 'UTF-8'))
+    signed_block = state_transition_and_sign_block(spec, state, block_0)
+    yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID)
+    assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+
+    state_0 = state.copy()
+
+    # Create VALID chain `a`
+    signed_blocks_a = []
+    for i in range(3):
+        block = build_empty_block_for_next_slot(spec, state)
+        block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_a_{i}', 'UTF-8'))
+        block.body.execution_payload.parent_hash = (
+            spec.hash(bytes(f'chain_a_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash
+        )
+
+        signed_block = state_transition_and_sign_block(spec, state, block)
+        yield from add_optimistic_block(spec, mega_store, signed_block, test_steps, status=PayloadStatusV1Status.VALID)
+        assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+        signed_blocks_a.append(signed_block.copy())
+
+    # Create SYNCING chain `b`
+    signed_blocks_b = []
+    state = state_0.copy()
+    for i in range(3):
+        block = build_empty_block_for_next_slot(spec, state)
+        block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_{i}', 'UTF-8'))
+        block.body.execution_payload.parent_hash = (
+            spec.hash(bytes(f'chain_b_{i - 1}', 'UTF-8')) if i != 0 else block_0.body.execution_payload.block_hash
+        )
+        signed_block = state_transition_with_full_block(spec, state, True, True, block=block)
+        signed_blocks_b.append(signed_block.copy())
+        yield from add_optimistic_block(spec, mega_store, signed_block, test_steps,
+                                        status=PayloadStatusV1Status.SYNCING)
+        assert spec.get_head(mega_store.fc_store) == mega_store.opt_store.head_block_root
+
+    # Now add block 4 to chain `b` with INVALID
+    block = build_empty_block_for_next_slot(spec, state)
+    block.body.execution_payload.block_hash = spec.hash(bytes(f'chain_b_3', 'UTF-8'))
+    block.body.execution_payload.parent_hash = signed_blocks_b[-1].message.body.execution_payload.block_hash
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    payload_status = PayloadStatusV1(
+        status=PayloadStatusV1Status.INVALID,
+        latest_valid_hash=block_0.body.execution_payload.block_hash,
+        validation_error="invalid",
+    )
+    yield from add_optimistic_block(spec, mega_store, signed_block, test_steps,
+                                    payload_status=payload_status)
+    assert mega_store.opt_store.head_block_root == signed_blocks_a[-1].message.hash_tree_root()
+
+    yield 'steps', test_steps

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -251,11 +251,13 @@ def state_transition_with_full_block(spec,
                                      fill_cur_epoch,
                                      fill_prev_epoch,
                                      participation_fn=None,
-                                     sync_aggregate=None):
+                                     sync_aggregate=None,
+                                     block=None):
     """
     Build and apply a block with attestions at the calculated `slot_to_attest` of current epoch and/or previous epoch.
     """
-    block = build_empty_block_for_next_slot(spec, state)
+    if block is None:
+        block = build_empty_block_for_next_slot(spec, state)
     if fill_cur_epoch and state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
         slot_to_attest = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
         if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)):

tests/core/pyspec/eth2spec/test/helpers/deposits.py

@@ -137,14 +137,21 @@ def prepare_random_genesis_deposits(spec,
     return deposits, root, deposit_data_list
 
 
-def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False):
+def prepare_state_and_deposit(spec, state, validator_index, amount,
+                              pubkey=None,
+                              privkey=None,
+                              withdrawal_credentials=None,
+                              signed=False):
     """
     Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount.
     """
     deposit_data_list = []
 
-    pubkey = pubkeys[validator_index]
+    if pubkey is None:
-    privkey = privkeys[validator_index]
+        pubkey = pubkeys[validator_index]
+
+    if privkey is None:
+        privkey = privkeys[validator_index]
 
     # insecurely use pubkey as withdrawal key if no credentials provided
     if withdrawal_credentials is None:
@@ -196,7 +203,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef
 
     yield 'post', state
 
-    if not effective:
+    if not effective or not bls.KeyValidate(deposit.data.pubkey):
         assert len(state.validators) == pre_validator_count
         assert len(state.balances) == pre_validator_count
         if validator_index < pre_validator_count:

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -13,18 +13,8 @@ def get_anchor_root(spec, state):
     return spec.hash_tree_root(anchor_block_header)
 
 
-def add_block_to_store(spec, store, signed_block):
-    pre_state = store.block_states[signed_block.message.parent_root]
-    block_time = pre_state.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT
-
-    if store.time < block_time:
-        spec.on_tick(store, block_time)
-
-    spec.on_block(store, signed_block)
-
-
 def tick_and_add_block(spec, store, signed_block, test_steps, valid=True,
-                       merge_block=False, block_not_found=False):
+                       merge_block=False, block_not_found=False, is_optimistic=False):
     pre_state = store.block_states[signed_block.message.parent_root]
     block_time = pre_state.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT
     if merge_block:
@@ -37,6 +27,7 @@ def tick_and_add_block(spec, store, signed_block, test_steps, valid=True,
         spec, store, signed_block, test_steps,
         valid=valid,
         block_not_found=block_not_found,
+        is_optimistic=is_optimistic,
     )
 
     return post_state
@@ -119,28 +110,36 @@ def add_block(spec,
               signed_block,
               test_steps,
               valid=True,
-              block_not_found=False):
+              block_not_found=False,
+              is_optimistic=False):
     """
     Run on_block and on_attestation
     """
     yield get_block_file_name(signed_block), signed_block
 
     if not valid:
-        try:
+        if is_optimistic:
             run_on_block(spec, store, signed_block, valid=True)
-        except (AssertionError, BlockNotFoundException) as e:
-            if isinstance(e, BlockNotFoundException) and not block_not_found:
-                assert False
             test_steps.append({
                 'block': get_block_file_name(signed_block),
                 'valid': False,
             })
-            return
         else:
-            assert False
+            try:
-
+                run_on_block(spec, store, signed_block, valid=True)
-    run_on_block(spec, store, signed_block, valid=True)
+            except (AssertionError, BlockNotFoundException) as e:
-    test_steps.append({'block': get_block_file_name(signed_block)})
+                if isinstance(e, BlockNotFoundException) and not block_not_found:
+                    assert False
+                test_steps.append({
+                    'block': get_block_file_name(signed_block),
+                    'valid': False,
+                })
+                return
+            else:
+                assert False
+    else:
+        run_on_block(spec, store, signed_block, valid=True)
+        test_steps.append({'block': get_block_file_name(signed_block)})
 
     # An on_block step implies receiving block's attestations
     for attestation in signed_block.message.body.attestations:
@@ -153,25 +152,26 @@ def add_block(spec,
     block_root = signed_block.message.hash_tree_root()
     assert store.blocks[block_root] == signed_block.message
     assert store.block_states[block_root].hash_tree_root() == signed_block.message.state_root
-    test_steps.append({
+    if not is_optimistic:
-        'checks': {
+        test_steps.append({
-            'time': int(store.time),
+            'checks': {
-            'head': get_formatted_head_output(spec, store),
+                'time': int(store.time),
-            'justified_checkpoint': {
+                'head': get_formatted_head_output(spec, store),
-                'epoch': int(store.justified_checkpoint.epoch),
+                'justified_checkpoint': {
-                'root': encode_hex(store.justified_checkpoint.root),
+                    'epoch': int(store.justified_checkpoint.epoch),
-            },
+                    'root': encode_hex(store.justified_checkpoint.root),
-            'finalized_checkpoint': {
+                },
-                'epoch': int(store.finalized_checkpoint.epoch),
+                'finalized_checkpoint': {
-                'root': encode_hex(store.finalized_checkpoint.root),
+                    'epoch': int(store.finalized_checkpoint.epoch),
-            },
+                    'root': encode_hex(store.finalized_checkpoint.root),
-            'best_justified_checkpoint': {
+                },
-                'epoch': int(store.best_justified_checkpoint.epoch),
+                'best_justified_checkpoint': {
-                'root': encode_hex(store.best_justified_checkpoint.root),
+                    'epoch': int(store.best_justified_checkpoint.epoch),
-            },
+                    'root': encode_hex(store.best_justified_checkpoint.root),
-            'proposer_boost_root': encode_hex(store.proposer_boost_root),
+                },
-        }
+                'proposer_boost_root': encode_hex(store.proposer_boost_root),
-    })
+            }
+        })
 
     return store.block_states[signed_block.message.hash_tree_root()]
 

tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py

@@ -0,0 +1,204 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import (
+    Dict,
+    Optional,
+)
+
+from eth_utils import encode_hex
+
+from eth2spec.utils.ssz.ssz_typing import Bytes32
+from eth2spec.test.helpers.fork_choice import (
+    add_block,
+)
+
+
+class PayloadStatusV1StatusAlias(Enum):
+    NOT_VALIDATED = "NOT_VALIDATED"
+    INVALIDATED = "INVALIDATED"
+
+
+class PayloadStatusV1Status(Enum):
+    VALID = "VALID"
+    INVALID = "INVALID"
+    SYNCING = "SYNCING"
+    ACCEPTED = "ACCEPTED"
+    INVALID_BLOCK_HASH = "INVALID_BLOCK_HASH"
+
+    @property
+    def alias(self) -> PayloadStatusV1StatusAlias:
+        if self.value in (self.SYNCING.value, self.ACCEPTED.value):
+            return PayloadStatusV1StatusAlias.NOT_VALIDATED
+        elif self.value in (self.INVALID.value, self.INVALID_BLOCK_HASH.value):
+            return PayloadStatusV1StatusAlias.INVALIDATED
+
+
+@dataclass
+class PayloadStatusV1:
+    status: PayloadStatusV1Status = PayloadStatusV1Status.VALID
+    latest_valid_hash: Optional[Bytes32] = None
+    validation_error: Optional[str] = None
+
+    @property
+    def formatted_output(self):
+        return {
+            'status': str(self.status.value),
+            'latest_valid_hash': encode_hex(self.latest_valid_hash) if self.latest_valid_hash is not None else None,
+            'validation_error': str(self.validation_error) if self.validation_error is not None else None
+        }
+
+
+class MegaStore(object):
+    spec = None
+    fc_store = None
+    opt_store = None
+    block_payload_statuses: Dict[Bytes32, PayloadStatusV1] = dict()
+
+    def __init__(self, spec, fc_store, opt_store):
+        self.spec = spec
+        self.fc_store = fc_store
+        self.opt_store = opt_store
+
+
+def get_optimistic_store(spec, anchor_state, anchor_block):
+    assert anchor_block.state_root == anchor_state.hash_tree_root()
+
+    opt_store = spec.OptimisticStore(
+        optimistic_roots=set(),
+        head_block_root=anchor_block.hash_tree_root(),
+
+    )
+    anchor_block_root = anchor_block.hash_tree_root()
+    opt_store.blocks[anchor_block_root] = anchor_block.copy()
+    opt_store.block_states[anchor_block_root] = anchor_state.copy()
+
+    return opt_store
+
+
+def get_valid_flag_value(status: PayloadStatusV1Status) -> bool:
+    if status == PayloadStatusV1Status.VALID:
+        return True
+    elif status.alias == PayloadStatusV1StatusAlias.NOT_VALIDATED:
+        return True
+    else:
+        # status.alias == PayloadStatusV1StatusAlias.INVALIDATED or other cases
+        return False
+
+
+def add_optimistic_block(spec, mega_store, signed_block, test_steps,
+                         payload_status=None, status=PayloadStatusV1Status.SYNCING):
+    """
+    Add a block with optimistic sync logic
+
+    ``valid`` indicates if the given ``signed_block.message.body.execution_payload`` is valid/invalid
+    from ``notify_new_payload`` method response.
+    """
+    block = signed_block.message
+    block_root = block.hash_tree_root()
+    el_block_hash = block.body.execution_payload.block_hash
+
+    if payload_status is None:
+        payload_status = PayloadStatusV1(status=status)
+        if payload_status.status == PayloadStatusV1Status.VALID:
+            payload_status.latest_valid_hash = el_block_hash
+
+    mega_store.block_payload_statuses[block_root] = payload_status
+    test_steps.append({
+        'block_hash': encode_hex(el_block_hash),
+        'payload_status': payload_status.formatted_output,
+    })
+
+    # Set `valid` flag
+    valid = get_valid_flag_value(payload_status.status)
+
+    # Optimistic sync
+
+    # Case: INVALID
+    if payload_status.status == PayloadStatusV1Status.INVALID:
+        # Update parent status to INVALID
+        assert payload_status.latest_valid_hash is not None
+        current_block = block
+        while el_block_hash != payload_status.latest_valid_hash and el_block_hash != spec.Bytes32():
+            current_block_root = current_block.hash_tree_root()
+            assert current_block_root in mega_store.block_payload_statuses
+            mega_store.block_payload_statuses[current_block_root].status = PayloadStatusV1Status.INVALID
+            # Get parent
+            current_block = mega_store.fc_store.blocks[current_block.parent_root]
+            el_block_hash = current_block.body.execution_payload.block_hash
+
+    yield from add_block(spec, mega_store.fc_store, signed_block,
+                         valid=valid,
+                         test_steps=test_steps,
+                         is_optimistic=True)
+
+    # Update stores
+    is_optimistic_candidate = spec.is_optimistic_candidate_block(
+        mega_store.opt_store,
+        current_slot=spec.get_current_slot(mega_store.fc_store),
+        block=signed_block.message,
+    )
+    if is_optimistic_candidate:
+        mega_store.opt_store.optimistic_roots.add(block_root)
+        mega_store.opt_store.blocks[block_root] = signed_block.message.copy()
+        if not is_invalidated(mega_store, block_root):
+            mega_store.opt_store.block_states[block_root] = mega_store.fc_store.block_states[block_root].copy()
+
+    # Clean up the invalidated blocks
+    clean_up_store(mega_store)
+
+    # Update head
+    mega_store.opt_store.head_block_root = get_opt_head_block_root(spec, mega_store)
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_optimistic_head_output(mega_store),
+        }
+    })
+
+
+def get_opt_head_block_root(spec, mega_store):
+    """
+    Copied and modified from fork-choice spec `get_head` function.
+    """
+    store = mega_store.fc_store
+
+    # Get filtered block tree that only includes viable branches
+    blocks = spec.get_filtered_block_tree(store)
+    # Execute the LMD-GHOST fork choice
+    head = store.justified_checkpoint.root
+    while True:
+        children = [
+            root for root in blocks.keys()
+            if (
+                blocks[root].parent_root == head
+                and not is_invalidated(mega_store, root)  # For optimistic sync
+            )
+        ]
+        if len(children) == 0:
+            return head
+        # Sort by latest attesting balance with ties broken lexicographically
+        # Ties broken by favoring block with lexicographically higher root
+        head = max(children, key=lambda root: (spec.get_latest_attesting_balance(store, root), root))
+
+
+def is_invalidated(mega_store, block_root):
+    if block_root in mega_store.block_payload_statuses:
+        return mega_store.block_payload_statuses[block_root].status.alias == PayloadStatusV1StatusAlias.INVALIDATED
+    else:
+        return False
+
+
+def get_formatted_optimistic_head_output(mega_store):
+    head = mega_store.opt_store.head_block_root
+    slot = mega_store.fc_store.blocks[head].slot
+    return {
+        'slot': int(slot),
+        'root': encode_hex(head),
+    }
+
+
+def clean_up_store(mega_store):
+    """
+    Remove invalidated blocks
+    """
+    # TODO
+    ...

tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py

@@ -233,3 +233,33 @@ def test_bad_merkle_proof(spec, state):
     sign_deposit_data(spec, deposit.data, privkeys[validator_index])
 
     yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False)
+
+
+@with_all_phases
+@spec_state_test
+def test_key_validate_invalid_subgroup(spec, state):
+    validator_index = len(state.validators)
+    amount = spec.MAX_EFFECTIVE_BALANCE
+
+    # All-zero pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception.
+    pubkey = b'\x00' * 48
+
+    deposit = prepare_state_and_deposit(spec, state, validator_index, amount, pubkey=pubkey, signed=True)
+
+    yield from run_deposit_processing(spec, state, deposit, validator_index)
+
+
+@with_all_phases
+@spec_state_test
+def test_key_validate_invalid_decompression(spec, state):
+    validator_index = len(state.validators)
+    amount = spec.MAX_EFFECTIVE_BALANCE
+
+    # `deserialization_fails_infinity_with_true_b_flag` BLS G1 deserialization test case.
+    # This pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception.
+    pubkey_hex = 'c01000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
+    pubkey = bytes.fromhex(pubkey_hex)
+
+    deposit = prepare_state_and_deposit(spec, state, validator_index, amount, pubkey=pubkey, signed=True)
+
+    yield from run_deposit_processing(spec, state, deposit, validator_index)

tests/core/pyspec/eth2spec/utils/bls.py

@@ -138,3 +138,8 @@ def pairing_check(values):
         * pairing(p_q_2[1], p_q_2[0], final_exponentiate=False)
     )
     return final_exponentiation == FQ12.one()
+
+
+@only_with_bls(alt_return=True)
+def KeyValidate(pubkey):
+    return py_ecc_bls.KeyValidate(pubkey)

tests/formats/bls/aggregate.md

@@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file:
 
 ```yaml
 input: List[BLS Signature] -- list of input BLS signatures
-output: BLS Signature -- expected output, single BLS signature or empty.
+output: BLS Signature -- expected output, single BLS signature or `null`.
 ```
 
 - `BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`.
-- No output value if the input is invalid.
+- output value is `null` if the input is invalid.
 
 All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`.
 

tests/formats/bls/eth_aggregate_pubkeys.md

@@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file:
 
 ```yaml
 input: List[BLS Pubkey] -- list of input BLS pubkeys
-output: BLSPubkey -- expected output, single BLS pubkeys or empty.
+output: BLSPubkey -- expected output, single BLS pubkeys or `null`.
 ```
 
 - `BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`.
-- No output value if the input is invalid.
+- output value is `null` if the input is invalid.
 
 ## Condition
 

tests/formats/bls/sign.md

@@ -10,7 +10,12 @@ The test data is declared in a `data.yaml` file:
 input:
   privkey: bytes32 -- the private key used for signing
   message: bytes32 -- input message to sign (a hash)
-output: BLS Signature -- expected output, single BLS signature or empty.
+output: BLS Signature -- expected output, single BLS signature or `null`.
 ```
 
-All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`.
+- All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`.
+- output value is `null` if the input is invalid.
+
+## Condition
+
+The `sign` handler should sign `message` with `privkey`, and the resulting signature should match the expected `output`.

tests/formats/fork_choice/README.md

@@ -69,7 +69,7 @@ The file is located in the same folder (see below).
 
 After this step, the `store` object may have been updated.
 
-#### `on_merge_block` execution
+#### `on_merge_block` execution step
 
 Adds `PowBlock` data which is required for executing `on_block(store, block)`.
 ```yaml
@@ -97,6 +97,30 @@ The file is located in the same folder (see below).
 
 After this step, the `store` object may have been updated.
 
+#### `on_payload_info` execution step
+
+Optional step for optimistic sync tests.
+
+```yaml
+{
+    block_hash: string,             -- Encoded 32-byte value of payload's block hash.
+    payload_status: {
+        status: string,             -- Enum, "VALID" | "INVALID" | "SYNCING" | "ACCEPTED" | "INVALID_BLOCK_HASH".
+        latest_valid_hash: string,    -- Encoded 32-byte value of the latest valid block hash, may be `null`.
+        validation_error: string,    -- Message providing additional details on the validation error, may be `null`.
+    }
+}
+```
+
+This step sets the [`payloadStatus`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#PayloadStatusV1)
+value that Execution Layer client mock returns in responses to the following Engine API calls:
+* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash`
+* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash`
+
+*Note:* Status of a payload must be *initialized* via `on_payload_info` before the corresponding `on_block` execution step.
+
+*Note:* Status of the same payload may be updated for several times throughout the test.
+
 #### Checks step
 
 The checks to verify the current status of `store`.

tests/formats/sync/README.md

@@ -0,0 +1,3 @@
+# Sync tests
+
+It re-uses the [fork choice test format](../fork_choice/README.md) to apply the test script.

tests/generators/sync/main.py

@@ -0,0 +1,14 @@
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.test.helpers.constants import BELLATRIX
+
+
+if __name__ == "__main__":
+    bellatrix_mods = {key: 'eth2spec.test.bellatrix.sync.test_' + key for key in [
+        'optimistic',
+    ]}
+
+    all_mods = {
+        BELLATRIX: bellatrix_mods,
+    }
+
+    run_state_test_generators(runner_name="sync", all_mods=all_mods)

tests/generators/sync/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]