Merge pull request #2746 from ethereum/vbuterin-patch-12

Simplify sync protocol and update to calculate optimistic heads

presets/mainnet/altair.yaml

@@ -22,3 +22,5 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256
 # ---------------------------------------------------------------
 # 1
 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1
+# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256)
+UPDATE_TIMEOUT: 8192

presets/minimal/altair.yaml

@@ -22,3 +22,5 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8
 # ---------------------------------------------------------------
 # 1
 MIN_SYNC_COMMITTEE_PARTICIPANTS: 1
+# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 8 * 8)
+UPDATE_TIMEOUT: 64

setup.py

@@ -683,6 +683,7 @@ ignored_dependencies = [
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
     'bytes', 'byte', 'ByteList', 'ByteVector',
     'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', 'Set',
+    'Optional',
 ]
 
 

specs/altair/sync-protocol.md

@@ -13,12 +13,14 @@
 - [Preset](#preset)
   - [Misc](#misc)
 - [Containers](#containers)
-  - [`LightClientSnapshot`](#lightclientsnapshot)
   - [`LightClientUpdate`](#lightclientupdate)
   - [`LightClientStore`](#lightclientstore)
 - [Helper functions](#helper-functions)
   - [`get_subtree_index`](#get_subtree_index)
+  - [`get_active_header`](#get_active_header)
+  - [`get_safety_threshold`](#get_safety_threshold)
 - [Light client state updates](#light-client-state-updates)
+    - [`process_slot_for_light_client_store`](#process_slot_for_light_client_store)
     - [`validate_light_client_update`](#validate_light_client_update)
     - [`apply_light_client_update`](#apply_light_client_update)
     - [`process_light_client_update`](#process_light_client_update)
@@ -47,38 +49,27 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain.
 
 ### Misc
 
-| Name | Value |
+| Name | Value | Notes |
-| - | - |
+| - | - | - |
-| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` |
+| `MIN_SYNC_COMMITTEE_PARTICIPANTS` | `1` | |
+| `UPDATE_TIMEOUT` | `SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD` | ~27.3 hours |
 
 ## Containers
 
-### `LightClientSnapshot`
-
-```python
-class LightClientSnapshot(Container):
-    # Beacon block header
-    header: BeaconBlockHeader
-    # Sync committees corresponding to the header
-    current_sync_committee: SyncCommittee
-    next_sync_committee: SyncCommittee
-```
-
 ### `LightClientUpdate`
 
 ```python
 class LightClientUpdate(Container):
-    # Update beacon block header
+    # The beacon block header that is attested to by the sync committee
-    header: BeaconBlockHeader
+    attested_header: BeaconBlockHeader
-    # Next sync committee corresponding to the header
+    # Next sync committee corresponding to the active header
     next_sync_committee: SyncCommittee
     next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)]
-    # Finality proof for the update header
+    # The finalized beacon block header attested to by Merkle branch
-    finality_header: BeaconBlockHeader
+    finalized_header: BeaconBlockHeader
     finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)]
     # Sync committee aggregate signature
-    sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE]
+    sync_committee_aggregate: SyncAggregate
-    sync_committee_signature: BLSSignature
     # Fork version for the aggregate signature
     fork_version: Version
 ```
@@ -88,8 +79,18 @@ class LightClientUpdate(Container):
 ```python
 @dataclass
 class LightClientStore(object):
-    snapshot: LightClientSnapshot
+    # Beacon block header that is finalized
-    valid_updates: Set[LightClientUpdate]
+    finalized_header: BeaconBlockHeader
+    # Sync committees corresponding to the header
+    current_sync_committee: SyncCommittee
+    next_sync_committee: SyncCommittee
+    # Best available header to switch finalized head to if we see nothing else
+    best_valid_update: Optional[LightClientUpdate]
+    # Most recent available reasonably-safe header
+    optimistic_header: BeaconBlockHeader
+    # Max number of active participants in a sync committee (used to calculate safety threshold)
+    previous_max_active_participants: uint64
+    current_max_active_participants: uint64
 ```
 
 ## Helper functions
@@ -101,95 +102,157 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64:
     return uint64(generalized_index % 2**(floorlog2(generalized_index)))
 ```
 
+### `get_active_header`
+
+```python
+def get_active_header(update: LightClientUpdate) -> BeaconBlockHeader:
+    # The "active header" is the header that the update is trying to convince us
+    # to accept. If a finalized header is present, it's the finalized header,
+    # otherwise it's the attested header
+    if update.finalized_header != BeaconBlockHeader():
+        return update.finalized_header
+    else:
+        return update.attested_header
+```
+
+### `get_safety_threshold`
+
+```python
+def get_safety_threshold(store: LightClientStore) -> uint64:
+    return max(
+        store.previous_max_active_participants,
+        store.current_max_active_participants,
+    ) // 2
+```
+
 ## Light client state updates
 
-A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock.
+A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. `process_slot_for_light_client_store` is processed every time the current slot increments.
+
+#### `process_slot_for_light_client_store`
+
+```python
+def process_slot_for_light_client_store(store: LightClientStore, current_slot: Slot) -> None:
+    if current_slot % UPDATE_TIMEOUT == 0:
+        store.previous_max_active_participants = store.current_max_active_participants
+        store.current_max_active_participants = 0
+    if (
+        current_slot > store.finalized_header.slot + UPDATE_TIMEOUT
+        and store.best_valid_update is not None
+    ):
+        # Forced best update when the update timeout has elapsed
+        apply_light_client_update(store, store.best_valid_update)
+        store.best_valid_update = None
+```
 
 #### `validate_light_client_update`
 
 ```python
-def validate_light_client_update(snapshot: LightClientSnapshot,
+def validate_light_client_update(store: LightClientStore,
                                  update: LightClientUpdate,
+                                 current_slot: Slot,
                                  genesis_validators_root: Root) -> None:
-    # Verify update slot is larger than snapshot slot
+    # Verify update slot is larger than slot of current best finalized header
-    assert update.header.slot > snapshot.header.slot
+    active_header = get_active_header(update)
+    assert current_slot >= active_header.slot > store.finalized_header.slot
 
     # Verify update does not skip a sync committee period
-    snapshot_period = compute_epoch_at_slot(snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    finalized_period = compute_epoch_at_slot(store.finalized_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
-    update_period = compute_epoch_at_slot(update.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    update_period = compute_epoch_at_slot(active_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
-    assert update_period in (snapshot_period, snapshot_period + 1)
+    assert update_period in (finalized_period, finalized_period + 1)
 
-    # Verify update header root is the finalized root of the finality header, if specified
+    # Verify that the `finalized_header`, if present, actually is the finalized header saved in the
-    if update.finality_header == BeaconBlockHeader():
+    # state of the `attested header`
-        signed_header = update.header
+    if update.finalized_header == BeaconBlockHeader():
         assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))]
     else:
-        signed_header = update.finality_header
         assert is_valid_merkle_branch(
-            leaf=hash_tree_root(update.header),
+            leaf=hash_tree_root(update.finalized_header),
             branch=update.finality_branch,
             depth=floorlog2(FINALIZED_ROOT_INDEX),
             index=get_subtree_index(FINALIZED_ROOT_INDEX),
-            root=update.finality_header.state_root,
+            root=update.attested_header.state_root,
         )
 
     # Verify update next sync committee if the update period incremented
-    if update_period == snapshot_period:
+    if update_period == finalized_period:
-        sync_committee = snapshot.current_sync_committee
+        sync_committee = store.current_sync_committee
         assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))]
     else:
-        sync_committee = snapshot.next_sync_committee
+        sync_committee = store.next_sync_committee
         assert is_valid_merkle_branch(
             leaf=hash_tree_root(update.next_sync_committee),
             branch=update.next_sync_committee_branch,
             depth=floorlog2(NEXT_SYNC_COMMITTEE_INDEX),
             index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX),
-            root=update.header.state_root,
+            root=active_header.state_root,
         )
+        
+    sync_aggregate = update.sync_committee_aggregate
 
     # Verify sync committee has sufficient participants
-    assert sum(update.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS
+    assert sum(sync_aggregate.sync_committee_bits) >= MIN_SYNC_COMMITTEE_PARTICIPANTS
 
     # Verify sync committee aggregate signature
-    participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit]
+    participant_pubkeys = [
+        pubkey for (bit, pubkey) in zip(sync_aggregate.sync_committee_bits, sync_committee.pubkeys)
+        if bit
+    ]
     domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version, genesis_validators_root)
-    signing_root = compute_signing_root(signed_header, domain)
+    signing_root = compute_signing_root(update.attested_header, domain)
-    assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature)
+    assert bls.FastAggregateVerify(participant_pubkeys, signing_root, sync_aggregate.sync_committee_signature)
 ```
 
 #### `apply_light_client_update`
 
 ```python
-def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None:
+def apply_light_client_update(store: LightClientStore, update: LightClientUpdate) -> None:
-    snapshot_period = compute_epoch_at_slot(snapshot.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    active_header = get_active_header(update)
-    update_period = compute_epoch_at_slot(update.header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    finalized_period = compute_epoch_at_slot(store.finalized_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
-    if update_period == snapshot_period + 1:
+    update_period = compute_epoch_at_slot(active_header.slot) // EPOCHS_PER_SYNC_COMMITTEE_PERIOD
-        snapshot.current_sync_committee = snapshot.next_sync_committee
+    if update_period == finalized_period + 1:
-        snapshot.next_sync_committee = update.next_sync_committee
+        store.current_sync_committee = store.next_sync_committee
-    snapshot.header = update.header
+        store.next_sync_committee = update.next_sync_committee
+    store.finalized_header = active_header
 ```
 
 #### `process_light_client_update`
 
 ```python
-def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot,
+def process_light_client_update(store: LightClientStore,
+                                update: LightClientUpdate,
+                                current_slot: Slot,
                                 genesis_validators_root: Root) -> None:
-    validate_light_client_update(store.snapshot, update, genesis_validators_root)
+    validate_light_client_update(store, update, current_slot, genesis_validators_root)
-    store.valid_updates.add(update)
 
-    update_timeout = SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    sync_committee_bits = update.sync_committee_aggregate.sync_committee_bits
+
+    # Update the best update in case we have to force-update to it if the timeout elapses
     if (
-        sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2
+        store.best_valid_update is None
-        and update.finality_header != BeaconBlockHeader()
+        or sum(sync_committee_bits) > sum(store.best_valid_update.sync_committee_aggregate.sync_committee_bits)
     ):
-        # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof.
+        store.best_valid_update = update
-        # Note that (2) means that the current light client design needs finality.
+
-        # It may be changed to re-organizable light client design. See the on-going issue consensus-specs#2182.
+    # Track the maximum number of active participants in the committee signatures
-        apply_light_client_update(store.snapshot, update)
+    store.current_max_active_participants = max(
-        store.valid_updates = set()
+        store.current_max_active_participants,
-    elif current_slot > store.snapshot.header.slot + update_timeout:
+        sum(sync_committee_bits),
-        # Forced best update when the update timeout has elapsed
+    )
-        apply_light_client_update(store.snapshot,
+
-                                  max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits)))
+    # Update the optimistic header
-        store.valid_updates = set()
+    if (
+        sum(sync_committee_bits) > get_safety_threshold(store)
+        and update.attested_header.slot > store.optimistic_header.slot
+    ):
+        store.optimistic_header = update.attested_header
+
+    # Update finalized header
+    if (
+        sum(sync_committee_bits) * 3 >= len(sync_committee_bits) * 2
+        and update.finalized_header != BeaconBlockHeader()
+    ):
+        # Normal update through 2/3 threshold
+        apply_light_client_update(store, update)
+        store.best_valid_update = None
 ```

tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py

@@ -1,3 +1,5 @@
+from copy import deepcopy
+
 from eth2spec.test.context import (
     spec_state_test,
     with_presets,
@@ -19,20 +21,24 @@ from eth2spec.test.helpers.sync_committee import (
 from eth2spec.test.helpers.merkle import build_proof
 
 
-@with_altair_and_later
+def _initialize_light_client_store(spec, state):
-@spec_state_test
+    return spec.LightClientStore(
-def test_process_light_client_update_not_updated(spec, state):
+        finalized_header=spec.BeaconBlockHeader(),
-    pre_snapshot = spec.LightClientSnapshot(
-        header=spec.BeaconBlockHeader(),
         current_sync_committee=state.current_sync_committee,
         next_sync_committee=state.next_sync_committee,
-    )
+        best_valid_update=None,
-    store = spec.LightClientStore(
+        optimistic_header=spec.BeaconBlockHeader(),
-        snapshot=pre_snapshot,
+        previous_max_active_participants=0,
-        valid_updates=set(),
+        current_max_active_participants=0,
     )
 
-    # Block at slot 1 doesn't increase sync committee period, so it won't update snapshot
+
+@with_altair_and_later
+@spec_state_test
+def test_process_light_client_update_not_timeout(spec, state):
+    store = _initialize_light_client_store(spec, state)
+
+    # Block at slot 1 doesn't increase sync committee period, so it won't force update store.finalized_header
     block = build_empty_block_for_next_slot(spec, state)
     signed_block = state_transition_and_sign_block(spec, state, block)
     block_header = spec.BeaconBlockHeader(
@@ -52,6 +58,10 @@ def test_process_light_client_update_not_updated(spec, state):
         block_header.slot,
         committee,
     )
+    sync_committee_aggregate = spec.SyncAggregate(
+        sync_committee_bits=sync_committee_bits,
+        sync_committee_signature=sync_committee_signature,
+    )
     next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))]
 
     # Ensure that finality checkpoint is genesis
@@ -61,40 +71,34 @@ def test_process_light_client_update_not_updated(spec, state):
     finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))]
 
     update = spec.LightClientUpdate(
-        header=block_header,
+        attested_header=block_header,
         next_sync_committee=state.next_sync_committee,
         next_sync_committee_branch=next_sync_committee_branch,
-        finality_header=finality_header,
+        finalized_header=finality_header,
         finality_branch=finality_branch,
-        sync_committee_bits=sync_committee_bits,
+        sync_committee_aggregate=sync_committee_aggregate,
-        sync_committee_signature=sync_committee_signature,
         fork_version=state.fork.current_version,
     )
 
+    pre_store = deepcopy(store)
+
     spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root)
 
-    assert len(store.valid_updates) == 1
+    assert store.current_max_active_participants > 0
-    assert store.valid_updates.pop() == update
+    assert store.optimistic_header == update.attested_header
-    assert store.snapshot == pre_snapshot
+    assert store.finalized_header == pre_store.finalized_header
+    assert store.best_valid_update == update
 
 
 @with_altair_and_later
 @spec_state_test
 @with_presets([MINIMAL], reason="too slow")
 def test_process_light_client_update_timeout(spec, state):
-    pre_snapshot = spec.LightClientSnapshot(
+    store = _initialize_light_client_store(spec, state)
-        header=spec.BeaconBlockHeader(),
-        current_sync_committee=state.current_sync_committee,
-        next_sync_committee=state.next_sync_committee,
-    )
-    store = spec.LightClientStore(
-        snapshot=pre_snapshot,
-        valid_updates=set(),
-    )
 
     # Forward to next sync committee period
-    next_slots(spec, state, spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD))
+    next_slots(spec, state, spec.UPDATE_TIMEOUT)
-    snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    snapshot_period = spec.compute_epoch_at_slot(store.optimistic_header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
     update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
     assert snapshot_period + 1 == update_period
 
@@ -119,6 +123,10 @@ def test_process_light_client_update_timeout(spec, state):
         committee,
         block_root=spec.Root(block_header.hash_tree_root()),
     )
+    sync_committee_aggregate = spec.SyncAggregate(
+        sync_committee_bits=sync_committee_bits,
+        sync_committee_signature=sync_committee_signature,
+    )
 
     # Sync committee is updated
     next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX)
@@ -127,36 +135,30 @@ def test_process_light_client_update_timeout(spec, state):
     finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))]
 
     update = spec.LightClientUpdate(
-        header=block_header,
+        attested_header=block_header,
         next_sync_committee=state.next_sync_committee,
         next_sync_committee_branch=next_sync_committee_branch,
-        finality_header=finality_header,
+        finalized_header=finality_header,
         finality_branch=finality_branch,
-        sync_committee_bits=sync_committee_bits,
+        sync_committee_aggregate=sync_committee_aggregate,
-        sync_committee_signature=sync_committee_signature,
         fork_version=state.fork.current_version,
     )
 
+    pre_store = deepcopy(store)
+
     spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root)
 
-    # snapshot has been updated
+    assert store.current_max_active_participants > 0
-    assert len(store.valid_updates) == 0
+    assert store.optimistic_header == update.attested_header
-    assert store.snapshot.header == update.header
+    assert store.best_valid_update == update
+    assert store.finalized_header == pre_store.finalized_header
 
 
 @with_altair_and_later
 @spec_state_test
 @with_presets([MINIMAL], reason="too slow")
 def test_process_light_client_update_finality_updated(spec, state):
-    pre_snapshot = spec.LightClientSnapshot(
+    store = _initialize_light_client_store(spec, state)
-        header=spec.BeaconBlockHeader(),
-        current_sync_committee=state.current_sync_committee,
-        next_sync_committee=state.next_sync_committee,
-    )
-    store = spec.LightClientStore(
-        snapshot=pre_snapshot,
-        valid_updates=set(),
-    )
 
     # Change finality
     blocks = []
@@ -167,7 +169,7 @@ def test_process_light_client_update_finality_updated(spec, state):
     # Ensure that finality checkpoint has changed
     assert state.finalized_checkpoint.epoch == 3
     # Ensure that it's same period
-    snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+    snapshot_period = spec.compute_epoch_at_slot(store.optimistic_header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
     update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD
     assert snapshot_period == update_period
 
@@ -199,20 +201,24 @@ def test_process_light_client_update_finality_updated(spec, state):
         committee,
         block_root=spec.Root(block_header.hash_tree_root()),
     )
-
+    sync_committee_aggregate = spec.SyncAggregate(
-    update = spec.LightClientUpdate(
-        header=finalized_block_header,
-        next_sync_committee=state.next_sync_committee,
-        next_sync_committee_branch=next_sync_committee_branch,
-        finality_header=block_header,  # block_header is the signed header
-        finality_branch=finality_branch,
         sync_committee_bits=sync_committee_bits,
         sync_committee_signature=sync_committee_signature,
+    )
+
+    update = spec.LightClientUpdate(
+        attested_header=block_header,
+        next_sync_committee=state.next_sync_committee,
+        next_sync_committee_branch=next_sync_committee_branch,
+        finalized_header=finalized_block_header,
+        finality_branch=finality_branch,
+        sync_committee_aggregate=sync_committee_aggregate,
         fork_version=state.fork.current_version,
     )
 
     spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root)
 
-    # snapshot has been updated
+    assert store.current_max_active_participants > 0
-    assert len(store.valid_updates) == 0
+    assert store.optimistic_header == update.attested_header
-    assert store.snapshot.header == update.header
+    assert store.finalized_header == update.finalized_header
+    assert store.best_valid_update is None