From bcde37c39ff551a0e5d71af9a379670c0853934c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 9 Dec 2020 17:35:22 +0800 Subject: [PATCH 01/13] Make `sync-protocol.md` pass the linter --- setup.py | 73 ++++++++++++++++--- specs/lightclient/sync-protocol.md | 27 ++++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index bd043dccc..168e19cca 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] constants: Dict[str, str] + ssz_dep_constants: Dict[str, str] # the constants that depend on ssz_objects ssz_objects: Dict[str, str] dataclasses: Dict[str, str] @@ -35,6 +36,7 @@ def get_spec(file_name: str) -> SpecObject: current_name = None # most recent section title functions: Dict[str, str] = {} constants: Dict[str, str] = {} + ssz_dep_constants: Dict[str, str] = {} ssz_objects: Dict[str, str] = {} dataclasses: Dict[str, str] = {} function_matcher = re.compile(FUNCTION_REGEX) @@ -88,10 +90,20 @@ def get_spec(file_name: str) -> SpecObject: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': is_constant_def = False if is_constant_def: - constants[row[0]] = row[1].replace('**TBD**', '2**32') + if row[1].startswith('get_generalized_index'): + ssz_dep_constants[row[0]] = row[1] + else: + constants[row[0]] = row[1].replace('**TBD**', '2**32') elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) CONFIG_LOADER = ''' @@ -160,7 +172,7 @@ CONFIG_NAME = 'mainnet' LIGHTCLIENT_IMPORT = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union ) from dataclasses import ( @@ -174,6 +186,7 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Path, ) from eth2spec.utils import bls @@ -277,6 +290,24 @@ get_start_shard = cache_this( _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' +LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS = ''' + +def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: + ssz_path = Path(ssz_class) + for item in path: + ssz_path = ssz_path / item + return GeneralizedIndex(ssz_path.gindex()) +''' + + +# The constants that depend on SSZ objects +# Will verify the value at the end of the spec +LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(54)', +} + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -303,14 +334,30 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl + '\n\n' + f"fork = \'{fork}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - + '\n\n' + constants_spec + ) + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += ( + LIGHTCLIENT_PATCH_SUNDRY_FUNCTIONS + + '\n\n' + lightclient_patch_ssz_dep_constants + ) + + spec += ( + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + + if fork == 'lightclient_patch': + lightclient_patch_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), LIGHTCLIENT_PATCH_HARDCODED_SSZ_DEP_CONSTANTS)) + spec += '\n\n' + lightclient_patch_ssz_dep_constants_verification + spec += '\n' return spec @@ -332,7 +379,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', + 'Dict', 'dict', 'field', 'ceillog2', ] @@ -373,14 +420,22 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: """ Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. """ - functions0, custom_types0, constants0, ssz_objects0, dataclasses0 = spec0 - functions1, custom_types1, constants1, ssz_objects1, dataclasses1 = spec1 + functions0, custom_types0, constants0, ssz_dep_constants0, ssz_objects0, dataclasses0 = spec0 + functions1, custom_types1, constants1, ssz_dep_constants1, ssz_objects1, dataclasses1 = spec1 functions = combine_functions(functions0, functions1) custom_types = combine_constants(custom_types0, custom_types1) constants = combine_constants(constants0, constants1) + ssz_dep_constants = combine_constants(ssz_dep_constants0, ssz_dep_constants1) ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types) dataclasses = combine_functions(dataclasses0, dataclasses1) - return SpecObject(functions, custom_types, constants, ssz_objects, dataclasses) + return SpecObject( + functions=functions, + custom_types=custom_types, + constants=constants, + ssz_dep_constants=ssz_dep_constants, + ssz_objects=ssz_objects, + dataclasses=dataclasses, + ) fork_imports = { @@ -461,8 +516,8 @@ class PySpecCommand(Command): specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md specs/lightclient/lightclient-fork.md + specs/lightclient/sync-protocol.md """ - # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 310aad2df..a8dc59639 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -39,8 +39,8 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. | Name | Value | | - | - | -| `FINALIZED_ROOT_INDEX` | `Index(BeaconState, 'finalized_checkpoint', 'root')` | -| `NEXT_SYNC_COMMITTEE_INDEX` | `Index(BeaconState, 'next_sync_committee')` | +| `FINALIZED_ROOT_INDEX` | `get_generalized_index(BeaconState, 'finalized_checkpoint', 'root')` | +| `NEXT_SYNC_COMMITTEE_INDEX` | `get_generalized_index(BeaconState, 'next_sync_committee')` | ## Configuration @@ -78,10 +78,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, log2(NEXT_SYNC_COMMITTEE_INDEX)] + next_sync_committee_branch: Vector[Bytes32, ceillog2(NEXT_SYNC_COMMITTEE_INDEX)] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, log2(FINALIZED_ROOT_INDEX)] + finality_branch: Vector[Bytes32, ceillog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -116,28 +116,28 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [ZERO_HASH for _ in range(log2(FINALIZED_ROOT_INDEX))] + assert update.finality_branch == [Bytes32() for _ in range(ceillog2(FINALIZED_ROOT_INDEX))] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=log2(FINALIZED_ROOT_INDEX), - index=FINALIZED_ROOT_INDEX % 2**log2(FINALIZED_ROOT_INDEX), + depth=ceillog2(FINALIZED_ROOT_INDEX), + index=FINALIZED_ROOT_INDEX % 2**ceillog2(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [ZERO_HASH for _ in range(log2(NEXT_SYNC_COMMITTEE_INDEX))] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(ceillog2(NEXT_SYNC_COMMITTEE_INDEX))] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=log2(NEXT_SYNC_COMMITTEE_INDEX), - index=NEXT_SYNC_COMMITTEE_INDEX % 2**log2(NEXT_SYNC_COMMITTEE_INDEX), + depth=ceillog2(NEXT_SYNC_COMMITTEE_INDEX), + index=NEXT_SYNC_COMMITTEE_INDEX % 2**ceillog2(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) @@ -173,11 +173,14 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda assert is_valid_light_client_update(store.snapshot, update) store.valid_updates.append(update) - if sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 and update.header != update.finality_header: + if ( + sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + and update.header != update.finality_header + ): # Apply update if 2/3 quorum is reached and we have a finality proof apply_light_client_update(store, update) store.valid_updates = [] - elif current_slot > snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: + elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py index b3a0b9962..6626e26d6 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -5,4 +5,4 @@ from remerkleable.complex import Container, Vector, List from remerkleable.basic import boolean, bit, uint, byte, uint8, uint16, uint32, uint64, uint128, uint256 from remerkleable.bitfields import Bitvector, Bitlist from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList -from remerkleable.core import BasicView, View +from remerkleable.core import BasicView, View, Path From cf6933ac457614f78052399e9d16dff1bc251640 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Dec 2020 17:04:11 +0800 Subject: [PATCH 02/13] Fix depth calculation and add `get_subtree_index` helper --- specs/lightclient/sync-protocol.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index a8dc59639..073d570cb 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -17,6 +17,8 @@ - [`LightClientSnapshot`](#lightclientsnapshot) - [`LightClientUpdate`](#lightclientupdate) - [`LightClientStore`](#lightclientstore) +- [Helper functions](#helper-functions) + - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) - [`is_valid_light_client_update`](#is_valid_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) @@ -78,10 +80,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, ceillog2(NEXT_SYNC_COMMITTEE_INDEX)] + next_sync_committee_branch: Vector[Bytes32, NEXT_SYNC_COMMITTEE_INDEX.bit_length()] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, ceillog2(FINALIZED_ROOT_INDEX)] + finality_branch: Vector[Bytes32, FINALIZED_ROOT_INDEX.bit_length()] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -97,6 +99,15 @@ class LightClientStore(Container): valid_updates: List[LightClientUpdate, MAX_VALID_LIGHT_CLIENT_UPDATES] ``` +## Helper functions + +### `get_subtree_index` + +```python +def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: + return uint64(generalized_index % 2**((generalized_index).bit_length())) +``` + ## Light client state updates A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. @@ -116,28 +127,28 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [Bytes32() for _ in range(ceillog2(FINALIZED_ROOT_INDEX))] + assert update.finality_branch == [Bytes32() for _ in range(FINALIZED_ROOT_INDEX.bit_length())] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=ceillog2(FINALIZED_ROOT_INDEX), - index=FINALIZED_ROOT_INDEX % 2**ceillog2(FINALIZED_ROOT_INDEX), + depth=FINALIZED_ROOT_INDEX.bit_length(), + index=get_subtree_index(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [Bytes32() for _ in range(ceillog2(NEXT_SYNC_COMMITTEE_INDEX))] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(NEXT_SYNC_COMMITTEE_INDEX.bit_length())] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=ceillog2(NEXT_SYNC_COMMITTEE_INDEX), - index=NEXT_SYNC_COMMITTEE_INDEX % 2**ceillog2(NEXT_SYNC_COMMITTEE_INDEX), + depth=NEXT_SYNC_COMMITTEE_INDEX.bit_length(), + index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) From d01a4ad82374dbfbabf047315536f90fe90bf928 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 16 Dec 2020 15:00:06 +0800 Subject: [PATCH 03/13] Fix depth calculation...again(!) and add unittests --- setup.py | 10 ++++-- specs/lightclient/sync-protocol.md | 14 ++++---- .../pyspec/eth2spec/test/helpers/custody.py | 22 +----------- .../pyspec/eth2spec/test/helpers/merkle.py | 21 +++++++++++ .../unittests/test_helpers.py | 35 +++++++++++++++++++ 5 files changed, 72 insertions(+), 30 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/merkle.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py diff --git a/setup.py b/setup.py index 168e19cca..8115fdfe5 100644 --- a/setup.py +++ b/setup.py @@ -209,6 +209,12 @@ def ceillog2(x: int) -> uint64: if x < 1: raise ValueError(f"ceillog2 accepts only positive values, x={x}") return uint64((x - 1).bit_length()) + + +def floorlog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"floorlog2 accepts only positive values, x={x}") + return uint64(x.bit_length() - 1) ''' PHASE0_SUNDRY_FUNCTIONS = ''' def get_eth1_data(block: Eth1Block) -> Eth1Data: @@ -321,7 +327,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl ) ) for k in list(spec_object.functions): - if "ceillog2" in k: + if "ceillog2" in k or "floorlog2" in k: del spec_object.functions[k] functions_spec = '\n\n'.join(spec_object.functions.values()) for k in list(spec_object.constants.keys()): @@ -379,7 +385,7 @@ ignored_dependencies = [ 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes', 'byte', 'ByteList', 'ByteVector', - 'Dict', 'dict', 'field', 'ceillog2', + 'Dict', 'dict', 'field', 'ceillog2', 'floorlog2', ] diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 073d570cb..156e6e78e 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -80,10 +80,10 @@ class LightClientUpdate(Container): header: BeaconBlockHeader # Next sync committee corresponding to the header next_sync_committee: SyncCommittee - next_sync_committee_branch: Vector[Bytes32, NEXT_SYNC_COMMITTEE_INDEX.bit_length()] + next_sync_committee_branch: Vector[Bytes32, floorlog2(NEXT_SYNC_COMMITTEE_INDEX)] # Finality proof for the update header finality_header: BeaconBlockHeader - finality_branch: Vector[Bytes32, FINALIZED_ROOT_INDEX.bit_length()] + finality_branch: Vector[Bytes32, floorlog2(FINALIZED_ROOT_INDEX)] # Sync committee aggregate signature sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] sync_committee_signature: BLSSignature @@ -105,7 +105,7 @@ class LightClientStore(Container): ```python def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: - return uint64(generalized_index % 2**((generalized_index).bit_length())) + return uint64(generalized_index % 2**(floorlog2(generalized_index))) ``` ## Light client state updates @@ -127,13 +127,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update header root is the finalized root of the finality header, if specified if update.finality_header == BeaconBlockHeader(): signed_header = update.header - assert update.finality_branch == [Bytes32() for _ in range(FINALIZED_ROOT_INDEX.bit_length())] + assert update.finality_branch == [Bytes32() for _ in range(floorlog2(FINALIZED_ROOT_INDEX))] else: signed_header = update.finality_header assert is_valid_merkle_branch( leaf=hash_tree_root(update.header), branch=update.finality_branch, - depth=FINALIZED_ROOT_INDEX.bit_length(), + depth=floorlog2(FINALIZED_ROOT_INDEX), index=get_subtree_index(FINALIZED_ROOT_INDEX), root=update.finality_header.state_root, ) @@ -141,13 +141,13 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify update next sync committee if the update period incremented if update_period == snapshot_period: sync_committee = snapshot.current_sync_committee - assert update.next_sync_committee_branch == [Bytes32() for _ in range(NEXT_SYNC_COMMITTEE_INDEX.bit_length())] + assert update.next_sync_committee_branch == [Bytes32() for _ in range(floorlog2(NEXT_SYNC_COMMITTEE_INDEX))] else: sync_committee = snapshot.next_sync_committee assert is_valid_merkle_branch( leaf=hash_tree_root(update.next_sync_committee), branch=update.next_sync_committee_branch, - depth=NEXT_SYNC_COMMITTEE_INDEX.bit_length(), + depth=floorlog2(NEXT_SYNC_COMMITTEE_INDEX), index=get_subtree_index(NEXT_SYNC_COMMITTEE_INDEX), root=update.header.state_root, ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index b3a8c0a95..8e9aafa66 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.merkle import build_proof from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, ByteList -from remerkleable.tree import gindex_bit_iter BYTES_PER_CHUNK = 32 @@ -116,26 +116,6 @@ def custody_chunkify(spec, x): return [ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](c) for c in chunks] -def build_proof(anchor, leaf_index): - if leaf_index <= 1: - return [] # Nothing to prove / invalid index - node = anchor - proof = [] - # Walk down, top to bottom to the leaf - bit_iter, _ = gindex_bit_iter(leaf_index) - for bit in bit_iter: - # Always take the opposite hand for the proof. - # 1 = right as leaf, thus get left - if bit: - proof.append(node.get_left().merkle_root()) - node = node.get_right() - else: - proof.append(node.get_right().merkle_root()) - node = node.get_left() - - return list(reversed(proof)) - - def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_index, block_length_or_custody_data, invalid_chunk_data=False): diff --git a/tests/core/pyspec/eth2spec/test/helpers/merkle.py b/tests/core/pyspec/eth2spec/test/helpers/merkle.py new file mode 100644 index 000000000..d49827954 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/merkle.py @@ -0,0 +1,21 @@ +from remerkleable.tree import gindex_bit_iter + + +def build_proof(anchor, leaf_index): + if leaf_index <= 1: + return [] # Nothing to prove / invalid index + node = anchor + proof = [] + # Walk down, top to bottom to the leaf + bit_iter, _ = gindex_bit_iter(leaf_index) + for bit in bit_iter: + # Always take the opposite hand for the proof. + # 1 = right as leaf, thus get left + if bit: + proof.append(node.get_left().merkle_root()) + node = node.get_right() + else: + proof.append(node.get_right().merkle_root()) + node = node.get_left() + + return list(reversed(proof)) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py new file mode 100644 index 000000000..33e749380 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py @@ -0,0 +1,35 @@ +from eth2spec.test.context import ( + spec_state_test, + with_phases, + LIGHTCLIENT_PATCH, +) +from eth2spec.test.helpers.merkle import build_proof + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_next_sync_committee_tree(spec, state): + state.next_sync_committee = spec.SyncCommittee( + pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)] + ) + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + assert spec.is_valid_merkle_branch( + leaf=state.next_sync_committee.hash_tree_root(), + branch=next_sync_committee_branch, + depth=spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX), + index=spec.get_subtree_index(spec.NEXT_SYNC_COMMITTEE_INDEX), + root=state.hash_tree_root(), + ) + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_finality_root_tree(spec, state): + finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + assert spec.is_valid_merkle_branch( + leaf=state.finalized_checkpoint.root, + branch=finality_branch, + depth=spec.floorlog2(spec.FINALIZED_ROOT_INDEX), + index=spec.get_subtree_index(spec.FINALIZED_ROOT_INDEX), + root=state.hash_tree_root(), + ) From ce879143266f9f60a1bbb65e6c96adc6f6d7e7c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 7 Jan 2021 11:14:32 +0800 Subject: [PATCH 04/13] Fix the 2/3 threshold calculation Co-authored-by: Danny Ryan --- specs/lightclient/sync-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 156e6e78e..562b7b5bb 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -185,7 +185,7 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda store.valid_updates.append(update) if ( - sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 and update.header != update.finality_header ): # Apply update if 2/3 quorum is reached and we have a finality proof From fb0c6d54f87eba3d9454484cd4ae14e5397d0bb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:12:36 +0800 Subject: [PATCH 05/13] Add @ralexstokes's fix --- specs/lightclient/sync-protocol.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 562b7b5bb..b8d443fb2 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -185,10 +185,12 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda store.valid_updates.append(update) if ( - sum(update.sync_committee_bits) * 3 >= len(update.sync_committee_bits) * 2 - and update.header != update.finality_header + sum(update.sync_committee_bits) * 3 > len(update.sync_committee_bits) * 2 + and update.finality_header != BeaconBlockHeader() ): - # Apply update if 2/3 quorum is reached and we have a finality proof + # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof. + # Note that (2) means that the current light client design needs finality. + # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. apply_light_client_update(store, update) store.valid_updates = [] elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: From f3d7dee71f15622f0dd258c25046e31bbdbb9510 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:44:21 +0800 Subject: [PATCH 06/13] Apply @djrtwo's suggestion --- specs/lightclient/sync-protocol.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index b8d443fb2..7d442897e 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -20,7 +20,7 @@ - [Helper functions](#helper-functions) - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) - - [`is_valid_light_client_update`](#is_valid_light_client_update) + - [`validate_light_client_update`](#validate_light_client_update) - [`apply_light_client_update`](#apply_light_client_update) - [`process_light_client_update`](#process_light_client_update) @@ -112,10 +112,10 @@ def get_subtree_index(generalized_index: GeneralizedIndex) -> uint64: A light client maintains its state in a `store` object of type `LightClientStore` and receives `update` objects of type `LightClientUpdate`. Every `update` triggers `process_light_client_update(store, update, current_slot)` where `current_slot` is the current slot based on some local clock. -#### `is_valid_light_client_update` +#### `validate_light_client_update` ```python -def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> bool: +def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None: # Verify update slot is larger than snapshot slot assert update.header.slot > snapshot.header.slot @@ -160,8 +160,6 @@ def is_valid_light_client_update(snapshot: LightClientSnapshot, update: LightCli domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) signing_root = compute_signing_root(signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) - - return True ``` #### `apply_light_client_update` @@ -180,8 +178,7 @@ def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClient ```python def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: - # Validate update - assert is_valid_light_client_update(store.snapshot, update) + validate_light_client_update(store.snapshot, update) store.valid_updates.append(update) if ( From 86fe6bc0943a7fdb29645941637df086a35b5b58 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 14:27:23 +0800 Subject: [PATCH 07/13] Fix apply_light_client_update calls --- specs/lightclient/sync-protocol.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 7d442897e..306fca3b2 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -14,9 +14,9 @@ - [Misc](#misc) - [Time parameters](#time-parameters) - [Containers](#containers) - - [`LightClientSnapshot`](#lightclientsnapshot) - - [`LightClientUpdate`](#lightclientupdate) - - [`LightClientStore`](#lightclientstore) + - [`LightClientSnapshot`](#lightclientsnapshot) + - [`LightClientUpdate`](#lightclientupdate) + - [`LightClientStore`](#lightclientstore) - [Helper functions](#helper-functions) - [`get_subtree_index`](#get_subtree_index) - [Light client state updates](#light-client-state-updates) @@ -61,7 +61,7 @@ uses sync committees introduced in [this beacon chain extension](./beacon-chain. ## Containers -#### `LightClientSnapshot` +### `LightClientSnapshot` ```python class LightClientSnapshot(Container): @@ -72,7 +72,7 @@ class LightClientSnapshot(Container): next_sync_committee: SyncCommittee ``` -#### `LightClientUpdate` +### `LightClientUpdate` ```python class LightClientUpdate(Container): @@ -91,7 +91,7 @@ class LightClientUpdate(Container): fork_version: Version ``` -#### `LightClientStore` +### `LightClientStore` ```python class LightClientStore(Container): @@ -188,10 +188,11 @@ def process_light_client_update(store: LightClientStore, update: LightClientUpda # Apply update if (1) 2/3 quorum is reached and (2) we have a finality proof. # Note that (2) means that the current light client design needs finality. # It may be changed to re-organizable light client design. See the on-going issue eth2.0-specs#2182. - apply_light_client_update(store, update) + apply_light_client_update(store.snapshot, update) store.valid_updates = [] elif current_slot > store.snapshot.header.slot + LIGHT_CLIENT_UPDATE_TIMEOUT: # Forced best update when the update timeout has elapsed - apply_light_client_update(store, max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) + apply_light_client_update(store.snapshot, + max(store.valid_updates, key=lambda update: sum(update.sync_committee_bits))) store.valid_updates = [] ``` From 9d3556668b2bfb096c386de3e0432c4b8296605c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:02:05 +0800 Subject: [PATCH 08/13] Fix domain generation --- specs/lightclient/sync-protocol.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/specs/lightclient/sync-protocol.md b/specs/lightclient/sync-protocol.md index 306fca3b2..28705803b 100644 --- a/specs/lightclient/sync-protocol.md +++ b/specs/lightclient/sync-protocol.md @@ -115,7 +115,8 @@ A light client maintains its state in a `store` object of type `LightClientStore #### `validate_light_client_update` ```python -def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate) -> None: +def validate_light_client_update(snapshot: LightClientSnapshot, update: LightClientUpdate, + genesis_validators_root: Root) -> None: # Verify update slot is larger than snapshot slot assert update.header.slot > snapshot.header.slot @@ -157,7 +158,7 @@ def validate_light_client_update(snapshot: LightClientSnapshot, update: LightCli # Verify sync committee aggregate signature participant_pubkeys = [pubkey for (bit, pubkey) in zip(update.sync_committee_bits, sync_committee.pubkeys) if bit] - domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version) + domain = compute_domain(DOMAIN_SYNC_COMMITTEE, update.fork_version, genesis_validators_root) signing_root = compute_signing_root(signed_header, domain) assert bls.FastAggregateVerify(participant_pubkeys, signing_root, update.sync_committee_signature) ``` @@ -177,8 +178,9 @@ def apply_light_client_update(snapshot: LightClientSnapshot, update: LightClient #### `process_light_client_update` ```python -def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot) -> None: - validate_light_client_update(store.snapshot, update) +def process_light_client_update(store: LightClientStore, update: LightClientUpdate, current_slot: Slot, + genesis_validators_root: Root) -> None: + validate_light_client_update(store.snapshot, update, genesis_validators_root) store.valid_updates.append(update) if ( From 2de64cbda15cb944f4378db532f210954a62ae69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 21:04:16 +0800 Subject: [PATCH 09/13] Add `process_light_client_update` tests --- configs/mainnet/lightclient_patch.yaml | 6 + configs/minimal/lightclient_patch.yaml | 6 + .../eth2spec/test/helpers/sync_committee.py | 14 +- .../unittests/test_helpers.py | 2 +- .../unittests/test_sync_protocol.py | 212 ++++++++++++++++++ 5 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index a9ddc16f6..ce4b509c4 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -29,3 +29,9 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 # Signature domains # --------------------------------------------------------------- DOMAIN_SYNC_COMMITTEE: 0x07000000 + + +# Sync protocol +# --------------------------------------------------------------- +# 2**13 (=8192) +LIGHT_CLIENT_UPDATE_TIMEOUT: 8192 diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 56ce34591..4e8cf3fd3 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -29,3 +29,9 @@ EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 # Signature domains # --------------------------------------------------------------- DOMAIN_SYNC_COMMITTEE: 0x07000000 + + +# Sync protocol +# --------------------------------------------------------------- +# [customized] +LIGHT_CLIENT_UPDATE_TIMEOUT: 32 diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index b7b2381e3..da85fad60 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -5,17 +5,18 @@ from eth2spec.test.helpers.block import ( from eth2spec.utils import bls -def compute_sync_committee_signature(spec, state, slot, privkey): +def compute_sync_committee_signature(spec, state, slot, privkey, block_root=None): domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) - if slot == state.slot: - block_root = build_empty_block_for_next_slot(spec, state).parent_root - else: - block_root = spec.get_block_root_at_slot(state, slot) + if block_root is None: + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) signing_root = spec.compute_signing_root(block_root, domain) return bls.Sign(privkey, signing_root) -def compute_aggregate_sync_committee_signature(spec, state, slot, participants): +def compute_aggregate_sync_committee_signature(spec, state, slot, participants, block_root=None): if len(participants) == 0: return spec.G2_POINT_AT_INFINITY @@ -28,6 +29,7 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): state, slot, privkey, + block_root=block_root, ) ) return bls.Aggregate(signatures) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py index 33e749380..48054d088 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_helpers.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.merkle import build_proof @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test def test_next_sync_committee_tree(spec, state): - state.next_sync_committee = spec.SyncCommittee( + state.next_sync_committee: object = spec.SyncCommittee( pubkeys=[state.validators[i]for i in range(spec.SYNC_COMMITTEE_SIZE)] ) next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py new file mode 100644 index 000000000..b9fdb66ba --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py @@ -0,0 +1,212 @@ +from eth2spec.test.context import ( + LIGHTCLIENT_PATCH, + spec_state_test, + with_phases, +) +from eth2spec.test.helpers.attestations import next_epoch_with_attestations +from eth2spec.test.helpers.block import ( + build_empty_block, + build_empty_block_for_next_slot, +) +from eth2spec.test.helpers.state import ( + next_slots, + state_transition_and_sign_block, +) +from eth2spec.test.helpers.sync_committee import ( + compute_aggregate_sync_committee_signature, +) +from eth2spec.test.helpers.merkle import build_proof + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_not_updated(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Block at slot 1 doesn't increase sync committee period, so it won't update snapshot + block = build_empty_block_for_next_slot(spec, state) + signed_block = state_transition_and_sign_block(spec, state, block) + block_header = spec.BeaconBlockHeader( + slot=signed_block.message.slot, + proposer_index=signed_block.message.proposer_index, + parent_root=signed_block.message.parent_root, + state_root=signed_block.message.state_root, + body_root=signed_block.message.body.hash_tree_root(), + ) + # Sync committee signing the header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot, + committee, + ) + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + + # Ensure that finality checkpoint is genesis + assert state.finalized_checkpoint.epoch == 0 + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + update = spec.LightClientUpdate( + header=block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=finality_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + assert len(store.valid_updates) == 1 + assert store.valid_updates[0] == update + assert store.snapshot == pre_snapshot + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_timeout(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Forward to next sync committee period + next_slots(spec, state, spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD)) + snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert snapshot_period + 1 == update_period + + block = build_empty_block_for_next_slot(spec, state) + signed_block = state_transition_and_sign_block(spec, state, block) + block_header = spec.BeaconBlockHeader( + slot=signed_block.message.slot, + proposer_index=signed_block.message.proposer_index, + parent_root=signed_block.message.parent_root, + state_root=signed_block.message.state_root, + body_root=signed_block.message.body.hash_tree_root(), + ) + + # Sync committee is updated + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block_header.slot, + committee, + block_root=spec.Root(block_header.hash_tree_root()), + ) + + update = spec.LightClientUpdate( + header=block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=finality_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + # snapshot has been updated + assert len(store.valid_updates) == 0 + assert store.snapshot.header == update.header + + +@with_phases([LIGHTCLIENT_PATCH]) +@spec_state_test +def test_process_light_client_update_finality_updated(spec, state): + pre_snapshot = spec.LightClientSnapshot( + header=spec.BeaconBlockHeader(), + current_sync_committee=state.current_sync_committee, + next_sync_committee=state.next_sync_committee, + ) + store = spec.LightClientStore( + snapshot=pre_snapshot, + valid_updates=[] + ) + + # Change finality + blocks = [] + next_slots(spec, state, spec.SLOTS_PER_EPOCH * 2) + for epoch in range(3): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) + blocks += new_blocks + # Ensure that finality checkpoint has changed + assert state.finalized_checkpoint.epoch == 3 + # Ensure that it's same period + snapshot_period = spec.compute_epoch_at_slot(pre_snapshot.header.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + update_period = spec.compute_epoch_at_slot(state.slot) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + assert snapshot_period == update_period + + # Updated sync_committee and finality + next_sync_committee_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.NEXT_SYNC_COMMITTEE_INDEX))] + finalized_block_header = blocks[spec.SLOTS_PER_EPOCH - 1].message + assert finalized_block_header.slot == spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) + assert finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root + finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) + + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + finalized_block_header.slot, + committee, + block_root=spec.Root(finalized_block_header.hash_tree_root()), + ) + + # Build block header + block = build_empty_block(spec, state) + block_header = spec.BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=state.hash_tree_root(), + body_root=block.body.hash_tree_root(), + ) + + update = spec.LightClientUpdate( + header=finalized_block_header, + next_sync_committee=state.next_sync_committee, + next_sync_committee_branch=next_sync_committee_branch, + finality_header=block_header, + finality_branch=finality_branch, + sync_committee_bits=sync_committee_bits, + sync_committee_signature=sync_committee_signature, + fork_version=state.fork.current_version, + ) + + spec.process_light_client_update(store, update, state.slot, state.genesis_validators_root) + + # snapshot has been updated + assert len(store.valid_updates) == 0 + assert store.snapshot.header == update.header From 72832c8b9c99e8950e74d3e8c1d0650ca4f04f36 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Mar 2021 22:12:20 +0800 Subject: [PATCH 10/13] Fix test --- .../unittests/test_sync_protocol.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py index b9fdb66ba..d65e0ad01 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/unittests/test_sync_protocol.py @@ -1,6 +1,8 @@ from eth2spec.test.context import ( LIGHTCLIENT_PATCH, + MINIMAL, spec_state_test, + with_configs, with_phases, ) from eth2spec.test.helpers.attestations import next_epoch_with_attestations @@ -78,6 +80,7 @@ def test_process_light_client_update_not_updated(spec, state): @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): pre_snapshot = spec.LightClientSnapshot( header=spec.BeaconBlockHeader(), @@ -105,12 +108,6 @@ def test_process_light_client_update_timeout(spec, state): body_root=signed_block.message.body.hash_tree_root(), ) - # Sync committee is updated - next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) - # Finality is unchanged - finality_header = spec.BeaconBlockHeader() - finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] - # Sync committee signing the finalized_block_header committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) sync_committee_bits = [True] * len(committee) @@ -122,6 +119,12 @@ def test_process_light_client_update_timeout(spec, state): block_root=spec.Root(block_header.hash_tree_root()), ) + # Sync committee is updated + next_sync_committee_branch = build_proof(state.get_backing(), spec.NEXT_SYNC_COMMITTEE_INDEX) + # Finality is unchanged + finality_header = spec.BeaconBlockHeader() + finality_branch = [spec.Bytes32() for _ in range(spec.floorlog2(spec.FINALIZED_ROOT_INDEX))] + update = spec.LightClientUpdate( header=block_header, next_sync_committee=state.next_sync_committee, @@ -142,6 +145,7 @@ def test_process_light_client_update_timeout(spec, state): @with_phases([LIGHTCLIENT_PATCH]) @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): pre_snapshot = spec.LightClientSnapshot( header=spec.BeaconBlockHeader(), @@ -173,17 +177,6 @@ def test_process_light_client_update_finality_updated(spec, state): assert finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) - # Sync committee signing the finalized_block_header - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) - sync_committee_bits = [True] * len(committee) - sync_committee_signature = compute_aggregate_sync_committee_signature( - spec, - state, - finalized_block_header.slot, - committee, - block_root=spec.Root(finalized_block_header.hash_tree_root()), - ) - # Build block header block = build_empty_block(spec, state) block_header = spec.BeaconBlockHeader( @@ -194,11 +187,22 @@ def test_process_light_client_update_finality_updated(spec, state): body_root=block.body.hash_tree_root(), ) + # Sync committee signing the finalized_block_header + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + sync_committee_bits = [True] * len(committee) + sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block_header.slot, + committee, + block_root=spec.Root(block_header.hash_tree_root()), + ) + update = spec.LightClientUpdate( header=finalized_block_header, next_sync_committee=state.next_sync_committee, next_sync_committee_branch=next_sync_committee_branch, - finality_header=block_header, + finality_header=block_header, # block_header is the signed header finality_branch=finality_branch, sync_committee_bits=sync_committee_bits, sync_committee_signature=sync_committee_signature, From a1e74b2c18a8885dab633c85470b99e7e162afcc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 00:51:31 +0800 Subject: [PATCH 11/13] Fix conflicts --- .../pyspec/eth2spec/test/altair/unittests/test_helpers.py | 6 +++--- .../eth2spec/test/altair/unittests/test_sync_protocol.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py index 48054d088..d89ff6d84 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_helpers.py @@ -1,12 +1,12 @@ from eth2spec.test.context import ( spec_state_test, with_phases, - LIGHTCLIENT_PATCH, + ALTAIR, ) from eth2spec.test.helpers.merkle import build_proof -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_next_sync_committee_tree(spec, state): state.next_sync_committee: object = spec.SyncCommittee( @@ -22,7 +22,7 @@ def test_next_sync_committee_tree(spec, state): ) -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_finality_root_tree(spec, state): finality_branch = build_proof(state.get_backing(), spec.FINALIZED_ROOT_INDEX) diff --git a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py index d65e0ad01..4c9b98e0a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py +++ b/tests/core/pyspec/eth2spec/test/altair/unittests/test_sync_protocol.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, + ALTAIR, MINIMAL, spec_state_test, with_configs, @@ -20,7 +20,7 @@ from eth2spec.test.helpers.sync_committee import ( from eth2spec.test.helpers.merkle import build_proof -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test def test_process_light_client_update_not_updated(spec, state): pre_snapshot = spec.LightClientSnapshot( @@ -78,7 +78,7 @@ def test_process_light_client_update_not_updated(spec, state): assert store.snapshot == pre_snapshot -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_timeout(spec, state): @@ -143,7 +143,7 @@ def test_process_light_client_update_timeout(spec, state): assert store.snapshot.header == update.header -@with_phases([LIGHTCLIENT_PATCH]) +@with_phases([ALTAIR]) @spec_state_test @with_configs([MINIMAL], reason="too slow") def test_process_light_client_update_finality_updated(spec, state): From 22fe06829b0accebc920d744508a93980eba54ee Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 20:26:26 +0800 Subject: [PATCH 12/13] Refactor `objects_to_spec` --- setup.py | 59 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 8557f38b1..6022f9d1e 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,12 @@ from typing import Dict, NamedTuple, List FUNCTION_REGEX = r'^def [\w_]*' +# Definitions in context.py +PHASE0 = 'phase0' +ALTAIR = 'altair' +PHASE1 = 'phase1' + + class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] @@ -302,8 +308,7 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable ssz_path = Path(ssz_class) for item in path: ssz_path = ssz_path / item - return GeneralizedIndex(ssz_path.gindex()) -''' + return GeneralizedIndex(ssz_path.gindex())''' # The constants that depend on SSZ objects @@ -314,6 +319,18 @@ ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = { } +def is_phase0(fork): + return fork == PHASE0 + + +def is_altair(fork): + return fork == ALTAIR + + +def is_phase1(fork): + return fork == PHASE1 + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -335,34 +352,32 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) + + if is_altair(fork): + altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) + spec = ( imports + '\n\n' + f"fork = \'{fork}\'\n" + '\n\n' + new_type_definitions + '\n' + SUNDRY_CONSTANTS_FUNCTIONS - ) - - if fork == 'altair': - altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += ( - ALTAIR_SUNDRY_FUNCTIONS - + '\n\n' + altair_ssz_dep_constants - ) - - spec += ( - '\n\n' + constants_spec + # The constants that some SSZ containers require. Need to be defined before `constants_spec` + + ('\n\n' + altair_ssz_dep_constants if is_altair(fork) else '') + + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec + # Functions to make pyspec work + '\n' + PHASE0_SUNDRY_FUNCTIONS + + ('\n' + ALTAIR_SUNDRY_FUNCTIONS if is_altair(fork) else '') + + ('\n' + PHASE1_SUNDRY_FUNCTIONS if is_phase1(fork) else '') ) - if fork == 'phase1': - spec += '\n' + PHASE1_SUNDRY_FUNCTIONS - - if fork == 'altair': + # Since some contants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + if is_altair(fork): altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += '\n\n' + altair_ssz_dep_constants_verification + spec += '\n\n\n' + altair_ssz_dep_constants_verification spec += '\n' return spec @@ -484,7 +499,7 @@ class PySpecCommand(Command): def initialize_options(self): """Set default values for options.""" # Each user option must be listed here with their default value. - self.spec_fork = 'phase0' + self.spec_fork = PHASE0 self.md_doc_paths = '' self.out_dir = 'pyspec_output' @@ -493,14 +508,14 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if self.spec_fork == "phase0": + if is_phase0(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md """ - elif self.spec_fork == "phase1": + elif is_phase1(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -514,7 +529,7 @@ class PySpecCommand(Command): specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ - elif self.spec_fork == "altair": + elif is_altair(self.spec_fork): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md From 310301236f5b292dd0cd86434fc23569fdaf51da Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 12 Mar 2021 20:29:28 +0800 Subject: [PATCH 13/13] Fix typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6022f9d1e..3bd64869f 100644 --- a/setup.py +++ b/setup.py @@ -373,7 +373,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl + ('\n' + PHASE1_SUNDRY_FUNCTIONS if is_phase1(fork) else '') ) - # Since some contants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. if is_altair(fork): altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS))