From 6f0f2a8f5373f94652f45ce3f36af47fa01ab6b8 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sat, 30 Mar 2019 16:21:09 -0700 Subject: [PATCH 01/26] Update the descriptive text to refer to the correct type of root --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d08828692..629d2990c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -942,7 +942,7 @@ def get_block_root(state: BeaconState, return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` -`get_block_root(_, s)` should always return `hash_tree_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. +`get_block_root(_, s)` should always return `signed_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. ### `get_state_root` From 2bda58fbdcd54355d035096d863e92279fe79664 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 11 Apr 2019 17:13:45 +1000 Subject: [PATCH 02/26] Clean up light client spec --- specs/light_client/merkle_proofs.md | 36 ++++++++++++++--------------- specs/light_client/sync_protocol.md | 21 +++++++++-------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 285445ca8..a3c8fa154 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -47,33 +47,33 @@ y_data_root len(y) We can now define a concept of a "path", a way of describing a function that takes as input an SSZ object and outputs some specific (possibly deeply nested) member. For example, `foo -> foo.x` is a path, as are `foo -> len(foo.y)` and `foo -> foo.y[5].w`. We'll describe paths as lists, which can have two representations. In "human-readable form", they are `["x"]`, `["y", "__len__"]` and `["y", 5, "w"]` respectively. In "encoded form", they are lists of `uint64` values, in these cases (assuming the fields of `foo` in order are `x` then `y`, and `w` is the first field of `y[i]`) `[0]`, `[1, 2**64-1]`, `[1, 5, 0]`. ```python -def path_to_encoded_form(obj: Any, path: List[str or int]) -> List[int]: +def path_to_encoded_form(obj: Any, path: List[Union[str, int]]) -> List[int]: if len(path) == 0: return [] - if isinstance(path[0], "__len__"): + elif isinstance(path[0], "__len__"): assert len(path) == 1 return [LENGTH_FLAG] elif isinstance(path[0], str) and hasattr(obj, "fields"): return [list(obj.fields.keys()).index(path[0])] + path_to_encoded_form(getattr(obj, path[0]), path[1:]) - elif isinstance(obj, (StaticList, DynamicList)): + elif isinstance(obj, (Vector, List)): return [path[0]] + path_to_encoded_form(obj[path[0]], path[1:]) else: raise Exception("Unknown type / path") ``` -We can now define a function `get_generalized_indices(object: Any, path: List[int], root=1: int) -> int` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access. +We can now define a function `get_generalized_indices(object: Any, path: List[int], root: int=1) -> List[int]` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access. ```python -def get_generalized_indices(obj: Any, path: List[int], root=1) -> List[int]: +def get_generalized_indices(obj: Any, path: List[int], root: int=1) -> List[int]: if len(path) == 0: return [root] - elif isinstance(obj, StaticList): + elif isinstance(obj, Vector): items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 new_root = root * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk return get_generalized_indices(obj[path[0]], path[1:], new_root) - elif isinstance(obj, DynamicList) and path[0] == LENGTH_FLAG: + elif isinstance(obj, List) and path[0] == LENGTH_FLAG: return [root * 2 + 1] - elif isinstance(obj, DynamicList) and isinstance(path[0], int): + elif isinstance(obj, List) and isinstance(path[0], int): assert path[0] < len(obj) items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 new_root = root * 2 * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk @@ -137,19 +137,19 @@ Generating a proof is simply a matter of taking the node of the SSZ hash tree wi Here is the verification function: ```python -def verify_multi_proof(root, indices, leaves, proof): +def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[bytes]): tree = {} for index, leaf in zip(indices, leaves): tree[index] = leaf for index, proofitem in zip(get_proof_indices(indices), proof): tree[index] = proofitem - indexqueue = sorted(tree.keys())[:-1] + index_queue = sorted(tree.keys())[:-1] i = 0 - while i < len(indexqueue): - index = indexqueue[i] - if index >= 2 and index^1 in tree: - tree[index//2] = hash(tree[index - index%2] + tree[index - index%2 + 1]) - indexqueue.append(index//2) + while i < len(index_queue): + index = index_queue[i] + if index >= 2 and index ^ 1 in tree: + tree[index // 2] = hash(tree[index - index % 2] + tree[index - index % 2 + 1]) + index_queue.append(index // 2) i += 1 return (indices == []) or (1 in tree and tree[1] == root) ``` @@ -158,7 +158,7 @@ def verify_multi_proof(root, indices, leaves, proof): We define: -#### `MerklePartial` +#### `SSZMerklePartial` ```python @@ -172,6 +172,6 @@ We define: #### Proofs for execution -We define `MerklePartial(f, arg1, arg2..., focus=0)` as being a `MerklePartial` object wrapping a Merkle multiproof of the set of nodes in the hash tree of the SSZ object `arg[focus]` that is needed to authenticate the parts of the object needed to compute `f(arg1, arg2...)`. +We define `MerklePartial(f, arg1, arg2..., focus=0)` as being a `SSZMerklePartial` object wrapping a Merkle multiproof of the set of nodes in the hash tree of the SSZ object `arg[focus]` that is needed to authenticate the parts of the object needed to compute `f(arg1, arg2...)`. -Ideally, any function which accepts an SSZ object should also be able to accept a `MerklePartial` object as a substitute. +Ideally, any function which accepts an SSZ object should also be able to accept a `SSZMerklePartial` object as a substitute. diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 94ab8a2e4..7db02050e 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -5,16 +5,19 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers ## Table of Contents + - [Beacon Chain Light Client Syncing](#beacon-chain-light-client-syncing) - [Table of Contents](#table-of-contents) - - [Light client state](#light-client-state) - - [Updating the shuffled committee](#updating-the-shuffled-committee) - - [Computing the current committee](#computing-the-current-committee) - - [Verifying blocks](#verifying-blocks) + - [Preliminaries](#preliminaries) + - [Light client state](#light-client-state) + - [Updating the shuffled committee](#updating-the-shuffled-committee) + - [Computing the current committee](#computing-the-current-committee) + - [Verifying blocks](#verifying-blocks) + -### Preliminaries +## Preliminaries We define an "expansion" of an object as an object where a field in an object that is meant to represent the `hash_tree_root` of another object is replaced by the object. Note that defining expansions is not a consensus-layer-change; it is merely a "re-interpretation" of the object. Particularly, the `hash_tree_root` of an expansion of an object is identical to that of the original object, and we can define expansions where, given a complete history, it is always possible to compute the expansion of any object in the history. The opposite of an expansion is a "summary" (eg. `BeaconBlockHeader` is a summary of `BeaconBlock`). @@ -85,7 +88,7 @@ later_period_data = get_period_data(new_committee_proof, finalized_header, shard The maximum size of a proof is `128 * ((22-7) * 32 + 110) = 75520` bytes for validator records and `(22-7) * 32 + 128 * 8 = 1504` for the active index proof (much smaller because the relevant active indices are all beside each other in the Merkle tree). This needs to be done once per `PERSISTENT_COMMITTEE_PERIOD` epochs (2048 epochs / 9 days), or ~38 bytes per epoch. -### Computing the current committee +## Computing the current committee Here is a helper to compute the committee at a slot given the maximal earlier and later committees: @@ -134,16 +137,16 @@ def compute_committee(header: BeaconBlockHeader, Note that this method makes use of the fact that the committee for any given shard always starts and ends at the same validator index independently of the committee count (this is because the validator set is split into `SHARD_COUNT * committee_count` slices but the first slice of a shard is a multiple `committee_count * i`, so the start of the slice is `n * committee_count * i // (SHARD_COUNT * committee_count) = n * i // SHARD_COUNT`, using the slightly nontrivial algebraic identity `(x * a) // ab == x // b`). -### Verifying blocks +## Verifying blocks If a client wants to update its `finalized_header` it asks the network for a `BlockValidityProof`, which is simply: ```python { - 'header': BlockHeader, + 'header': BeaconBlockHeader, 'shard_aggregate_signature': 'bytes96', 'shard_bitfield': 'bytes', - 'shard_parent_block': ShardBlock + 'shard_parent_block': ShardBlock, } ``` From 8c32128ffbda5c7e056c218cdb78ab76d856c5f5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Apr 2019 22:28:42 +1000 Subject: [PATCH 03/26] initial pass on genesis slot == 0 --- specs/core/0_beacon-chain.md | 20 ++++++++++++------- .../test_process_attester_slashing.py | 3 +++ tests/phase0/helpers.py | 13 ++++++++++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 185b07443..ad7efc648 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -216,8 +216,8 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` | -| `GENESIS_SLOT` | `2**32` | -| `GENESIS_EPOCH` | `slot_to_epoch(GENESIS_SLOT)` | +| `GENESIS_SLOT` | `0` | +| `GENESIS_EPOCH` | `0` | | `GENESIS_START_SHARD` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `int_to_bytes32(0)` | @@ -1044,12 +1044,12 @@ def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: ```python def get_crosslink_committee_for_attestation(state: BeaconState, - attestation_data: AttestationData) -> List[ValidatorIndex]: + attestation_data: AttestationData) -> List[ValidatorIndex]: """ Return the crosslink committee corresponding to ``attestation_data``. - """ + """ crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) - + # Find the committee in the list with the desired shard assert attestation_data.shard in [shard for _, shard in crosslink_committees] crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] @@ -1554,7 +1554,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Finality previous_epoch_attestations=[], current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH - 1, + previous_justified_epoch=GENESIS_EPOCH, current_justified_epoch=GENESIS_EPOCH, previous_justified_root=ZERO_HASH, current_justified_root=ZERO_HASH, @@ -1814,6 +1814,9 @@ Run the following function: ```python def update_justification_and_finalization(state: BeaconState) -> None: + if get_current_epoch(state) == GENESIS_EPOCH: + return + new_justified_epoch = state.current_justified_epoch new_finalized_epoch = state.finalized_epoch @@ -1864,7 +1867,7 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - previous_epoch = max(current_epoch - 1, GENESIS_EPOCH) + previous_epoch = current_epoch if current_epoch == GENESIS_EPOCH else get_previous_epoch(state) next_epoch = current_epoch + 1 for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): @@ -1999,6 +2002,9 @@ Run the following: ```python def apply_rewards(state: BeaconState) -> None: + if get_current_epoch(state) == GENESIS_EPOCH: + return + rewards1, penalties1 = get_justification_and_finalization_deltas(state) rewards2, penalties2 = get_crosslink_deltas(state) for i in range(len(state.validator_registry)): diff --git a/tests/phase0/block_processing/test_process_attester_slashing.py b/tests/phase0/block_processing/test_process_attester_slashing.py index 06f214c4b..8b7334294 100644 --- a/tests/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/phase0/block_processing/test_process_attester_slashing.py @@ -9,6 +9,7 @@ from build.phase0.spec import ( ) from tests.phase0.helpers import ( get_valid_attester_slashing, + next_epoch, ) # mark entire file as 'attester_slashing' @@ -59,6 +60,8 @@ def test_success_double(state): def test_success_surround(state): + next_epoch(state) + state.current_justified_epoch += 1 attester_slashing = get_valid_attester_slashing(state) # set attestion1 to surround attestation 2 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 61f02ea8c..020e51831 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -4,6 +4,9 @@ from py_ecc import bls import build.phase0.spec as spec from build.phase0.utils.minimal_ssz import signing_root +from build.phase0.state_transition import ( + state_transition, +) from build.phase0.spec import ( # constants EMPTY_SIGNATURE, @@ -144,7 +147,7 @@ def build_attestation_data(state, slot, shard): if epoch_start_slot == slot: epoch_boundary_root = block_root else: - get_block_root(state, epoch_start_slot) + epoch_boundary_root = get_block_root(state, epoch_start_slot) if slot < epoch_start_slot: justified_block_root = state.previous_justified_root @@ -260,7 +263,7 @@ def get_valid_attester_slashing(state): def get_valid_attestation(state, slot=None): if slot is None: slot = state.slot - shard = state.latest_start_shard + shard = state.latest_start_shard + slot % spec.SLOTS_PER_EPOCH attestation_data = build_attestation_data(state, slot, shard) crosslink_committee = get_crosslink_committee_for_attestation(state, attestation_data) @@ -312,3 +315,9 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) domain_type=spec.DOMAIN_ATTESTATION, ) ) + + +def next_epoch(state): + block = build_empty_block_for_next_slot(state) + block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition(state, block) From 6ca550489ed23c53d58e61c071236400d6ba433f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Apr 2019 22:48:27 +1000 Subject: [PATCH 04/26] prevent underflow of previous epoch in all cases --- specs/core/0_beacon-chain.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ad7efc648..394b26ee1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -702,7 +702,8 @@ def get_previous_epoch(state: BeaconState) -> Epoch: """` Return the previous epoch of the given ``state``. """ - return get_current_epoch(state) - 1 + current_epoch = get_current_epoch(state) + return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch ``` ### `get_current_epoch` @@ -2338,6 +2339,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check target epoch, source epoch, and source root target_epoch = slot_to_epoch(attestation.data.slot) + assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), From 47b9cdbb367b5227eb8a17ea625accec9bbc2f2f Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 12 Apr 2019 13:18:10 +1000 Subject: [PATCH 05/26] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 394b26ee1..0cf1a59c0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,8 +224,6 @@ These configurations are updated for releases, but may be out of sync during `de | `EMPTY_SIGNATURE` | `int_to_bytes96(0)` | | `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes1(0)` | -* `GENESIS_SLOT` should be at least as large in terms of time as the largest of the time parameters or state list lengths below (ie. it should be at least as large as any value measured in slots, and at least `SLOTS_PER_EPOCH` times as large as any value measured in epochs). - ### Time parameters | Name | Value | Unit | Duration | @@ -1608,7 +1606,7 @@ For a beacon chain block, `block`, to be processed by a node, the following cond * The parent block with root `block.previous_block_root` has been processed and accepted. * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + (block.slot - GENESIS_SLOT) * SECONDS_PER_SLOT`. (Note that leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year.) +* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. (Note that leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year.) If these conditions are not met, the client should delay processing the beacon block until the conditions are all satisfied. @@ -2334,8 +2332,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. Note that this function mutates ``state``. """ - assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY + assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation.data.slot + SLOTS_PER_EPOCH # Check target epoch, source epoch, and source root target_epoch = slot_to_epoch(attestation.data.slot) From e9a44545be522f7fa55d73840fa582df6d739226 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 12 Apr 2019 13:38:58 +1000 Subject: [PATCH 06/26] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0cf1a59c0..07dc5be6f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1366,14 +1366,16 @@ Note: All functions in this section mutate `state`. #### `activate_validator` ```python -def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: +def activate_validator(state: BeaconState, index: ValidatorIndex) -> None: """ Activate the validator of the given ``index``. Note that this function mutates ``state``. """ validator = state.validator_registry[index] - - validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) + if state.slot == GENESIS_SLOT: + validator.activation_epoch = GENESIS_EPOCH + else: + validator.activation_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) ``` #### `initiate_validator_exit` @@ -1583,7 +1585,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], # Process genesis activations for validator_index, _ in enumerate(state.validator_registry): if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, validator_index, is_genesis=True) + activate_validator(state, validator_index) genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): @@ -1865,9 +1867,8 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - previous_epoch = current_epoch if current_epoch == GENESIS_EPOCH else get_previous_epoch(state) - next_epoch = current_epoch + 1 + previous_epoch = get_previous_epoch(state) + next_epoch = get_current_epoch(state) + 1 for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_root, participants = get_winning_root_and_participants(state, shard) @@ -2062,7 +2063,7 @@ def update_validator_registry(state: BeaconState) -> None: break # Activate validator - activate_validator(state, index, is_genesis=False) + activate_validator(state, index) # Exit validators within the allowable balance churn if current_epoch < state.validator_registry_update_epoch + LATEST_SLASHED_EXIT_LENGTH: From 3c55854072465787ccdd64d31a251553710fa5cb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 12 Apr 2019 21:16:34 +1000 Subject: [PATCH 07/26] clean up --- specs/core/0_beacon-chain.md | 5 ++--- tests/phase0/conftest.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 07dc5be6f..a45e0565b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1867,9 +1867,9 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: - previous_epoch = get_previous_epoch(state) + start_epoch = get_previous_epoch(state) next_epoch = get_current_epoch(state) + 1 - for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): + for slot in range(get_epoch_start_slot(start_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_root, participants = get_winning_root_and_participants(state, shard) participating_balance = get_total_balance(state, participants) @@ -2337,7 +2337,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check target epoch, source epoch, and source root target_epoch = slot_to_epoch(attestation.data.slot) - assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 36a087941..c0461cb67 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -14,7 +14,6 @@ MINIMAL_CONFIG = { "MIN_ATTESTATION_INCLUSION_DELAY": 2, "TARGET_COMMITTEE_SIZE": 4, "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, "SLOTS_PER_HISTORICAL_ROOT": 64, "LATEST_RANDAO_MIXES_LENGTH": 64, "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, From a3b020384ffadcb677ba4374cbfa9d0ff2b73a9b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 12 Apr 2019 23:17:57 +1000 Subject: [PATCH 08/26] add tests for finality through the first few epochs --- specs/core/0_beacon-chain.md | 2 +- tests/phase0/helpers.py | 44 +++++++++++++++++++++++++---- tests/phase0/test_sanity.py | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a45e0565b..dfa26c86d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1007,7 +1007,7 @@ def get_beacon_proposer_index(state: BeaconState, the epoch in question, this can only be run for the current epoch. """ current_epoch = get_current_epoch(state) - assert slot_to_epoch(slot) == current_epoch + # assert slot_to_epoch(slot) == current_epoch first_committee, _ = get_crosslink_committees_at_slot(state, slot)[0] i = 0 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 020e51831..66ae17f2c 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -34,6 +34,7 @@ from build.phase0.spec import ( get_empty_block, get_epoch_start_slot, get_genesis_beacon_state, + get_previous_epoch, slot_to_epoch, verify_merkle_branch, hash, @@ -50,6 +51,19 @@ pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} +def set_bitfield_bit(bitfield, i): + """ + Set the bit in ``bitfield`` at position ``i`` to ``1``. + """ + byte_index = i // 8 + bit_index = i % 8 + return ( + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index+1:] + ) + + def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): if not deposit_data_leaves: deposit_data_leaves = [] @@ -141,24 +155,31 @@ def build_deposit_data(state, pubkey, privkey, amount): def build_attestation_data(state, slot, shard): assert state.slot >= slot - block_root = build_empty_block_for_next_slot(state).previous_block_root + if slot == state.slot: + block_root = build_empty_block_for_next_slot(state).previous_block_root + else: + block_root = get_block_root(state, slot) - epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if epoch_start_slot == slot: + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if slot < current_epoch_start_slot: + epoch_boundary_root = get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + elif slot == current_epoch_start_slot: epoch_boundary_root = block_root else: - epoch_boundary_root = get_block_root(state, epoch_start_slot) + epoch_boundary_root = get_block_root(state, current_epoch_start_slot) - if slot < epoch_start_slot: + if slot < current_epoch_start_slot: + justified_epoch = state.previous_justified_epoch justified_block_root = state.previous_justified_root else: + justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root return AttestationData( slot=slot, shard=shard, beacon_block_root=block_root, - source_epoch=state.current_justified_epoch, + source_epoch=justified_epoch, source_root=justified_block_root, target_root=epoch_boundary_root, crosslink_data_root=spec.ZERO_HASH, @@ -317,6 +338,17 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) ) +def fill_aggregate_attestation(state, attestation): + crosslink_committee = get_crosslink_committee_for_attestation(state, attestation.data) + for i in range(len(crosslink_committee)): + attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + + +def next_slot(state): + block = build_empty_block_for_next_slot(state) + state_transition(state, block) + + def next_epoch(state): block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 0930bad07..67a27f3f2 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -39,10 +39,12 @@ from build.phase0.utils.merkle_minimal import ( from tests.phase0.helpers import ( build_deposit_data, build_empty_block_for_next_slot, + fill_aggregate_attestation, force_registry_change_at_next_epoch, get_valid_attestation, get_valid_attester_slashing, get_valid_proposer_slashing, + next_slot, privkeys, pubkeys, ) @@ -52,6 +54,33 @@ from tests.phase0.helpers import ( pytestmark = pytest.mark.sanity +def check_finality(state, + prev_state, + current_justified_changed, + previous_justified_changed, + finalized_changed): + if current_justified_changed: + assert state.current_justified_epoch > prev_state.current_justified_epoch + assert state.current_justified_root != prev_state.current_justified_root + else: + assert state.current_justified_epoch == prev_state.current_justified_epoch + assert state.current_justified_root == prev_state.current_justified_root + + if previous_justified_changed: + assert state.previous_justified_epoch > prev_state.previous_justified_epoch + assert state.previous_justified_root != prev_state.previous_justified_root + else: + assert state.previous_justified_epoch == prev_state.previous_justified_epoch + assert state.previous_justified_root == prev_state.previous_justified_root + + if finalized_changed: + assert state.finalized_epoch > prev_state.finalized_epoch + assert state.finalized_root != prev_state.finalized_root + else: + assert state.finalized_epoch == prev_state.finalized_epoch + assert state.finalized_root == prev_state.finalized_root + + def test_slot_transition(state): test_state = deepcopy(state) cache_state(test_state) @@ -116,6 +145,32 @@ def test_empty_epoch_transition_not_finalizing(state): return state, [block], test_state +def test_full_attestations_finalizing(state): + test_state = deepcopy(state) + + for slot in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(test_state) + + for epoch in range(5): + for slot in range(spec.SLOTS_PER_EPOCH): + attestation = get_valid_attestation(test_state, test_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY) + fill_aggregate_attestation(test_state, attestation) + block = build_empty_block_for_next_slot(test_state) + block.body.attestations.append(attestation) + state_transition(test_state, block) + + if epoch == 0: + check_finality(test_state, state, False, False, False) + elif epoch == 1: + check_finality(test_state, state, False, False, False) + elif epoch == 2: + check_finality(test_state, state, True, False, False) + elif epoch == 3: + check_finality(test_state, state, True, True, False) + elif epoch == 4: + check_finality(test_state, state, True, True, True) + + def test_proposer_slashing(state): test_state = deepcopy(state) proposer_slashing = get_valid_proposer_slashing(state) From 8807781a8dd22c73865bd9d6deb9c368bfca3484 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Apr 2019 18:16:44 +1000 Subject: [PATCH 09/26] formatting --- specs/light_client/merkle_proofs.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index a3c8fa154..47195b2ca 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -20,10 +20,10 @@ In a binary Merkle tree, we define a "generalized index" of a node as `2**depth Note that the generalized index has the convenient property that the two children of node `k` are `2k` and `2k+1`, and also that it equals the position of a node in the linear representation of the Merkle tree that's computed by this function: ```python -def merkle_tree(leaves): +def merkle_tree(leaves: List[Bytes32]) -> List[Bytes32]: o = [0] * len(leaves) + leaves - for i in range(len(leaves)-1, 0, -1): - o[i] = hash(o[i*2] + o[i*2+1]) + for i in range(len(leaves) - 1, 0, -1): + o[i] = hash(o[i * 2] + o[i * 2 + 1]) return o ``` @@ -102,8 +102,8 @@ x x . . . . x * Here is code for creating and verifying a multiproof. First a helper: ```python -def log2(x): - return 0 if x == 1 else 1 + log2(x//2) +def log2(x: int) -> int: + return 0 if x == 1 else 1 + log2(x // 2) ``` First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: @@ -111,7 +111,7 @@ First, a method for computing the generalized indices of the auxiliary tree node ```python def get_proof_indices(tree_indices: List[int]) -> List[int]: # Get all indices touched by the proof - maximal_indices = set({}) + maximal_indices = set() for i in tree_indices: x = i while x > 1: @@ -119,7 +119,7 @@ def get_proof_indices(tree_indices: List[int]) -> List[int]: x //= 2 maximal_indices = tree_indices + sorted(list(maximal_indices))[::-1] # Get indices that cannot be recalculated from earlier indices - redundant_indices = set({}) + redundant_indices = set() proof = [] for index in maximal_indices: if index not in redundant_indices: @@ -130,19 +130,19 @@ def get_proof_indices(tree_indices: List[int]) -> List[int]: break index //= 2 return [i for i in proof if i not in tree_indices] -```` +``` Generating a proof is simply a matter of taking the node of the SSZ hash tree with the union of the given generalized indices for each index given by `get_proof_indices`, and outputting the list of nodes in the same order. Here is the verification function: ```python -def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[bytes]): +def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[bytes]) -> bool: tree = {} for index, leaf in zip(indices, leaves): tree[index] = leaf - for index, proofitem in zip(get_proof_indices(indices), proof): - tree[index] = proofitem + for index, proof_item in zip(get_proof_indices(indices), proof): + tree[index] = proof_item index_queue = sorted(tree.keys())[:-1] i = 0 while i < len(index_queue): From 449e8a44a4e9c3040334cac58b204d9ebede0951 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Apr 2019 18:17:09 +1000 Subject: [PATCH 10/26] Remove unused `log2` --- specs/light_client/merkle_proofs.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 47195b2ca..371f0ffde 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -99,14 +99,7 @@ x x . . . . x * . are unused nodes, * are used nodes, x are the values we are trying to prove. Notice how despite being a multiproof for 3 values, it requires only 3 auxiliary nodes, only one node more than would be required to prove a single value. Normally the efficiency gains are not quite that extreme, but the savings relative to individual Merkle proofs are still significant. As a rule of thumb, a multiproof for k nodes at the same level of an n-node tree has size `k * (n/k + log(n/k))`. -Here is code for creating and verifying a multiproof. First a helper: - -```python -def log2(x: int) -> int: - return 0 if x == 1 else 1 + log2(x // 2) -``` - -First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: +Here is code for creating and verifying a multiproof. First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: ```python def get_proof_indices(tree_indices: List[int]) -> List[int]: From 30fe6f5657715f71510707f987d74078436a971c Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 14 Apr 2019 00:47:14 +1000 Subject: [PATCH 11/26] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dfa26c86d..2737d3055 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1867,9 +1867,9 @@ Run the following function: ```python def process_crosslinks(state: BeaconState) -> None: - start_epoch = get_previous_epoch(state) + previous_epoch = get_previous_epoch(state) next_epoch = get_current_epoch(state) + 1 - for slot in range(get_epoch_start_slot(start_epoch), get_epoch_start_slot(next_epoch)): + for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): winning_root, participants = get_winning_root_and_participants(state, shard) participating_balance = get_total_balance(state, participants) From 705b553139c5e8cb523a62010e76850a85054b68 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 12:11:50 +1000 Subject: [PATCH 12/26] Fix --- specs/light_client/sync_protocol.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 7db02050e..795c057ab 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -43,7 +43,7 @@ We add a data type `PeriodData` and four helpers: { 'validator_count': 'uint64', 'seed': 'bytes32', - 'committee': [Validator] + 'committee': [Validator], } ``` @@ -94,8 +94,7 @@ Here is a helper to compute the committee at a slot given the maximal earlier an ```python def compute_committee(header: BeaconBlockHeader, - validator_memory: ValidatorMemory): - + validator_memory: ValidatorMemory) -> List[ValidatorIndex]: earlier_validator_count = validator_memory.earlier_period_data.validator_count later_validator_count = validator_memory.later_period_data.validator_count maximal_earlier_committee = validator_memory.earlier_period_data.committee @@ -108,11 +107,13 @@ def compute_committee(header: BeaconBlockHeader, earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), later_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), ) + 1 - - def get_offset(count, end:bool): - return get_split_offset(count, - SHARD_COUNT * committee_count, - validator_memory.shard_id * committee_count + (1 if end else 0)) + + def get_offset(count: int, end: bool) -> int: + return get_split_offset( + count, + SHARD_COUNT * committee_count, + validator_memory.shard_id * committee_count + (1 if end else 0), + ) actual_earlier_committee = maximal_earlier_committee[ 0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False) @@ -132,7 +133,6 @@ def compute_committee(header: BeaconBlockHeader, [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) - ``` Note that this method makes use of the fact that the committee for any given shard always starts and ends at the same validator index independently of the committee count (this is because the validator set is split into `SHARD_COUNT * committee_count` slices but the first slice of a shard is a multiple `committee_count * i`, so the start of the slice is `n * committee_count * i // (SHARD_COUNT * committee_count) = n * i // SHARD_COUNT`, using the slightly nontrivial algebraic identity `(x * a) // ab == x // b`). From 5ed4cb29f6b62887b42db4a4651389594805d9e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 12:15:24 +1000 Subject: [PATCH 13/26] ValidatorMemory --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 795c057ab..985796863 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -75,7 +75,7 @@ A light client will keep track of: * `later_period_data = get_period_data(finalized_header, shard_id, later=True)` * `earlier_period_data = get_period_data(finalized_header, shard_id, later=False)` -We use the struct `validator_memory` to keep track of these variables. +We use the struct `ValidatorMemory` to keep track of these variables. ### Updating the shuffled committee From f7d3e02eb254749913e3ed652b890983291a153a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 17:17:09 +1000 Subject: [PATCH 14/26] Add ToC --- specs/light_client/merkle_proofs.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 371f0ffde..46fa23f82 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -1,12 +1,26 @@ **NOTICE**: This document is a work-in-progress for researchers and implementers. -### Constants +## Table of Contents + + +- [Table of Contents](#table-of-contents) +- [Constants](#constants) +- [Generalized Merkle tree index](#generalized-merkle-tree-index) +- [SSZ object to index](#ssz-object-to-index) +- [Merkle multiproofs](#merkle-multiproofs) +- [MerklePartial](#merklepartial) + - [`SSZMerklePartial`](#sszmerklepartial) + - [Proofs for execution](#proofs-for-execution) + + + +## Constants | Name | Value | | - | - | | `LENGTH_FLAG` | `2**64 - 1` | -### Generalized Merkle tree index +## Generalized Merkle tree index In a binary Merkle tree, we define a "generalized index" of a node as `2**depth + index`. Visually, this looks as follows: @@ -29,7 +43,7 @@ def merkle_tree(leaves: List[Bytes32]) -> List[Bytes32]: We will define Merkle proofs in terms of generalized indices. -### SSZ object to index +## SSZ object to index We can describe the hash tree of any SSZ object, rooted in `hash_tree_root(object)`, as a binary Merkle tree whose depth may vary. For example, an object `{x: bytes32, y: List[uint64]}` would look as follows: @@ -86,7 +100,7 @@ def get_generalized_indices(obj: Any, path: List[int], root: int=1) -> List[int] raise Exception("Unknown type / path") ``` -### Merkle multiproofs +## Merkle multiproofs We define a Merkle multiproof as a minimal subset of nodes in a Merkle tree needed to fully authenticate that a set of nodes actually are part of a Merkle tree with some specified root, at a particular set of generalized indices. For example, here is the Merkle multiproof for positions 0, 1, 6 in an 8-node Merkle tree (ie. generalized indices 8, 9, 14): @@ -147,11 +161,11 @@ def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], return (indices == []) or (1 in tree and tree[1] == root) ``` -### MerklePartial +## MerklePartial We define: -#### `SSZMerklePartial` +### `SSZMerklePartial` ```python @@ -163,7 +177,7 @@ We define: } ``` -#### Proofs for execution +### Proofs for execution We define `MerklePartial(f, arg1, arg2..., focus=0)` as being a `SSZMerklePartial` object wrapping a Merkle multiproof of the set of nodes in the hash tree of the SSZ object `arg[focus]` that is needed to authenticate the parts of the object needed to compute `f(arg1, arg2...)`. From 2f2e7847de10e4b424b885108936f5aaacca4dac Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 18:13:43 +1000 Subject: [PATCH 15/26] More fix --- specs/light_client/sync_protocol.md | 63 ++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 985796863..3850f077d 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -9,6 +9,13 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers - [Beacon Chain Light Client Syncing](#beacon-chain-light-client-syncing) - [Table of Contents](#table-of-contents) - [Preliminaries](#preliminaries) + - [Expansions](#expansions) + - [`get_active_validator_indices`](#get_active_validator_indices) + - [`MerklePartial`](#merklepartial) + - [`PeriodData`](#perioddata) + - [`get_earlier_start_epoch`](#get_earlier_start_epoch) + - [`get_later_start_epoch`](#get_later_start_epoch) + - [`get_period_data`](#get_period_data) - [Light client state](#light-client-state) - [Updating the shuffled committee](#updating-the-shuffled-committee) - [Computing the current committee](#computing-the-current-committee) @@ -16,28 +23,34 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers - ## Preliminaries +### Expansions + We define an "expansion" of an object as an object where a field in an object that is meant to represent the `hash_tree_root` of another object is replaced by the object. Note that defining expansions is not a consensus-layer-change; it is merely a "re-interpretation" of the object. Particularly, the `hash_tree_root` of an expansion of an object is identical to that of the original object, and we can define expansions where, given a complete history, it is always possible to compute the expansion of any object in the history. The opposite of an expansion is a "summary" (eg. `BeaconBlockHeader` is a summary of `BeaconBlock`). We define two expansions: -* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState` -* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])` +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])`. +* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`. + +### `get_active_validator_indices` Note that there is now a new way to compute `get_active_validator_indices`: ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: +def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: return state.latest_active_indices[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] ``` Note that it takes `state` instead of `state.validator_registry` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. + +### `MerklePartial` + A `MerklePartial(f, *args)` is an object that contains a minimal Merkle proof needed to compute `f(*args)`. A `MerklePartial` can be used in place of a regular SSZ object, though a computation would return an error if it attempts to access part of the object that is not contained in the proof. -We add a data type `PeriodData` and four helpers: +### `PeriodData` ```python { @@ -47,13 +60,23 @@ We add a data type `PeriodData` and four helpers: } ``` +### `get_earlier_start_epoch` + ```python def get_earlier_start_epoch(slot: Slot) -> int: return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD * 2 - +``` + +### `get_later_start_epoch` + +```python def get_later_start_epoch(slot: Slot) -> int: return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD - +``` + +### `get_period_data` + +```python def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> PeriodData: period_start = get_later_start_epoch(header.slot) if later else get_earlier_start_epoch(header.slot) validator_count = len(get_active_validator_indices(state, period_start)) @@ -62,7 +85,7 @@ def get_period_data(block: ExtendedBeaconBlock, shard_id: Shard, later: bool) -> return PeriodData( validator_count, generate_seed(block.state, period_start), - [block.state.validator_registry[i] for i in indices] + [block.state.validator_registry[i] for i in indices], ) ``` @@ -114,7 +137,7 @@ def compute_committee(header: BeaconBlockHeader, SHARD_COUNT * committee_count, validator_memory.shard_id * committee_count + (1 if end else 0), ) - + actual_earlier_committee = maximal_earlier_committee[ 0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False) ] @@ -123,15 +146,15 @@ def compute_committee(header: BeaconBlockHeader, ] def get_switchover_epoch(index): return ( - bytes_to_int(hash(validator_memory.earlier_period_data.seed + bytes3(index))[0:8]) % + bytes_to_int(hash(validator_memory.earlier_period_data.seed + int_to_bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(list(set( - [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + - [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] + [i for i in actual_earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + + [i for i in actual_later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) ``` @@ -154,23 +177,23 @@ The verification procedure is as follows: ```python def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: ValidatorMemory) -> bool: - assert proof.shard_parent_block.beacon_chain_ref == hash_tree_root(proof.header) + assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([c.high_balance for i, c in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) - total_balance = sum([c.high_balance for i, c in enumerate(committee)] + support_balance = sum([v.high_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + total_balance = sum([v.high_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ - v.pubkey for v, index in enumerate(committee) if - get_bitfield_bit(proof.shard_bitfield, i) is True + v.pubkey for v, index in enumerate(committee) + if get_bitfield_bit(proof.shard_bitfield, index) is True ]) assert bls_verify( pubkey=group_public_key, message_hash=hash_tree_root(shard_parent_block), - signature=shard_aggregate_signature, - domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER) + signature=proof.shard_aggregate_signature, + domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER), ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_ref, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. From 02cfbca81f1da76d2eb92c934f9fdd81550d0566 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 18:17:43 +1000 Subject: [PATCH 16/26] Remove blanks --- specs/light_client/sync_protocol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 3850f077d..d5647f0c5 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -125,7 +125,7 @@ def compute_committee(header: BeaconBlockHeader, earlier_start_epoch = get_earlier_start_epoch(header.slot) later_start_epoch = get_later_start_epoch(header.slot) epoch = slot_to_epoch(header.slot) - + committee_count = max( earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), later_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), @@ -137,7 +137,7 @@ def compute_committee(header: BeaconBlockHeader, SHARD_COUNT * committee_count, validator_memory.shard_id * committee_count + (1 if end else 0), ) - + actual_earlier_committee = maximal_earlier_committee[ 0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False) ] From 3c8d1b23a52c8be12c01628d8546a4443a217ca7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 14 Apr 2019 22:12:39 +1000 Subject: [PATCH 17/26] Update specs/core/0_beacon-chain.md Co-Authored-By: djrtwo --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3be5deda5..accade599 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -691,6 +691,7 @@ def slot_to_epoch(slot: Slot) -> Epoch: def get_previous_epoch(state: BeaconState) -> Epoch: """` Return the previous epoch of the given ``state``. + Return the current epoch if it's genesis epoch. """ current_epoch = get_current_epoch(state) return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch From a4d87d44fb664d68a5d6500bb5ecf23eedcc32ba Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 17 Apr 2019 22:22:53 +1000 Subject: [PATCH 18/26] Remove custody_bitfield from PendingAttestation I don't think we need it :) --- specs/core/0_beacon-chain.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index edd52b2e3..2e8c0b593 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -425,8 +425,6 @@ The types are defined topologically to aid in facilitating an executable version 'aggregation_bitfield': 'bytes', # Attestation data 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', # Inclusion slot 'inclusion_slot': 'uint64', } @@ -2130,7 +2128,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=attestation.data, aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, inclusion_slot=state.slot ) if target_epoch == get_current_epoch(state): From 4bffa87646f3f42ca91ac2c0cdf408762d3dfadc Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 17 Apr 2019 08:57:23 -0600 Subject: [PATCH 19/26] fix finalization bug --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index edd52b2e3..6f829821e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1711,11 +1711,11 @@ def update_justification_and_finalization(state: BeaconState) -> None: state.finalized_epoch = antepenultimate_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_root == current_epoch - 2: + if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 2: state.finalized_epoch = state.previous_justified_root state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_root == current_epoch - 1: + if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 1: state.finalized_epoch = state.previous_justified_root state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` From 73bd821417bd972b886fc33a475b01e897c18cde Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 17 Apr 2019 09:26:23 -0600 Subject: [PATCH 20/26] bug fix --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6f829821e..874d253f8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1712,11 +1712,11 @@ def update_justification_and_finalization(state: BeaconState) -> None: state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 2: - state.finalized_epoch = state.previous_justified_root + state.finalized_epoch = state.previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 1: - state.finalized_epoch = state.previous_justified_root + state.finalized_epoch = state.previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` From af4d06c8bf89acb2f55f5ef73f08b304018d4019 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 17 Apr 2019 09:38:10 -0600 Subject: [PATCH 21/26] only run justification/finalization after 2 epochs --- specs/core/0_beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 83f4ec926..86623dff5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1588,7 +1588,8 @@ def cache_state(state: BeaconState) -> None: state.latest_block_header.state_root = previous_slot_state_root # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signing_root(state.latest_block_header) + latest_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root ``` ### Per-epoch processing @@ -1683,7 +1684,7 @@ Run the following function: ```python def update_justification_and_finalization(state: BeaconState) -> None: - if get_current_epoch(state) == GENESIS_EPOCH: + if get_current_epoch(state) <= GENESIS_EPOCH + 1: return antepenultimate_justified_epoch = state.previous_justified_epoch From 5531adcdd152b91f20edc3ce81e7b0d87a5865e2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 17 Apr 2019 09:41:23 -0600 Subject: [PATCH 22/26] remove old assertion in get_beacon_proposer_index --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86623dff5..14f2536fc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -976,7 +976,6 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: Return the beacon proposer index at ``state.slot``. """ current_epoch = get_current_epoch(state) - # assert slot_to_epoch(slot) == current_epoch first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0] i = 0 From 0a1517c9de4ec91a93fbd11015423ffb39d97a82 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 18 Apr 2019 08:56:46 +0800 Subject: [PATCH 23/26] Update specs/light_client/merkle_proofs.md Co-Authored-By: hwwhww --- specs/light_client/merkle_proofs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 46fa23f82..63c018f2f 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -144,7 +144,7 @@ Generating a proof is simply a matter of taking the node of the SSZ hash tree wi Here is the verification function: ```python -def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[bytes]) -> bool: +def verify_multi_proof(root: Bytes32, indices: List[int], leaves: List[Bytes32], proof: List[Bytes32]) -> bool: tree = {} for index, leaf in zip(indices, leaves): tree[index] = leaf From 71e1a598d283d94242b579e931c57ed58e52fe73 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 11:09:30 +1000 Subject: [PATCH 24/26] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b98d53154..e0556a7f3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1575,16 +1575,15 @@ At every `slot > GENESIS_SLOT` run the following function: ```python def cache_state(state: BeaconState) -> None: - previous_slot_state_root = hash_tree_root(state) + # Cache latest known state root (for previous slot) + latest_state_root = hash_tree_root(state) + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_state_root - # store the previous slot's post state transition root - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root - - # cache state root in stored latest_block_header if empty + # Store latest known state root (for previous slot) in latest_block_header if it is empty if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_slot_state_root + state.latest_block_header.state_root = latest_state_root - # store latest known block for previous slot + # Cache latest known block root (for previous slot) latest_block_root = signing_root(state.latest_block_header) state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root ``` From 443786017fb76f4ab9b86e67b7f5666c893c251a Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 18 Apr 2019 13:31:19 +1000 Subject: [PATCH 25/26] Further decouple justification and finalisation (#958) Addresses @djrtwo's comment [here](https://github.com/ethereum/eth2.0-specs/pull/925#issuecomment-484123950). --- specs/core/0_beacon-chain.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e0556a7f3..1ee9108ad 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1683,7 +1683,8 @@ def update_justification_and_finalization(state: BeaconState) -> None: if get_current_epoch(state) <= GENESIS_EPOCH + 1: return - antepenultimate_justified_epoch = state.previous_justified_epoch + old_previous_justified_epoch = state.previous_justified_epoch + old_current_justified_epoch = state.current_justified_epoch # Process justifications state.previous_justified_epoch = state.current_justified_epoch @@ -1704,20 +1705,20 @@ def update_justification_and_finalization(state: BeaconState) -> None: bitfield = state.justification_bitfield current_epoch = get_current_epoch(state) # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and antepenultimate_justified_epoch == current_epoch - 3: - state.finalized_epoch = antepenultimate_justified_epoch + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and antepenultimate_justified_epoch == current_epoch - 2: - state.finalized_epoch = antepenultimate_justified_epoch + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 2: - state.finalized_epoch = state.previous_justified_epoch + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 1: - state.finalized_epoch = state.previous_justified_epoch + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch)) ``` From 4bca7f984d9e86ef21a54875e8fcd466ab9c348a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 18 Apr 2019 00:43:22 -0700 Subject: [PATCH 26/26] Add the prefix `state` when accessing the latest deposit count (#961) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1ee9108ad..e6f9872c9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2155,7 +2155,7 @@ def process_proposer_attestation_rewards(state: BeaconState) -> None: ##### Deposits -Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`. +Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index)`. For each `deposit` in `block.body.deposits`, run the following function: