From f7ef9a1ba5e7b863538ab48097060f98ec536412 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Sat, 20 Apr 2019 17:26:07 -0700 Subject: [PATCH 01/31] Don't use SSZ in RPC request/response wrappers --- specs/networking/rpc-interface.md | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index 5d408b5a0..ca6008a40 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,32 +39,36 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` -( - id: uint64 - method_id: uint16 - body: Request -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| method_id (uint16) | body_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | body (body_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` and their corresponding responses are wrapped in a "response" structure: ``` -( - id: uint64 - response_code: uint16 - result: bytes -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| response_code (uint16) | result_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | result (result_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` -If an error occurs, a variant of the response structure is returned: - -``` -( - id: uint64 - response_code: uint16 - result: bytes -) -``` +Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From 66b152f79ecad957b38ecccc16ce2f131afd10cc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 3 May 2019 05:07:11 -0500 Subject: [PATCH 02/31] Allow multiple bit challenges, and recover withdrawability Resolves #864 items 4, 7, 14 --- specs/core/1_custody-game.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d56526611..0de84ff06 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -146,7 +146,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'depth': 'uint64', 'chunk_index': 'uint64', @@ -160,7 +160,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'chunk_count': 'uint64', 'chunk_bits_merkle_root': Hash, @@ -485,7 +485,7 @@ def process_chunk_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, chunk_index=challenge.chunk_index, @@ -528,10 +528,9 @@ def process_bit_challenge(state: BeaconState, attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - # A validator can be the challenger or responder for at most one challenge at a time + # A validator can be the challenger for at most one challenge at a time for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - assert record.responder_index != challenge.responder_index # Verify the responder is a valid custody key epoch_to_sign = get_randao_epoch_for_custody_period( @@ -563,7 +562,7 @@ def process_bit_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), @@ -604,6 +603,8 @@ def process_chunk_challenge_response(state: BeaconState, assert response.chunk_index == challenge.chunk_index # Verify bit challenge data is null assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == ZERO_HASH + # Verify minimum delay + assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -626,6 +627,9 @@ def process_bit_challenge_response(state: BeaconState, challenge: CustodyBitChallengeRecord) -> None: # Verify chunk index assert response.chunk_index < challenge.chunk_count + # Verify responder has not been slashed + responder = state.validator_registry[record.responder_index] + assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -671,13 +675,13 @@ Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadl ```python def process_challenge_deadlines(state: BeaconState) -> None: for challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_chunk_challenge_records records[records.index(challenge)] = CustodyChunkChallengeRecord() for challenge in state.custody_bit_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() @@ -688,6 +692,18 @@ Append this to `process_final_updates(state)`: ```python # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] + # Reset withdrawable epochs if challenge records are empty + for index, validator in enumerate(state.validator_registry): + eligible = True + for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): + for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): + if len(list(filter(filter_func, records))) > 0: + eligible = False + if eligible: + if validator.exit_epoch == FAR_FUTURE_EPOCH: + validator.withdrawable_epoch = FAR_FUTURE_EPOCH + else: + validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): From 2ccd357f0eab881eaf7f6bd39e09987467731c91 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 3 May 2019 21:05:54 +0800 Subject: [PATCH 03/31] Update specs/core/1_custody-game.md Co-Authored-By: vbuterin --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0de84ff06..307e18573 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -628,7 +628,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify chunk index assert response.chunk_index < challenge.chunk_count # Verify responder has not been slashed - responder = state.validator_registry[record.responder_index] + responder = state.validator_registry[challenge.responder_index] assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( From 4db4d879301b130eba3667d8f455de8eb07e7c80 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 14:57:36 +0800 Subject: [PATCH 04/31] Refactor `process_final_updates` --- specs/core/1_custody-game.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 1b6b1d2e4..e03e54ed0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -683,13 +683,12 @@ Append this to `process_final_updates(state)`: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] # Reset withdrawable epochs if challenge records are empty + records = state.custody_chunk_challenge_records + state.bit_challenge_records + validator_indices_in_records = set( + [record.challenger_index for record in records] + [record.responder_index for record in records] + ) for index, validator in enumerate(state.validator_registry): - eligible = True - for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): - for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): - if len(list(filter(filter_func, records))) > 0: - eligible = False - if eligible: + if index not in validator_indices_in_records: if validator.exit_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = FAR_FUTURE_EPOCH else: From c2942c00c6555cd03aa0595b9ee947e5eb06e6f1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 17:02:20 -0400 Subject: [PATCH 05/31] lint requires install_test in ci --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd7708f8d..2bfda20ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,10 +102,12 @@ workflows: test_spec: jobs: - checkout_specs - - lint - install_test: requires: - checkout_specs - test: requires: - install_test + - lint: + requires: + - install_test From f830f83a1d6ae1b6728bfd357da6cbef0d4a5107 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 14 May 2019 10:32:20 +0800 Subject: [PATCH 06/31] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bfda20ff..439afc9a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,12 +102,12 @@ workflows: test_spec: jobs: - checkout_specs + - lint: + requires: + - checkout_specs - install_test: requires: - checkout_specs - test: requires: - install_test - - lint: - requires: - - install_test From 5e7b173b22e0d550092ac967f1dc84d2c794e935 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:29:11 -0400 Subject: [PATCH 07/31] fix up validator guide crosslink committee instructions --- specs/validator/0_beacon-chain-validator.md | 119 +++++++++----------- 1 file changed, 53 insertions(+), 66 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef6e8bf25..67a09c797 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -20,6 +20,8 @@ - [Process deposit](#process-deposit) - [Validator index](#validator-index) - [Activation](#activation) + - [Validator assignments](#validator-assignments) + - [Lookahead](#lookahead) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Block header](#block-header) @@ -45,8 +47,6 @@ - [Aggregation bitfield](#aggregation-bitfield) - [Custody bitfield](#custody-bitfield) - [Aggregate signature](#aggregate-signature) - - [Validator assignments](#validator-assignments) - - [Lookahead](#lookahead) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) - [Attester slashing](#attester-slashing) @@ -127,13 +127,61 @@ Once a validator is activated, the validator is assigned [responsibilities](#bea *Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +## Validator assignments + +A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. + +```python +def get_committee_assignment( + state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the shard to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + """ + next_epoch = get_current_epoch(state) + 1 + assert epoch <= next_epoch + + committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH + epoch_start_slot = get_epoch_start_slot(epoch) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + for i in range(committees_per_slot): + shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + committee = get_crosslink_committee(state, epoch, shard) + if validator_index in committee: + return committee, shard, slot +``` + +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. + +```python +def is_proposer(state: BeaconState, + validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` + +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. + +### Lookahead + +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. + +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). + +Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. + ## Beacon chain responsibilities A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). @@ -229,7 +277,7 @@ Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntar ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The slot during which the validator performs this role is any slot at which `get_crosslink_committees_at_slot(state, slot)` contains a committee that contains `validator_index`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The committee, assigned shard, and assigned slot for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. @@ -259,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. Construct `attestation_data.crosslink` via the following -* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. * Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. @@ -310,67 +358,6 @@ signed_attestation_data = bls_sign( ) ``` -## Validator assignments - -A validator can get the current, previous, and next epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= next_epoch`. - -```python -def get_committee_assignment( - state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the shard to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - """ - previous_epoch = get_previous_epoch(state) - next_epoch = get_current_epoch(state) + 1 - assert previous_epoch <= epoch <= next_epoch - - epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): - crosslink_committees = get_crosslink_committees_at_slot( - state, - slot, - ) - selected_committees = [ - committee # Tuple[List[ValidatorIndex], Shard] - for committee in crosslink_committees - if validator_index in committee[0] - ] - if len(selected_committees) > 0: - validators = selected_committees[0][0] - shard = selected_committees[0][1] - - assignment = (validators, shard, slot) - return assignment -``` - -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. - -```python -def is_proposer_at_slot(state: BeaconState, - slot: Slot, - validator_index: ValidatorIndex) -> bool: - assert state.slot == slot - - return get_beacon_proposer_index(state) == validator_index -``` - -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. - - -### Lookahead - -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. - -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). - -Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. - ## How to avoid slashing "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. From 01efe52eb0d7523994b96ccb6e44b85929b0ad42 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:32:44 -0400 Subject: [PATCH 08/31] fix start shard --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 67a09c797..3b498bfd4 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,9 +149,9 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): - shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) if validator_index in committee: return committee, shard, slot From 2885f853c28d2442b4507b80d24471292a41ddb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 16:34:07 +0800 Subject: [PATCH 09/31] clean up lint --- .circleci/config.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 439afc9a9..3fe6643b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make pyspec && make lint + command: make install_lint && make lint workflows: version: 2.1 test_spec: diff --git a/Makefile b/Makefile index 8cc889f21..a6b379b71 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ citest: $(PY_SPEC_ALL_TARGETS) install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 -lint: +lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; From b0747703a77c36d821b6bf049aa8abc8e1603603 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:23:26 +0800 Subject: [PATCH 10/31] Make CI job `lint` require `test` --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fe6643b7..d2b284e25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo - install_test: + install_env: docker: - image: circleci/python:3.6 working_directory: ~/specs-repo @@ -64,7 +64,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test + command: make install_test && make install_lint - save_cached_venv: venv_name: v1-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' @@ -96,18 +96,18 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make lint + command: make lint workflows: version: 2.1 test_spec: jobs: - checkout_specs - - lint: - requires: - - checkout_specs - - install_test: + - install_env: requires: - checkout_specs - test: requires: - - install_test + - install_env + - lint: + requires: + - test From 1b3dfa67813e435761a71a3cc1c5e552efc043dd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:34:49 +0800 Subject: [PATCH 11/31] kick the cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b284e25..b9f09d9cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From 32f76641e34afb703355ab8b8ac93fafa0d59478 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:41:48 +0800 Subject: [PATCH 12/31] Add build_pyspec job --- .circleci/config.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9f09d9cc..a0700a004 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,19 @@ jobs: venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv + build_pyspec: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec-02 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Build pyspec + command: make pyspec test: docker: - image: circleci/python:3.6 @@ -105,9 +118,13 @@ workflows: - install_env: requires: - checkout_specs + - build_pyspec: + requires: + - checkout_specs - test: requires: - install_env + - build_pyspec - lint: requires: - - test + - build_pyspec From ee4fdf5d06c909b4d7bc6f1dff3bce5a253b54e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:45:09 +0800 Subject: [PATCH 13/31] Kick the cache again --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0700a004..972f0e351 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv build_pyspec: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Build pyspec @@ -90,7 +90,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -105,7 +105,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From be4c792fc301b7ee254b61466720e40e155b1601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 18:20:20 +0800 Subject: [PATCH 14/31] Change it back to serial workflow --- .circleci/config.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 972f0e351..f3c5f6a81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,19 +69,6 @@ jobs: venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv - build_pyspec: - docker: - - image: circleci/python:3.6 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - - run: - name: Build pyspec - command: make pyspec test: docker: - image: circleci/python:3.6 @@ -118,13 +105,9 @@ workflows: - install_env: requires: - checkout_specs - - build_pyspec: - requires: - - checkout_specs - test: requires: - install_env - - build_pyspec - lint: requires: - - build_pyspec + - test From 7f6896cca3931c0e4a6b352bfd0ba3fa3dca3212 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 16 May 2019 16:43:10 -0700 Subject: [PATCH 15/31] Update to use union type --- specs/networking/rpc-interface.md | 34 ++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index ca6008a40..48ee9333b 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,36 +39,24 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| method_id (uint16) | body_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | body (body_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + method_id: uint16 + body: (message_body...) +) ``` and their corresponding responses are wrapped in a "response" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| response_code (uint16) | result_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | result (result_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + response_code: uint16 + result: bytes +) ``` -Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. +A union type is used to determine the contents of the `body` field in the request structure. Each "body" entry in the RPC calls below corresponds to one subtype in the `body` type union. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From 05f9dc7baa356402a5a52b2bc0f910340ce75c52 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 05:59:01 -0400 Subject: [PATCH 16/31] Fix #1090 Avoid signed integer --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ee37d9217..aa8cf57ca 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -887,7 +887,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - # See the 'generalized domain' algorithm on page 3 for round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(round, length=1))[0:8]) % index_count - flip = (pivot - index) % index_count + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] From f19188816b2bac23b41222181be1bde31fdc79d1 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:07:38 -0400 Subject: [PATCH 17/31] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index aa8cf57ca..622eabaf8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1362,19 +1362,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` From 694b31b934d13ffbddf17574fa7f04b824d67c98 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:11:39 -0400 Subject: [PATCH 18/31] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 622eabaf8..b75f14153 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -647,7 +647,7 @@ def get_previous_epoch(state: BeaconState) -> Epoch: Return the current epoch if it's genesis epoch. """ current_epoch = get_current_epoch(state) - return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else current_epoch - 1 ``` ### `get_current_epoch` From 37aca60fae778268841a3e64253b85bb616b61ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:55:07 -0400 Subject: [PATCH 19/31] pr feedback --- specs/validator/0_beacon-chain-validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 5c92c9953..46eb55173 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -165,11 +165,11 @@ def is_proposer(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. +*Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). @@ -181,7 +181,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). From acb7444184b533de6c72fc65be1b0f7c506414c3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:58:02 -0400 Subject: [PATCH 20/31] pr feedback --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 46eb55173..302deeae1 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -157,7 +157,7 @@ def get_committee_assignment( return committee, shard, slot ``` -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. ```python def is_proposer(state: BeaconState, @@ -171,7 +171,7 @@ def is_proposer(state: BeaconState, The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments by noting at which future slot they will have to attest and also which shard they should begin syncing (in Phase 1+). Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. From e1d973d5467e25fe496cc67ba3800aab096d1c54 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:01:18 -0400 Subject: [PATCH 21/31] Update specs/validator/0_beacon-chain-validator.md Co-Authored-By: Hsiao-Wei Wang --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae1..9e4fa7960 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -148,7 +148,7 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT From 174e1e4dbef7605f1e4131f66d3fd0f52276b6a4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:04:05 -0400 Subject: [PATCH 22/31] pr feedback --- specs/validator/0_beacon-chain-validator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae1..0940d592a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,7 +149,8 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) From 24edca3456166c11ebce912d3397c1618b79a3dc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 17 May 2019 13:52:23 -0400 Subject: [PATCH 23/31] Fix to make Danny and hww happy --- specs/core/1_custody-game.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e03e54ed0..e28b30794 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -689,9 +689,7 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch == FAR_FUTURE_EPOCH: - validator.withdrawable_epoch = FAR_FUTURE_EPOCH - else: + if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 85c16544566f82be57c5905ccf3b7b48e2888d1d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 09:33:01 -0400 Subject: [PATCH 24/31] Crosslinks store start and end epoch Solves #1034 --- specs/core/0_beacon-chain.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153..643985e85 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -281,8 +281,9 @@ The types are defined topologically to aid in facilitating an executable version { # Shard number 'shard': 'uint64', - # Epoch number - 'epoch': 'uint64', + # Crosslinking data from epochs [start....end-1] + 'start_epoch': 'uint64', + 'end_epoch': 'uint64', # Root of the previous crosslink 'parent_root': 'bytes32', # Root of the crosslinked shard data since the previous crosslink @@ -1728,7 +1729,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) - assert data.crosslink.epoch == min(data.target_epoch, parent_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.start_epoch == parent_crosslink.end_epoch + assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From a2108741e804046b1ba721676482091a27f0e0fb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 15:47:59 -0600 Subject: [PATCH 25/31] fix tests with new starT_epoch and end_epoch in Crosslink --- test_libs/pyspec/eth2spec/utils/bls_stub.py | 2 +- .../tests/block_processing/test_process_attestation.py | 2 +- test_libs/pyspec/tests/helpers.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py index 108c4ef71..ae97de175 100644 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ b/test_libs/pyspec/eth2spec/utils/bls_stub.py @@ -9,4 +9,4 @@ def bls_verify_multiple(pubkeys, message_hashes, signature, domain): def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 + return b'\x42' * 48 diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 708d68dca..6851561e9 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -124,7 +124,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 + state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 pre_state, post_state = run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 5ddb2dc15..7af210f85 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -177,6 +177,7 @@ def build_attestation_data(state, slot, shard): justified_block_root = state.current_justified_root crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks + parent_crosslink = crosslinks[shard] return AttestationData( beacon_block_root=block_root, source_epoch=justified_epoch, @@ -185,9 +186,10 @@ def build_attestation_data(state, slot, shard): target_root=epoch_boundary_root, crosslink=Crosslink( shard=shard, - epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), + start_epoch=parent_crosslink.end_epoch, + end_epoch=min(slot_to_epoch(slot), parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK), data_root=spec.ZERO_HASH, - parent_root=hash_tree_root(crosslinks[shard]), + parent_root=hash_tree_root(parent_crosslink), ), ) From 62f8d19ffc2f80763e7dfef5a66a19782080d5c4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:06:10 -0600 Subject: [PATCH 26/31] add some attestation crosslink tests --- .../test_process_attestation.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 6851561e9..97eddb902 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -64,6 +64,22 @@ def test_success_prevous_epoch(state): return pre_state, attestation, post_state +def test_success_since_max_epochs_per_crosslink(state): + for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): + next_epoch(state) + + attestation = get_valid_attestation(state) + data = attestation.data + assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK + + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state + + def test_before_inclusion_delay(state): attestation = get_valid_attestation(state) # do not increment slot to allow for inclusion delay @@ -131,6 +147,32 @@ def test_bad_previous_crosslink(state): return pre_state, attestation, post_state +def test_bad_crosslink_start_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.start_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_crosslink_end_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.end_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 4c5e0548833bd3b4d92c22806afce48a1b4c1ab3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:11:39 -0600 Subject: [PATCH 27/31] fix previous crosslink root test --- .../pyspec/tests/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 97eddb902..763178717 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -140,7 +140,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 + attestation.data.crosslink.parent_root = b'\x27' * 32 pre_state, post_state = run_attestation_processing(state, attestation, False) From c14452bcf445761926da99a1259f0a792268bdff Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 19:44:12 -0400 Subject: [PATCH 28/31] Updated get_custody_chunk_count Co-requisite with #1097 --- specs/core/1_custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0e8fcf6e9..7965e25f9 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -255,8 +255,8 @@ The `empty` function accepts and SSZ type as input and returns an object of that ```python def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.epoch - crosslink_end_epoch = slot_to_epoch(attestation.data.slot) + crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch + crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK return crosslink_crosslink_length * chunks_per_epoch From a68aa82b894153eca18f290d306623bf4118357b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 11:39:13 +0800 Subject: [PATCH 29/31] Update validator guide --- specs/validator/0_beacon-chain-validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0940d592a..d05f25ef2 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -306,10 +306,12 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### Crosslink vote -Construct `attestation_data.crosslink` via the following +Construct `attestation_data.crosslink` via the following. * Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. -* Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. +* Let `parent_crosslink = head_state.current_crosslinks[shard]`. +* Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. +* Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. From 2018dd83f5a324472623db15a159be2882ffe8fa Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 20 May 2019 09:29:09 +0100 Subject: [PATCH 30/31] Update 1_custody-game.md --- specs/core/1_custody-game.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 7965e25f9..f91fe81de 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -254,12 +254,10 @@ The `empty` function accepts and SSZ type as input and returns an object of that ### `get_crosslink_chunk_count` ```python -def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch - crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch - crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) +def get_custody_chunk_count(crosslink: Crosslink) -> int: + crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, crosslink.end_epoch - crosslink.start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK - return crosslink_crosslink_length * chunks_per_epoch + return crosslink_length * chunks_per_epoch ``` ### `get_custody_chunk_bit` @@ -470,7 +468,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) + depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation.data.crosslink))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -544,7 +542,7 @@ def process_bit_challenge(state: BeaconState, ) # Verify the chunk count - chunk_count = get_custody_chunk_count(challenge.attestation) + chunk_count = get_custody_chunk_count(challenge.attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) From 83123a33da792397101bdee263d03a25ee281e0b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 17:16:20 +0800 Subject: [PATCH 31/31] Set genesis_state.latest_block_header with `body_root` of empty BeaconBlockBody (#1098) --- specs/core/0_beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153..b34bbe7d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1189,7 +1189,11 @@ Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.gene ```python def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: - state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) + state = BeaconState( + genesis_time=genesis_time, + latest_eth1_data=genesis_eth1_data, + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + ) # Process genesis deposits for deposit in deposits: