Merge pull request #808 from ethereum/vbuterin-patch-18

Replace committee exponential backoff with max progress
This commit is contained in:
Danny Ryan 2019-03-26 13:03:00 -06:00 committed by GitHub
commit 8550d7597a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 241 additions and 194 deletions

View File

@ -68,9 +68,7 @@
- [`get_split_offset`](#get_split_offset)
- [`get_epoch_committee_count`](#get_epoch_committee_count)
- [`compute_committee`](#compute_committee)
- [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count)
- [`get_current_epoch_committee_count`](#get_current_epoch_committee_count)
- [`get_next_epoch_committee_count`](#get_next_epoch_committee_count)
- [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot)
- [`get_block_root`](#get_block_root)
- [`get_state_root`](#get_state_root)
@ -80,7 +78,6 @@
- [`get_beacon_proposer_index`](#get_beacon_proposer_index)
- [`verify_merkle_branch`](#verify_merkle_branch)
- [`get_attestation_participants`](#get_attestation_participants)
- [`is_power_of_two`](#is_power_of_two)
- [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-)
- [`bytes_to_int`](#bytes_to_int)
- [`get_effective_balance`](#get_effective_balance)
@ -236,6 +233,10 @@ Code snippets appearing in `this style` are to be interpreted as Python code.
| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours |
| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours |
| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days |
| `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) |
* `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`
### State list lengths
@ -594,12 +595,7 @@ The types are defined topologically to aid in facilitating an executable version
# Randomness and committees
'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH],
'previous_shuffling_start_shard': 'uint64',
'current_shuffling_start_shard': 'uint64',
'previous_shuffling_epoch': 'uint64',
'current_shuffling_epoch': 'uint64',
'previous_shuffling_seed': 'bytes32',
'current_shuffling_seed': 'bytes32',
'latest_start_shard': 'uint64',
# Finality
'previous_epoch_attestations': [PendingAttestation],
@ -877,20 +873,6 @@ def compute_committee(validator_indices: List[ValidatorIndex],
**Note**: this definition and the next few definitions are highly inefficient as algorithms as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work.
### `get_previous_epoch_committee_count`
```python
def get_previous_epoch_committee_count(state: BeaconState) -> int:
"""
Return the number of committees in the previous epoch of the given ``state``.
"""
previous_active_validators = get_active_validator_indices(
state.validator_registry,
state.previous_shuffling_epoch,
)
return get_epoch_committee_count(len(previous_active_validators))
```
### `get_current_epoch_committee_count`
```python
@ -900,36 +882,18 @@ def get_current_epoch_committee_count(state: BeaconState) -> int:
"""
current_active_validators = get_active_validator_indices(
state.validator_registry,
state.current_shuffling_epoch,
get_current_epoch(state),
)
return get_epoch_committee_count(len(current_active_validators))
```
### `get_next_epoch_committee_count`
```python
def get_next_epoch_committee_count(state: BeaconState) -> int:
"""
Return the number of committees in the next epoch of the given ``state``.
"""
next_active_validators = get_active_validator_indices(
state.validator_registry,
get_current_epoch(state) + 1,
)
return get_epoch_committee_count(len(next_active_validators))
```
### `get_crosslink_committees_at_slot`
```python
def get_crosslink_committees_at_slot(state: BeaconState,
slot: Slot,
registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]:
slot: Slot) -> List[Tuple[List[ValidatorIndex], Shard]]:
"""
Return the list of ``(committee, shard)`` tuples for the ``slot``.
Note: There are two possible shufflings for crosslink committees for a
``slot`` in the next epoch -- with and without a `registry_change`
"""
epoch = slot_to_epoch(slot)
current_epoch = get_current_epoch(state)
@ -937,40 +901,24 @@ def get_crosslink_committees_at_slot(state: BeaconState,
next_epoch = current_epoch + 1
assert previous_epoch <= epoch <= next_epoch
indices = get_active_validator_indices(
state.validator_registry,
epoch,
)
committees_per_epoch = get_epoch_committee_count(len(indices))
if epoch == current_epoch:
committees_per_epoch = get_current_epoch_committee_count(state)
seed = state.current_shuffling_seed
shuffling_epoch = state.current_shuffling_epoch
shuffling_start_shard = state.current_shuffling_start_shard
start_shard = state.latest_start_shard
elif epoch == previous_epoch:
committees_per_epoch = get_previous_epoch_committee_count(state)
seed = state.previous_shuffling_seed
shuffling_epoch = state.previous_shuffling_epoch
shuffling_start_shard = state.previous_shuffling_start_shard
start_shard = (state.latest_start_shard - committees_per_epoch) % SHARD_COUNT
elif epoch == next_epoch:
epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch
if registry_change:
committees_per_epoch = get_next_epoch_committee_count(state)
seed = generate_seed(state, next_epoch)
shuffling_epoch = next_epoch
current_committees_per_epoch = get_current_epoch_committee_count(state)
shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT
elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update):
committees_per_epoch = get_next_epoch_committee_count(state)
seed = generate_seed(state, next_epoch)
shuffling_epoch = next_epoch
shuffling_start_shard = state.current_shuffling_start_shard
else:
committees_per_epoch = get_current_epoch_committee_count(state)
seed = state.current_shuffling_seed
shuffling_epoch = state.current_shuffling_epoch
shuffling_start_shard = state.current_shuffling_start_shard
current_epoch_committees = get_current_epoch_committee_count(state)
start_shard = (state.latest_start_shard + current_epoch_committees) % SHARD_COUNT
indices = get_active_validator_indices(state.validator_registry, shuffling_epoch)
committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH
offset = slot % SLOTS_PER_EPOCH
slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT
slot_start_shard = (start_shard + committees_per_slot * offset) % SHARD_COUNT
seed = generate_seed(state, epoch)
return [
(
@ -1114,16 +1062,6 @@ def get_attestation_participants(state: BeaconState,
return participants
```
### `is_power_of_two`
```python
def is_power_of_two(value: int) -> bool:
"""
Check if ``value`` is a power of two integer.
"""
return (value > 0) and (value & (value - 1) == 0)
```
### `int_to_bytes1`, `int_to_bytes2`, ...
`int_to_bytes1(x): return x.to_bytes(1, 'little')`, `int_to_bytes2(x): return x.to_bytes(2, 'little')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96.
@ -1573,12 +1511,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit],
# Randomness and committees
latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]),
previous_shuffling_start_shard=GENESIS_START_SHARD,
current_shuffling_start_shard=GENESIS_START_SHARD,
previous_shuffling_epoch=GENESIS_EPOCH - 1,
current_shuffling_epoch=GENESIS_EPOCH,
previous_shuffling_seed=ZERO_HASH,
current_shuffling_seed=ZERO_HASH,
latest_start_shard=GENESIS_START_SHARD,
# Finality
previous_epoch_attestations=[],
@ -1618,7 +1551,6 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit],
genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH))
for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH):
state.latest_active_index_roots[index] = genesis_active_index_root
state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH)
return state
```
@ -1893,7 +1825,7 @@ Run the following function:
```python
def process_crosslinks(state: BeaconState) -> None:
current_epoch = get_current_epoch(state)
previous_epoch = current_epoch - 1
previous_epoch = max(current_epoch - 1, GENESIS_EPOCH)
next_epoch = current_epoch + 1
for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)):
for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot):
@ -1902,7 +1834,7 @@ def process_crosslinks(state: BeaconState) -> None:
total_balance = get_total_balance(state, crosslink_committee)
if 3 * participating_balance >= 2 * total_balance:
state.latest_crosslinks[shard] = Crosslink(
epoch=slot_to_epoch(slot),
epoch=min(slot_to_epoch(slot), state.latest_crosslinks[shard].epoch + MAX_CROSSLINK_EPOCHS),
crosslink_data_root=winning_root
)
```
@ -2102,22 +2034,6 @@ def process_ejections(state: BeaconState) -> None:
#### Validator registry and shuffling seed data
```python
def should_update_validator_registry(state: BeaconState) -> bool:
# Must have finalized a new block
if state.finalized_epoch <= state.validator_registry_update_epoch:
return False
# Must have processed new crosslinks on all shards of the current epoch
shards_to_check = [
(state.current_shuffling_start_shard + i) % SHARD_COUNT
for i in range(get_current_epoch_committee_count(state))
]
for shard in shards_to_check:
if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch:
return False
return True
```
```python
def update_validator_registry(state: BeaconState) -> None:
"""
@ -2171,30 +2087,14 @@ def update_validator_registry(state: BeaconState) -> None:
Run the following function:
```python
def update_registry_and_shuffling_data(state: BeaconState) -> None:
# First set previous shuffling data to current shuffling data
state.previous_shuffling_epoch = state.current_shuffling_epoch
state.previous_shuffling_start_shard = state.current_shuffling_start_shard
state.previous_shuffling_seed = state.current_shuffling_seed
current_epoch = get_current_epoch(state)
next_epoch = current_epoch + 1
def update_registry(state: BeaconState) -> None:
# Check if we should update, and if so, update
if should_update_validator_registry(state):
if state.finalized_epoch > state.validator_registry_update_epoch:
update_validator_registry(state)
# If we update the registry, update the shuffling data and shards as well
state.current_shuffling_epoch = next_epoch
state.current_shuffling_start_shard = (
state.current_shuffling_start_shard +
state.latest_start_shard = (
state.latest_start_shard +
get_current_epoch_committee_count(state)
) % SHARD_COUNT
state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch)
else:
# If processing at least one crosslink keeps failing, then reshuffle every power of two,
# but don't update the current_shuffling_start_shard
epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch
if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update):
state.current_shuffling_epoch = next_epoch
state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch)
```
**Invariant**: the active index root that is hashed into the shuffling seed actually is the `hash_tree_root` of the validator set that is used for that epoch.
@ -2445,7 +2345,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
attestation.data.previous_crosslink, # Case 1: latest crosslink matches previous crosslink
Crosslink( # Case 2: latest crosslink matches current crosslink
crosslink_data_root=attestation.data.crosslink_data_root,
epoch=target_epoch,
epoch=min(slot_to_epoch(attestation.data.slot),
attestation.data.previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS)
),
}

View File

@ -331,16 +331,15 @@ signed_attestation_data = bls_sign(
## Validator assignments
A validator can get the current and previous epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= current_epoch`.
A validator can get the current, previous, and next epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= next_epoch`.
```python
def get_committee_assignment(
state: BeaconState,
epoch: Epoch,
validator_index: ValidatorIndex,
registry_change: bool=False) -> Tuple[List[ValidatorIndex], Shard, Slot]:
validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]:
"""
Return the committee assignment in the ``epoch`` for ``validator_index`` and ``registry_change``.
Return the committee assignment in the ``epoch`` for ``validator_index``.
``assignment`` returned is a tuple of the following form:
* ``assignment[0]`` is the list of validators in the committee
* ``assignment[1]`` is the shard to which the committee is assigned
@ -355,7 +354,6 @@ def get_committee_assignment(
crosslink_committees = get_crosslink_committees_at_slot(
state,
slot,
registry_change=registry_change,
)
selected_committees = [
committee # Tuple[List[ValidatorIndex], Shard]
@ -389,18 +387,9 @@ _Note_: If a validator is assigned to the 0th slot of an epoch, the validator mu
The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing which must checked during the epoch in question.
There are three possibilities for the shuffling at the next epoch:
1. The shuffling changes due to a "validator registry change".
2. The shuffling changes due to `epochs_since_last_registry_update` being an exact power of 2 greater than 1.
3. The shuffling remains the same (i.e. the validator is in the same shard committee).
`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in phase 1+).
Either (2) or (3) occurs if (1) fails. The choice between (2) and (3) is deterministic based upon `epochs_since_last_registry_update`.
When querying for assignments in the next epoch there are two options -- with and without a `registry_change` -- which is the optional fourth parameter of the `get_committee_assignment`.
`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should always plan for assignments from both values of `registry_change` unless the validator can concretely eliminate one of the options. Planning for future assignments involves noting at which future slot one might have to attest and also which shard one should begin syncing (in phase 1+).
Specifically, a validator should call both `get_committee_assignment(state, next_epoch, validator_index, registry_change=True)` and `get_committee_assignment(state, next_epoch, validator_index, registry_change=False)` when checking for next epoch assignments.
Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments.
## How to avoid slashing

View File

@ -0,0 +1,153 @@
from copy import deepcopy
import pytest
import build.phase0.spec as spec
from build.phase0.state_transition import (
state_transition,
)
from build.phase0.spec import (
ZERO_HASH,
get_current_epoch,
process_attestation,
slot_to_epoch,
)
from tests.phase0.helpers import (
build_empty_block_for_next_slot,
get_valid_attestation,
)
# mark entire file as 'attestations'
pytestmark = pytest.mark.attestations
def run_attestation_processing(state, attestation, valid=True):
"""
Run ``process_attestation`` returning the pre and post state.
If ``valid == False``, run expecting ``AssertionError``
"""
post_state = deepcopy(state)
if not valid:
with pytest.raises(AssertionError):
process_attestation(post_state, attestation)
return state, None
process_attestation(post_state, attestation)
current_epoch = get_current_epoch(state)
target_epoch = slot_to_epoch(attestation.data.slot)
if target_epoch == current_epoch:
assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1
else:
assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1
return state, post_state
def test_success(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
pre_state, post_state = run_attestation_processing(state, attestation)
return pre_state, attestation, post_state
def test_success_prevous_epoch(state):
attestation = get_valid_attestation(state)
block = build_empty_block_for_next_slot(state)
block.slot = state.slot + spec.SLOTS_PER_EPOCH
state_transition(state, block)
pre_state, post_state = run_attestation_processing(state, attestation)
return pre_state, attestation, post_state
def test_before_inclusion_delay(state):
attestation = get_valid_attestation(state)
# do not increment slot to allow for inclusion delay
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_after_epoch_slots(state):
attestation = get_valid_attestation(state)
block = build_empty_block_for_next_slot(state)
# increment past latest inclusion slot
block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1
state_transition(state, block)
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_bad_source_epoch(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
attestation.data.source_epoch += 10
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_bad_source_root(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
attestation.data.source_root = b'\x42'*32
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_non_zero_crosslink_data_root(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
attestation.data.crosslink_data_root = b'\x42'*32
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_bad_previous_crosslink(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
state.latest_crosslinks[attestation.data.shard].epoch += 10
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_non_empty_custody_bitfield(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
attestation.custody_bitfield = b'\x01' + attestation.custody_bitfield[1:]
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state
def test_empty_aggregation_bitfield(state):
attestation = get_valid_attestation(state)
state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield)
pre_state, post_state = run_attestation_processing(state, attestation, False)
return pre_state, attestation, post_state

View File

@ -9,7 +9,9 @@ from build.phase0.spec import (
EMPTY_SIGNATURE,
ZERO_HASH,
# SSZ
Attestation,
AttestationData,
AttestationDataAndCustodyBit,
BeaconBlockHeader,
Deposit,
DepositData,
@ -18,7 +20,9 @@ from build.phase0.spec import (
VoluntaryExit,
# functions
get_active_validator_indices,
get_attestation_participants,
get_block_root,
get_crosslink_committees_at_slot,
get_current_epoch,
get_domain,
get_empty_block,
@ -236,3 +240,49 @@ def get_valid_proposer_slashing(state):
header_1=header_1,
header_2=header_2,
)
def get_valid_attestation(state, slot=None):
if slot is None:
slot = state.slot
shard = state.latest_start_shard
attestation_data = build_attestation_data(state, slot, shard)
crosslink_committees = get_crosslink_committees_at_slot(state, slot)
crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0]
committee_size = len(crosslink_committee)
bitfield_length = (committee_size + 7) // 8
aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1)
custody_bitfield = b'\x00' * bitfield_length
attestation = Attestation(
aggregation_bitfield=aggregation_bitfield,
data=attestation_data,
custody_bitfield=custody_bitfield,
aggregate_signature=EMPTY_SIGNATURE,
)
participants = get_attestation_participants(
state,
attestation.data,
attestation.aggregation_bitfield,
)
assert len(participants) == 1
validator_index = participants[0]
privkey = privkeys[validator_index]
message_hash = AttestationDataAndCustodyBit(
data=attestation.data,
custody_bit=0b0,
).hash_tree_root()
attestation.aggregation_signature = bls.sign(
message_hash=message_hash,
privkey=privkey,
domain=get_domain(
fork=state.fork,
epoch=get_current_epoch(state),
domain_type=spec.DOMAIN_ATTESTATION,
)
)
return attestation

View File

@ -11,19 +11,13 @@ from build.phase0.spec import (
EMPTY_SIGNATURE,
ZERO_HASH,
# SSZ
Attestation,
AttestationDataAndCustodyBit,
BeaconBlockHeader,
Deposit,
Transfer,
ProposerSlashing,
VoluntaryExit,
# functions
get_active_validator_indices,
get_attestation_participants,
get_balance,
get_block_root,
get_crosslink_committees_at_slot,
get_current_epoch,
get_domain,
get_state_root,
@ -42,10 +36,10 @@ from build.phase0.utils.merkle_minimal import (
get_merkle_root,
)
from tests.phase0.helpers import (
build_attestation_data,
build_deposit_data,
build_empty_block_for_next_slot,
force_registry_change_at_next_epoch,
get_valid_attestation,
get_valid_proposer_slashing,
privkeys,
pubkeys,
@ -222,47 +216,7 @@ def test_deposit_top_up(state):
def test_attestation(state):
test_state = deepcopy(state)
slot = state.slot
shard = state.current_shuffling_start_shard
attestation_data = build_attestation_data(state, slot, shard)
crosslink_committees = get_crosslink_committees_at_slot(state, slot)
crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0]
committee_size = len(crosslink_committee)
bitfield_length = (committee_size + 7) // 8
aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1)
custody_bitfield = b'\x00' * bitfield_length
attestation = Attestation(
aggregation_bitfield=aggregation_bitfield,
data=attestation_data,
custody_bitfield=custody_bitfield,
aggregate_signature=EMPTY_SIGNATURE,
)
participants = get_attestation_participants(
test_state,
attestation.data,
attestation.aggregation_bitfield,
)
assert len(participants) == 1
validator_index = participants[0]
privkey = privkeys[validator_index]
message_hash = AttestationDataAndCustodyBit(
data=attestation.data,
custody_bit=0b0,
).hash_tree_root()
attestation.aggregation_signature = bls.sign(
message_hash=message_hash,
privkey=privkey,
domain=get_domain(
fork=test_state.fork,
epoch=get_current_epoch(test_state),
domain_type=spec.DOMAIN_ATTESTATION,
)
)
attestation = get_valid_attestation(state)
#
# Add to state via block transition

View File

@ -95,7 +95,7 @@ def process_epoch_transition(state: BeaconState) -> None:
spec.maybe_reset_eth1_period(state)
spec.apply_rewards(state)
spec.process_ejections(state)
spec.update_registry_and_shuffling_data(state)
spec.update_registry(state)
spec.process_slashings(state)
spec.process_exit_queue(state)
spec.finish_epoch_update(state)