From c5143ccefaddd9929bfb68401c6c00e1afa75e6d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:48:10 -0600 Subject: [PATCH 001/110] modify fork choice to utilize epochs as first class citizens --- specs/core/0_fork-choice.md | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6b7e4bdbe..3bcb6b889 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -63,17 +63,30 @@ class Target(object): root: Hash ``` +#### `RootSlot` + +```python +@dataclass +class RootSlot(object): + slot: Slot + root: Hash +``` + + + #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[Hash, BeaconState] = field(default_factory=dict) + states: Dict[RootSlot, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) justified_root: Hash = ZERO_HASH + justified_epoch: Epoch = 0 finalized_root: Hash = ZERO_HASH + finalized_epoch: Epoch = 0 ``` #### `get_genesis_store` @@ -98,8 +111,8 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[store.justified_root] - active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root @@ -112,8 +125,12 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_root + justified_slot = get_epoch_start_slot(store.justified_epoch) while True: - children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == head and store.blocks[root].slot > justified_slot + ] if len(children) == 0: return head # Sort by latest attesting balance with ties broken lexicographically @@ -137,30 +154,58 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check that block is later than the finalized epoch slot + assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time pre_state = deepcopy(store.states[block.parent_root]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) - # Add new state to the store - store.states[signing_root(block)] = state - # Update justified block root - if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + # Add new state for this block to the store + store.states[RootSlot(signing_root(block), block.slot)] = state + # Update justified block root and epoch + previous_justified_epoch = store.justified_epoch + justified_root_slot = None + if state.current_justified_epoch > store.justified_epoch: store.justified_root = state.current_justified_root - elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + state.justified_epoch = state.current_justified_epoch + elif state.previous_justified_epoch > store.justified_epoch: store.justified_root = state.previous_justified_root - # Update finalized block root - if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + state.justified_epoch = state.previous_justified_epoch + + # Store justified state + if previous_justified_epoch != store.justified_epoch: + justified_slot = get_epoch_start_slot(store.justified_epoch) + justified_root_slot = RootSlot(store.justified_root, justified_slot) + if justified_root_slot not in store.states: + base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] + store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) + + # Update finalized block root and epoch, and store finalized state + if state.finalized_epoch > state.finalized_epoch: store.finalized_root = state.finalized_root + store.finalized_epoch = state.finalized_epoch + finalized_slot = get_epoch_start_slot(store.finalized_epoch) + finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) + if finalized_root_slot not in store.states: + base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] + store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - state = store.states[get_head(store)] + # cannot calculate the current shuffling if have not seen the target + assert attestation.data.target_root in store.blocks + + # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees + state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) + + # update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) From fb9a5f0bc5e2358fbb2a57f3324b83ca2f82ed07 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:57:53 -0600 Subject: [PATCH 002/110] one more rootslot fix --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3bcb6b889..668b20714 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -157,7 +157,8 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check that block is later than the finalized epoch slot assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time - pre_state = deepcopy(store.states[block.parent_root]) + parent_block = store.blocks[block.parent_root] + pre_state = deepcopy(store.states[RootSlot(signing_root(parent_block), parent_block.slot)]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) From 1b66a1a2bd297d33f666deede5b403388197073a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 12:55:55 -0600 Subject: [PATCH 003/110] rework forkchoice to use Checkpoints --- specs/core/0_fork-choice.md | 104 +++++++++++++++--------------------- 1 file changed, 42 insertions(+), 62 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index ee318412d..1755c6df3 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -12,7 +12,7 @@ - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [`Target`](#target) + - [`Checkpoint`](#checkpoint) - [`Store`](#store) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) @@ -55,39 +55,27 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Target` +#### `Checkpoint` ```python @dataclass -class Target(object): +class Checkpoint(object): epoch: Epoch root: Hash ``` -#### `RootSlot` - -```python -@dataclass -class RootSlot(object): - slot: Slot - root: Hash -``` - - - #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[RootSlot, BeaconState] = field(default_factory=dict) - time: int = 0 - latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Hash = ZERO_HASH - justified_epoch: Epoch = 0 - finalized_root: Hash = ZERO_HASH - finalized_epoch: Epoch = 0 + block_states: Dict[Hash, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + time: int + latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -96,12 +84,15 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) + justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) + finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( blocks={root: genesis_block}, - states={root: genesis_state}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, - justified_root=root, - finalized_root=root, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, ) ``` @@ -118,7 +109,7 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices @@ -131,8 +122,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: ```python def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice - head = store.justified_root - justified_slot = get_epoch_start_slot(store.justified_epoch) + head = store.justified_checkpoint.root + justified_slot = get_epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -159,63 +150,52 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.states[RootSlot(signing_root(parent_block), parent_block.slot)].copy() + pre_state = store.block_states[parent_block.root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot - assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) + assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store - store.states[RootSlot(signing_root(block), block.slot)] = state - # Update justified block root and epoch - previous_justified_epoch = store.justified_epoch - justified_root_slot = None - if state.current_justified_epoch > store.justified_epoch: - store.justified_root = state.current_justified_root - state.justified_epoch = state.current_justified_epoch + store.block_states[signing_root(block)] = state + + # Update justified checkpoint + if state.current_justified_epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) elif state.previous_justified_epoch > store.justified_epoch: - store.justified_root = state.previous_justified_root - state.justified_epoch = state.previous_justified_epoch + store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) - # Store justified state - if previous_justified_epoch != store.justified_epoch: - justified_slot = get_epoch_start_slot(store.justified_epoch) - justified_root_slot = RootSlot(store.justified_root, justified_slot) - if justified_root_slot not in store.states: - base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] - store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) - - # Update finalized block root and epoch, and store finalized state + # Update finalized checkpoint if state.finalized_epoch > state.finalized_epoch: - store.finalized_root = state.finalized_root - store.finalized_epoch = state.finalized_epoch - finalized_slot = get_epoch_start_slot(store.finalized_epoch) - finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) - if finalized_root_slot not in store.states: - base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] - store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) + store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - # cannot calculate the current shuffling if have not seen the target - assert attestation.data.target_root in store.blocks + target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) - # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees - state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + # Cannot calculate the current shuffling if have not seen the target + assert target_checkpoint.root in store.blocks + # Store target checkpoint state if not yet seen + if target_checkpoint not in store.checkpoint_states: + base_state = store.block_states[target_checkpoint.root].copy() + store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + + # Get state at the `target_checkpoint` to validate attestation and calculate the committees + state = store.checkpoint_states[target_checkpoint] indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) - # update latest targets + # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) + if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target_checkpoint ``` From de5b9cc823dedd9673f3599bf8e209a363cf0dd7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 16:33:43 -0600 Subject: [PATCH 004/110] convert _root/_epoch tuples to Checkpoint obj in beacon spec --- specs/core/0_beacon-chain.md | 92 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5081dbf74..cf1d36862 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -26,6 +26,7 @@ - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - [`Validator`](#validator) + - [`Checkpoint`](#checkpoint) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) @@ -303,6 +304,14 @@ class Fork(Container): epoch: Epoch # Epoch of latest fork ``` +#### `Checkpoint` + +```python +class Checkpoint(Container): + epoch: Epoch + root: Hash +``` + #### `Validator` ```python @@ -337,10 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_epoch: Epoch - source_root: Hash - target_epoch: Epoch - target_root: Hash + source_checkpoint: Checkpoint + target_checkpoint: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -534,14 +541,11 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Justification - previous_justified_epoch: Epoch # Previous epoch snapshot - previous_justified_root: Hash # Previous epoch snapshot - current_justified_epoch: Epoch - current_justified_root: Hash + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint justification_bitfield: uint64 # Bit set for every recent justified epoch # Finality - finalized_epoch: Epoch - finalized_root: Hash + finalized_checkpoint: Checkpoint ``` ## Helper functions @@ -715,9 +719,10 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - committee_count = get_epoch_committee_count(state, data.target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + target_epoch = data.target_checkpoint.epoch + committee_count = get_epoch_committee_count(state, target_epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -876,12 +881,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, - attestation_data: AttestationData, + data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) + committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1001,7 +1006,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), ) ``` @@ -1014,9 +1019,12 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_epoch == data_2.target_epoch) or + (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or # Surround vote - (data_1.source_epoch < data_2.source_epoch and data_2.target_epoch < data_1.target_epoch) + ( + data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and + data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch + ) ) ``` @@ -1274,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_root == get_block_root(state, epoch) + if a.data.target_checkpoint.root == get_block_root(state, epoch) ] ``` @@ -1326,46 +1334,39 @@ def process_justification_and_finalization(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) current_epoch = get_current_epoch(state) - old_previous_justified_epoch = state.previous_justified_epoch - old_current_justified_epoch = state.current_justified_epoch + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint # Process justifications - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root + state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = previous_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = current_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint ``` #### Crosslinks @@ -1428,7 +1429,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards[index] += Gwei(max_attester_reward * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) # Inactivity penalty - finality_delay = previous_epoch - state.finalized_epoch + finality_delay = previous_epoch - state.finalized_checkpoint.epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in eligible_validator_indices: @@ -1493,7 +1494,7 @@ def process_registry_updates(state: BeaconState) -> None: activation_queue = sorted([ index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) + validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: @@ -1691,9 +1692,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data + target_epoch = data.target_checkpoint.epoch assert data.crosslink.shard < SHARD_COUNT - assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1705,19 +1707,19 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if data.target_epoch == get_current_epoch(state): - ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) + if target_epoch == get_current_epoch(state): + ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) + ffg_source = state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG data, crosslink data, and signature - assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) + # Check FFG source, crosslink data, and signature + assert ffg_source == data.source_checkpoint assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From a5b22e13b8b4bf8247b2e130b12917eee026e0f6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 22 Jun 2019 16:56:16 +0200 Subject: [PATCH 005/110] Resolves make masker sign mask --- specs/core/1_custody-game.md | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 13 +++++++++---- .../test_process_early_derived_secret_reveal.py | 16 +++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07..b051d13c6 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -204,7 +204,7 @@ class EarlyDerivedSecretReveal(Container): # Index of the validator who revealed (whistleblower) masker_index: ValidatorIndex # Mask used to hide the actual reveal signature (prevent reveal from being stolen) - mask: Bytes32 + mask: Hash ``` ### Phase 0 container updates diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcd..0167edc6a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -10,6 +10,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): if epoch is None: epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + # Generate the secret that is being revealed reveal = bls_sign( message_hash=spec.hash_tree_root(epoch), privkey=privkeys[revealed_index], @@ -19,8 +20,11 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + # Generate the mask (any random 32 bytes will do) + mask = reveal[:32] + # Generate masker's signature on the mask + masker_signature = bls_sign( + message_hash=mask, privkey=privkeys[masker_index], domain=spec.get_domain( state=state, @@ -28,11 +32,12 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) + masked_reveal = bls_aggregate_signatures([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, epoch=epoch, - reveal=reveal, + reveal=masked_reveal, masker_index=masker_index, mask=mask, ) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 87297d443..831ad35a5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,7 +1,13 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.context import with_all_phases_except, spec_state_test, expect_assertion_error +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, + always_bls, + never_bls, +) def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): @@ -36,6 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_success(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) @@ -44,6 +51,7 @@ def test_success(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_current_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -52,6 +60,7 @@ def test_reveal_from_current_epoch(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_past_epoch(spec, state): next_epoch(spec, state) @@ -62,6 +71,7 @@ def test_reveal_from_past_epoch(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -73,6 +83,7 @@ def test_reveal_with_custody_padding(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding_minus_one(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -84,6 +95,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_double_reveal(spec, state): randao_key_reveal1 = get_valid_early_derived_secret_reveal( @@ -108,6 +120,7 @@ def test_double_reveal(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -117,6 +130,7 @@ def test_revealer_is_slashed(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_far_future_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( From 8a732fbbcfbfae529cc34458681a965e135a36f8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 22 Jun 2019 12:00:26 -0600 Subject: [PATCH 006/110] pass on checkpoints working in testing --- specs/core/0_beacon-chain.md | 12 +++---- .../eth2spec/test/helpers/attestations.py | 22 ++++++------- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 31 ++++++++--------- .../test_process_attester_slashing.py | 10 +++--- .../test_process_crosslinks.py | 4 +-- .../pyspec/eth2spec/test/test_finality.py | 33 ++++++++----------- 7 files changed, 52 insertions(+), 62 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf1d36862..838178ea4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1344,28 +1344,28 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7eb..d2ce6e29b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -24,11 +24,11 @@ def build_attestation_data(spec, state, slot, shard): epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root + source_epoch = state.previous_justified_checkpoint.epoch + source_root = state.previous_justified_checkpoint.root else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root + source_epoch = state.current_justified_checkpoint.epoch + source_root = state.current_justified_checkpoint.root if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): parent_crosslink = state.current_crosslinks[shard] @@ -37,10 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=spec.slot_to_epoch(slot), - target_root=epoch_boundary_root, + source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), + target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -64,8 +62,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_epoch, - attestation_data.crosslink.shard + attestation_data.target_checkpoint.epoch, + attestation_data.crosslink.shard, ) committee_size = len(crosslink_committee) @@ -126,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, + message_epoch=attestation_data.target_checkpoint.epoch, ) ) @@ -134,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 9fd34520c..e339421e8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 + attestation_2.data.target_checkpoint.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..9a718b071 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_epoch == spec.get_current_epoch(state): + if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -119,16 +119,16 @@ def test_after_epoch_slots(spec, state): @spec_state_test def test_old_source_epoch(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 - state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.current_justified_epoch = 4 + state.finalized_checkpoint.epoch = 2 + state.previous_justified_checkpoint.epoch = 3 + state.current_justified_checkpoint.epoch = 4 attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_epoch == state.previous_justified_epoch + assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_epoch -= 1 + attestation.data.source_checkpoint.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_epoch += 1 + attestation.data.source_checkpoint.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = attestation.data.target_root + attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root sign_attestation(spec, state, attestation) @@ -180,21 +180,18 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.previous_justified_root = b'\x01' * 32 - - state.current_justified_epoch = 4 - state.current_justified_root = b'\xff' * 32 + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: - assert state.current_justified_root != state.previous_justified_root - assert attestation.data.source_root == state.previous_justified_root + assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root + assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_root = state.current_justified_root + attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -207,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = b'\x42' * 32 + attestation.data.source_checkpoint.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c51f5a8a9..71915f1d0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -68,12 +68,14 @@ def test_success_surround(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - state.current_justified_epoch += 1 + state.current_justified_checkpoint.epoch += 1 attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 - attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 + attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 + attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -120,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_epoch += 1 + attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index d51191efb..482085fd1 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_epoch, + attestation_2.data.target_checkpoint.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c8..e405312eb 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -14,25 +14,22 @@ def check_finality(spec, previous_justified_changed, finalized_changed): if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root + assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch + assert state.current_justified_checkpoint.root != prev_state.current_justified_checkpoint.root else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root + assert state.current_justified_checkpoint == prev_state.current_justified_checkpoint if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root + assert state.previous_justified_checkpoint.epoch > prev_state.previous_justified_checkpoint.epoch + assert state.previous_justified_checkpoint.root != prev_state.previous_justified_checkpoint.root else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root + assert state.previous_justified_checkpoint == prev_state.previous_justified_checkpoint if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root + assert state.finalized_checkpoint.epoch > prev_state.finalized_checkpoint.epoch + assert state.finalized_checkpoint.root != prev_state.finalized_checkpoint.root else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root + assert state.finalized_checkpoint == prev_state.finalized_checkpoint def next_epoch_with_attestations(spec, @@ -82,8 +79,7 @@ def test_finality_rule_4(spec, state): elif epoch >= 3: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -113,8 +109,7 @@ def test_finality_rule_1(spec, state): elif epoch == 2: # finalized by rule 1 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -144,8 +139,7 @@ def test_finality_rule_2(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) # finalized by rule 2 check_finality(spec, state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint blocks += new_blocks @@ -196,8 +190,7 @@ def test_finality_rule_3(spec, state): blocks += new_blocks # rule 3 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state From 181a2a876a369051a8a6db3b0c972d13cdb81202 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:49:53 +0200 Subject: [PATCH 007/110] Cleanups --- specs/core/0_beacon-chain.md | 38 ++++++++----------- .../eth2spec/test/helpers/attestations.py | 10 ++--- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 16 ++++---- .../test_process_attester_slashing.py | 6 +-- .../test_process_crosslinks.py | 4 +- 6 files changed, 35 insertions(+), 41 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 838178ea4..d8d3d6a76 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -346,8 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_checkpoint: Checkpoint - target_checkpoint: Checkpoint + source: Checkpoint + target: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -719,10 +719,9 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - target_epoch = data.target_checkpoint.epoch - committee_count = get_epoch_committee_count(state, target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + committee_count = get_epoch_committee_count(state, data.target.epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target.epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -886,7 +885,7 @@ def get_attesting_indices(state: BeaconState, """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) + committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1006,7 +1005,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), ) ``` @@ -1019,12 +1018,9 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or # Surround vote - ( - data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and - data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch - ) + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) ) ``` @@ -1282,7 +1278,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_checkpoint.root == get_block_root(state, epoch) + if a.data.target.root == get_block_root(state, epoch) ] ``` @@ -1344,13 +1340,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1692,10 +1688,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data - target_epoch = data.target_checkpoint.epoch - assert data.crosslink.shard < SHARD_COUNT - assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1707,7 +1701,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if target_epoch == get_current_epoch(state): + if data.target.epoch == get_current_epoch(state): ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) @@ -1717,9 +1711,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) # Check FFG source, crosslink data, and signature - assert ffg_source == data.source_checkpoint + assert ffg_source == data.source assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d2ce6e29b..d717bdc0d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -37,8 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), - target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), + source=spec.Checkpoint(epoch=source_epoch, root=source_root), + target=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -62,7 +62,7 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_checkpoint.epoch, + attestation_data.target.epoch, attestation_data.crosslink.shard, ) @@ -124,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_checkpoint.epoch, + message_epoch=attestation_data.target.epoch, ) ) @@ -132,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index e339421e8..9c68b7bbe 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_checkpoint.root = b'\x01' * 32 + attestation_2.data.target.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 9a718b071..d6cb615f8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): + if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -125,10 +125,10 @@ def test_old_source_epoch(spec, state): attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch + assert attestation.data.source.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_checkpoint.epoch -= 1 + attestation.data.source.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.epoch += 1 + attestation.data.source.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root + attestation.data.source.root = attestation.data.target.root sign_attestation(spec, state, attestation) @@ -188,10 +188,10 @@ def test_invalid_current_source_root(spec, state): # Test logic sanity checks: assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root - assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root + assert attestation.data.source.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root + attestation.data.source.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -204,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = b'\x42' * 32 + attestation.data.source.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 71915f1d0..e2b50ea0b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -74,8 +74,8 @@ def test_success_surround(spec, state): attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 - attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 + attestation_1.data.source.epoch = attestation_2.data.source.epoch - 1 + attestation_1.data.target.epoch = attestation_2.data.target.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -122,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 + attester_slashing.attestation_1.data.target.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 482085fd1..090858a2e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_checkpoint.epoch, + attestation_2.data.target.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 From e6e90c7736e69759c59dd6cf6fa2efc9c11e1d3c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:51:04 +0200 Subject: [PATCH 008/110] Minor reorg --- specs/core/0_beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d8d3d6a76..ae99a807a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -540,11 +540,10 @@ class BeaconState(Container): # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] - # Justification + # Finality + justification_bitfield: uint64 # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint - justification_bitfield: uint64 # Bit set for every recent justified epoch - # Finality finalized_checkpoint: Checkpoint ``` From 98692bf9d69198c8079d4af9458e5b4e13243cb2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:52:37 +0200 Subject: [PATCH 009/110] Fix ToC --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ae99a807a..a40bef6d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -25,8 +25,8 @@ - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - - [`Validator`](#validator) - [`Checkpoint`](#checkpoint) + - [`Validator`](#validator) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) From 2342c787c96b034e9c2db08445b15fdc3a7f8b50 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:54:07 +0200 Subject: [PATCH 010/110] Cleanup --- specs/core/0_beacon-chain.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a40bef6d1..30876ae29 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -878,9 +878,7 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, - data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ From 1e2bb08a747abc3b9ef4c5434c074bc078360a53 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:09:09 +0200 Subject: [PATCH 011/110] Cosmetic fix for consistency --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 30876ae29..c062bbf78 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1267,7 +1267,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - assert epoch in (get_current_epoch(state), get_previous_epoch(state)) + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` From bb0a492fea2f4e2e3a036999325d6d8141b7b4b4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:18:24 +0200 Subject: [PATCH 012/110] Cleanups --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c062bbf78..68a121cd4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1699,20 +1699,21 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ) if data.target.epoch == get_current_epoch(state): - ffg_source = state.current_justified_checkpoint + assert data.source == state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_source = state.previous_justified_checkpoint + assert data.source == state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG source, crosslink data, and signature - assert ffg_source == data.source + # Check crosslink against expected parent crosslink + assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] + + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` From d9644f518b6ce8a9269b08c18942d5ff1d6773f1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:08:13 +0200 Subject: [PATCH 013/110] mask is hash() in tests Co-Authored-By: dankrad --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 0167edc6a..8037755bc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -21,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) # Generate the mask (any random 32 bytes will do) - mask = reveal[:32] + mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( message_hash=mask, From 139d0f56f10972d6677b70683702f819b5c2fe02 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:26:21 +0200 Subject: [PATCH 014/110] Finishes moving mask to hash() --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 8037755bc..9ca770c11 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.hash_function import hash def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -20,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - # Generate the mask (any random 32 bytes will do) + # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( From 0e362d36b1eda660d013d32002acf7fce0c8ff91 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 17:18:22 -0600 Subject: [PATCH 015/110] pr feedback --- specs/core/0_fork-choice.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1755c6df3..1818de6c4 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -179,23 +179,31 @@ def on_block(store: Store, block: BeaconBlock) -> None: ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) + target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) # Cannot calculate the current shuffling if have not seen the target - assert target_checkpoint.root in store.blocks + assert target.root in store.blocks + + # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen - if target_checkpoint not in store.checkpoint_states: - base_state = store.block_states[target_checkpoint.root].copy() - store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + if target not in store.checkpoint_states: + base_state = store.block_states[target.root].copy() + store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + target_state = store.checkpoint_states[target] - # Get state at the `target_checkpoint` to validate attestation and calculate the committees - state = store.checkpoint_states[target_checkpoint] - indexed_attestation = convert_to_indexed(state, attestation) - validate_indexed_attestation(state, indexed_attestation) + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + attestation_slot = get_attestation_data_slot(target_state, attestation.data) + assert store.time >= (attestation_slot + 1) * SECONDS_PER_SLOT + + # Get state at the `target` to validate attestation and calculate the committees + indexed_attestation = convert_to_indexed(target_state, attestation) + validate_indexed_attestation(target_state, indexed_attestation) # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target_checkpoint + if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target ``` From 751738f411842c3cf1ac7d3ac9ec828d754912b0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:01:15 -0600 Subject: [PATCH 016/110] enhance fork choice testing --- specs/core/0_fork-choice.md | 24 ++-- .../test/fork_choice/test_on_attestation.py | 134 ++++++++++++++++++ .../test/fork_choice/test_on_block.py | 95 +++++++++++++ .../pyspec/eth2spec/test/test_fork_choice.py | 59 -------- 4 files changed, 241 insertions(+), 71 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1818de6c4..3ab041837 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -58,7 +58,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A #### `Checkpoint` ```python -@dataclass +@dataclass(eq=True, frozen=True) class Checkpoint(object): epoch: Epoch root: Hash @@ -69,13 +69,13 @@ class Checkpoint(object): ```python @dataclass class Store(object): + time: int + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - time: int latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -87,12 +87,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( - blocks={root: genesis_block}, - block_states={root: genesis_state}, - checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, + blocks={root: genesis_block}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -150,13 +150,13 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.block_states[parent_block.root].copy() + pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -167,7 +167,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Update justified checkpoint if state.current_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_epoch: + elif state.previous_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) # Update finalized checkpoint @@ -185,11 +185,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: assert target.root in store.blocks # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. - assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + base_state = store.block_states[target.root].copy() + assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - base_state = store.block_states[target.root].copy() store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) target_state = store.checkpoint_states[target] diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py new file mode 100644 index 000000000..fd42a6b29 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -0,0 +1,134 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +def run_on_attestation(spec, state, store, attestation, valid=True): + if not valid: + try: + spec.on_attestation(store, attestation) + except: + return + else: + assert False + + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Checkpoint( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_target_not_in_store(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # move to next epoch to make block new target + state.slot += spec.SLOTS_PER_EPOCH + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # do not add block to store + + next_slot(spec, state) + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_future_epoch(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + next_slot(spec, state) + + # move state forward but not store + attestation_slot = block.slot + spec.SLOTS_PER_EPOCH + state.slot = attestation_slot + + attestation = get_valid_attestation(spec, state, slot=state.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_same_slot(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 1 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_invalid_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + # make attestation invalid + attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] + run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py new file mode 100644 index 000000000..ef25317b6 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -0,0 +1,95 @@ +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot + + +def run_on_block(spec, state, store, block, valid=True): + if not valid: + try: + spec.on_block(store, block) + except: + return + else: + assert False + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + +@with_all_phases +@with_state +@bls_switch +def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block) + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + + run_on_block(spec, state, store, block) + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_future_block(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + + # do not tick time + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_bad_parent_root(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + block.parent_root = b'\x45' * 32 + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_before_finalized(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 4706f0eaf..8b1378917 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,60 +1 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root -from eth2spec.test.context import with_all_phases, with_state, bls_switch - -from eth2spec.test.helpers.block import build_empty_block_for_next_slot -from eth2spec.test.helpers.attestations import get_valid_attestation -from eth2spec.test.helpers.state import next_slot - - -@with_all_phases -@with_state -@bls_switch -def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - - # Initialization - store = spec.get_genesis_store(state) - blocks = [] - time = 100 - spec.on_tick(store, time) - assert store.time == time - - # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH - blocks.append(block) - - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # TODO: add tests for justified_root and finalized_root - - -@with_all_phases -@with_state -@bls_switch -def test_on_attestation(spec, state): - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - - next_slot(spec, state) - - attestation = get_valid_attestation(spec, state, slot=1) - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, - ) - ) From d9b97578c0f73fb0ccd03a5727596eaec4847654 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:09:57 -0600 Subject: [PATCH 017/110] lint --- specs/core/0_fork-choice.md | 12 ++++++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 4 ++-- .../eth2spec/test/fork_choice/test_on_block.py | 9 ++++++--- test_libs/pyspec/eth2spec/test/test_fork_choice.py | 1 - 4 files changed, 16 insertions(+), 10 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/test_fork_choice.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3ab041837..71920d2bc 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -149,14 +149,17 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues - parent_block = store.blocks[block.parent_root] + assert block.parent_root in store.block_states pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root + assert ( + get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == + store.finalized_checkpoint.root + ) # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -184,13 +187,14 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + process_slots(base_state, get_epoch_start_slot(target.epoch)) + store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] # Attestations can only affect the fork choice of subsequent slots. diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index fd42a6b29..cf0e7c9cb 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -11,7 +11,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if not valid: try: spec.on_attestation(store, attestation) - except: + except AssertionError: return else: assert False @@ -131,4 +131,4 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] - run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file + run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index ef25317b6..b18752f8c 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -9,7 +9,7 @@ def run_on_block(spec, state, store, block, valid=True): if not valid: try: spec.on_block(store, block) - except: + except AssertionError: return else: assert False @@ -88,8 +88,11 @@ def test_on_block_before_finalized(spec, state): time = 100 spec.on_tick(store, time) - store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + store.finalized_checkpoint = spec.Checkpoint( + epoch=store.finalized_checkpoint.epoch + 2, + root=store.finalized_checkpoint.root + ) # Fail receiving block of `GENESIS_SLOT + 1` slot block = build_empty_block_for_next_slot(spec, state) - run_on_block(spec, state, store, block, False) \ No newline at end of file + run_on_block(spec, state, store, block, False) diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py deleted file mode 100644 index 8b1378917..000000000 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ /dev/null @@ -1 +0,0 @@ - From b8c0985e6063db80a99ec1b87127cf1106761034 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:43:05 -0600 Subject: [PATCH 018/110] merge in fork choice. tests pass --- specs/core/0_beacon-chain.md | 7 ++--- specs/core/0_fork-choice.md | 27 +++++++------------ .../test/fork_choice/test_on_attestation.py | 4 +-- .../test_process_attestation.py | 4 +-- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 91e79e98f..1c9d02c33 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1337,13 +1337,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1352,7 +1353,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc..77f8e407d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -55,15 +55,6 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Checkpoint` - -```python -@dataclass(eq=True, frozen=True) -class Checkpoint(object): - epoch: Epoch - root: Hash -``` - #### `Store` ```python @@ -84,8 +75,8 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) - finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) + justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) + finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) return Store( time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, @@ -168,21 +159,21 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.block_states[signing_root(block)] = state # Update justified checkpoint - if state.current_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + elif state.previous_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.previous_justified_checkpoint # Update finalized checkpoint - if state.finalized_epoch > state.finalized_epoch: - store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) + target = attestation.data.target # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb..20c482d4f 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -21,8 +21,8 @@ def run_on_attestation(spec, state, store, attestation, valid=True): assert ( store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == spec.Checkpoint( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + epoch=attestation.data.target.epoch, + root=attestation.data.target.root, ) ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index d6cb615f8..5ceab4058 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -180,8 +180,8 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) - state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01' * 32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32' * 32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 846ca649aafece56d7aaf09bb16aa3bc5a76725b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 10:31:39 -0600 Subject: [PATCH 019/110] properly construct genesis latest block header in tests --- .../eth2spec/test/fork_choice/test_on_attestation.py | 12 ------------ .../eth2spec/test/fork_choice/test_on_block.py | 11 +---------- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 4 +++- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb..48f4bf46f 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,5 +1,3 @@ -from eth2spec.utils.ssz.ssz_impl import hash_tree_root - from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -31,8 +29,6 @@ def run_on_attestation(spec, state, store, attestation, valid=True): @with_state @bls_switch def test_on_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -52,8 +48,6 @@ def test_on_attestation(spec, state): @with_state @bls_switch def test_on_attestation_target_not_in_store(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -74,8 +68,6 @@ def test_on_attestation_target_not_in_store(spec, state): @with_state @bls_switch def test_on_attestation_future_epoch(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -98,8 +90,6 @@ def test_on_attestation_future_epoch(spec, state): @with_state @bls_switch def test_on_attestation_same_slot(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 1 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -117,8 +107,6 @@ def test_on_attestation_same_slot(spec, state): @with_state @bls_switch def test_on_attestation_invalid_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index b18752f8c..158ee1a58 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -1,7 +1,6 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.test.context import with_all_phases, with_state, bls_switch - from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -22,8 +21,6 @@ def run_on_block(spec, state, store, block, valid=True): @with_state @bls_switch def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -48,8 +45,6 @@ def test_basic(spec, state): @with_state @bls_switch def test_on_block_future_block(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) @@ -64,8 +59,6 @@ def test_on_block_future_block(spec, state): @with_state @bls_switch def test_on_block_bad_parent_root(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -81,8 +74,6 @@ def test_on_block_bad_parent_root(spec, state): @with_state @bls_switch def test_on_block_before_finalized(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d818908..49e4feb7d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -27,7 +27,9 @@ def create_genesis_state(spec, num_validators): deposit_root=deposit_root, deposit_count=num_validators, block_hash=spec.ZERO_HASH, - )) + ), + latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), + ) # We "hack" in the initial validators, # as it is much faster than creating and processing genesis deposits for every single test case. From 228195d89d464e5c4af7c4223808ffd33b96efda Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 11:48:55 -0600 Subject: [PATCH 020/110] get head tests --- specs/core/0_fork-choice.md | 9 +- .../test/fork_choice/test_get_head.py | 118 ++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc..f74f487a1 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -91,7 +91,7 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, blocks={root: genesis_block}, - block_states={root: genesis_state}, + block_states={root: genesis_state.copy()}, checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -110,10 +110,11 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) + active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( - state.validator_registry[i].effective_balance for i in active_indices - if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_targets and + get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) )) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py new file mode 100644 index 000000000..6ac46ba6c --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -0,0 +1,118 @@ +from eth2spec.test.context import with_all_phases, with_state, bls_switch +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block + + +def add_block_to_store(spec, store, block): + pre_state = store.block_states[block.parent_root] + block_time = pre_state.genesis_time + block.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.signing_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_genesis(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + +@with_all_phases +@with_state +@bls_switch +def test_chain_no_attestations(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # On receiving a block of `GENESIS_SLOT + 1` slot + block_1 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_1) + add_block_to_store(spec, store, block_1) + + # On receiving a block of next epoch + block_2 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_2) + add_block_to_store(spec, store, block_2) + + assert spec.get_head(store) == spec.signing_root(block_2) + + +@with_all_phases +@with_state +@bls_switch +def test_split_tie_breaker_no_attestations(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # block at slot 1 + block_1_state = genesis_state.copy() + block_1 = build_empty_block_for_next_slot(spec, block_1_state) + state_transition_and_sign_block(spec, block_1_state, block_1) + add_block_to_store(spec, store, block_1) + + # additional block at slot 1 + block_2_state = genesis_state.copy() + block_2 = build_empty_block_for_next_slot(spec, block_2_state) + block_2.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, block_2_state, block_2) + add_block_to_store(spec, store, block_2) + + highest_root = max(spec.signing_root(block_1), spec.signing_root(block_2)) + + assert spec.get_head(store) == highest_root + + +@with_all_phases +@with_state +@bls_switch +def test_shorter_chain_but_heavier_weight(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # build longer tree + long_state = genesis_state.copy() + for i in range(3): + long_block = build_empty_block_for_next_slot(spec, long_state) + state_transition_and_sign_block(spec, long_state, long_block) + add_block_to_store(spec, store, long_block) + + # build short tree + short_state = genesis_state.copy() + short_block = build_empty_block_for_next_slot(spec, short_state) + short_block.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, short_state, short_block) + add_block_to_store(spec, store, short_block) + + short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) + add_attestation_to_store(spec, store, short_attestation) + + assert spec.get_head(store) == spec.signing_root(short_block) From c64289677f0a5372dd1d8b6990ccc6510e0a14bc Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 14:42:37 -0600 Subject: [PATCH 021/110] fix gethead tests --- specs/core/0_fork-choice.md | 22 ++++++++++++++----- .../test/fork_choice/test_on_attestation.py | 6 ++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f74f487a1..55ff91a81 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -64,6 +64,16 @@ class Checkpoint(object): root: Hash ``` +#### `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Hash +``` + + #### `Store` ```python @@ -75,7 +85,7 @@ class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) ``` #### `get_genesis_store` @@ -113,8 +123,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_targets and - get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) + if (i in store.latest_messages and + get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -207,8 +217,8 @@ def on_attestation(store: Store, attestation: Attestation) -> None: indexed_attestation = convert_to_indexed(target_state, attestation) validate_indexed_attestation(target_state, indexed_attestation) - # Update latest targets + # Update latest messages for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 48f4bf46f..0e52e7fd0 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -17,10 +17,10 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.convert_to_indexed(state, attestation) spec.on_attestation(store, attestation) assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Checkpoint( + store.latest_messages[indexed_attestation.custody_bit_0_indices[0]] == + spec.LatestMessage( epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + root=attestation.data.beacon_block_root, ) ) From 8c34aa8c5f7fd7e1951843feae396ac66021e9db Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 26 Jun 2019 13:20:04 +0100 Subject: [PATCH 022/110] Initial draft --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a..6cf6db0cc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,6 +30,7 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) + - [`CompactCommitteees`](#CompactCommitteees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) @@ -68,7 +69,7 @@ - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - [`get_randao_mix`](#get_randao_mix) - - [`get_active_index_root`](#get_active_index_root) + - [`get_compact_committee_root`](#get_compact_committee_root) - [`generate_seed`](#generate_seed) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`verify_merkle_branch`](#verify_merkle_branch) @@ -187,7 +188,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `SHARD_COUNT` | `2**10` (= 1,024) | | `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) | -| `MAX_INDICES_PER_ATTESTATION` | `2**12` (= 4,096) | +| `MAX_VALIDATORS_PER_COMMITTEE` | `2**12` (= 4,096) | | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | @@ -344,8 +345,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 1 + custody_bit_0_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 1 data: AttestationData signature: BLSSignature ``` @@ -354,12 +355,22 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex ``` +#### `CompactCommitteees` + +```python +class CompactCommitteees(Container): + data: Vector[Container( + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] + ), SHARD_COUNT] +``` + #### `Eth1Data` ```python @@ -421,9 +432,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bytes[MAX_VALIDATORS_PER_COMMITTEE // 8] signature: BLSSignature ``` @@ -511,7 +522,7 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients + compact_committee_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations @@ -742,17 +753,23 @@ def get_randao_mix(state: BeaconState, return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` -### `get_active_index_root` +### `get_compact_committee_root` ```python -def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Hash: +def get_compact_committee_root(state: BeaconState, epoch: Epoch) -> Hash: """ - Return the index root at a recent ``epoch``. - ``epoch`` expected to be between - (current_epoch - EPOCHS_PER_HISTORICAL_VECTOR + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. + Return the compact committee root for the current epoch. """ - return state.active_index_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + committee_data = CompactCommitteees().data + for committee_number in range(get_epoch_committee_count(state, epoch)): + shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT + for index in get_crosslink_committee(state, epoch, shard): + validator = validators[index] + committee_data[shard].pubkeys.append(validator.pubkey) + # `index` (top 7 bytes) + `slashed` (8th bit) + `effective_balance` (bottom 7 bits) + compact_validator = index << 8 + validator.slashed << 7 + validator.effective_balance // GWEI_PER_ETH + committee_data[shard].compact_validators.append(compact_validator) + return hash_tree_root(committee_data) ``` ### `generate_seed` @@ -765,7 +782,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_active_index_root(state, epoch) + + get_compact_committee_root(state, epoch) + int_to_bytes(epoch, length=32) ) ``` @@ -973,7 +990,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted @@ -1173,12 +1190,9 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots - genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) - ) + # Populate compact_committee_roots for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.active_index_roots[index] = genesis_active_index_root + state.compact_committee_roots[index] = get_compact_committee_root(state, GENESIS_EPOCH) return state ``` @@ -1532,11 +1546,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT]( - get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) - ) - ) + state.compact_committee_roots[index_root_position] = get_compact_committee_root(state, next_epoch + ACTIVATION_EXIT_DELAY) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] From b133dedeaf9d48695d407d9800a49d3b886ba8aa Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 22:11:40 +0200 Subject: [PATCH 023/110] Eth1 data test --- .../eth2spec/test/sanity/test_blocks.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 286c0150c..27f0a884f 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -5,7 +5,7 @@ from eth2spec.utils.bls import bls_sign from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block # from eth2spec.test.helpers.transfers import get_valid_transfer -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -172,8 +172,6 @@ def test_attester_slashing(spec, state): ) -# TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector - @with_all_phases @spec_state_test def test_deposit_in_block(spec, state): @@ -386,29 +384,43 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -# @with_all_phases -# @spec_state_test -# def test_eth1_data_votes(spec, state): -# yield 'pre', state +@with_all_phases +@spec_state_test +def test_eth1_data_votes_success(spec, state): + # Don't run when it will take very, very long to simulate. Minimal configuration suffices. + if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: + return -# expected_votes = 0 -# assert len(state.eth1_data_votes) == expected_votes + offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + state_transition_and_sign_block(spec, state, offset_block) + yield 'pre', state -# blocks = [] -# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): -# block = build_empty_block_for_next_slot(spec, state) -# state_transition_and_sign_block(spec, state, block) -# expected_votes += 1 -# assert len(state.eth1_data_votes) == expected_votes -# blocks.append(block) + a = b'\xaa' * 32 + b = b'\xbb' * 32 + c = b'\xcc' * 32 -# block = build_empty_block_for_next_slot(spec, state) -# blocks.append(block) + blocks = [] -# state_transition_and_sign_block(spec, state, block) + for i in range(0, spec.SLOTS_PER_ETH1_VOTING_PERIOD): + block = build_empty_block_for_next_slot(spec, state) + # wait for over 50% for A, then start voting B + block.body.eth1_data.block_hash = b if i * 2 > spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + state_transition_and_sign_block(spec, state, block) + blocks.append(block) -# yield 'blocks', [block] -# yield 'post', state + assert len(state.eth1_data_votes) == spec.SLOTS_PER_ETH1_VOTING_PERIOD + assert state.eth1_data.block_hash == a -# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 -# assert len(state.eth1_data_votes) == 1 + # transition to next eth1 voting period + block = build_empty_block_for_next_slot(spec, state) + block.body.eth1_data.block_hash = c + state_transition_and_sign_block(spec, state, block) + blocks.append(block) + + yield 'blocks', blocks + yield 'post', state + + assert state.eth1_data.block_hash == a + assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 + assert len(state.eth1_data_votes) == 1 + assert state.eth1_data_votes[0].block_hash == c From f54d1a56f7192be254a8d8d87b8f67a84758af7f Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:23:37 +0200 Subject: [PATCH 024/110] eth1 voting no consensus test --- .../eth2spec/test/sanity/test_blocks.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 27f0a884f..b11e41d66 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -4,7 +4,6 @@ from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.bls import bls_sign from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block -# from eth2spec.test.helpers.transfers import get_valid_transfer from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing @@ -386,7 +385,7 @@ def test_historical_batch(spec, state): @with_all_phases @spec_state_test -def test_eth1_data_votes_success(spec, state): +def test_eth1_data_votes_consensus(spec, state): # Don't run when it will take very, very long to simulate. Minimal configuration suffices. if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: return @@ -424,3 +423,33 @@ def test_eth1_data_votes_success(spec, state): assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 assert len(state.eth1_data_votes) == 1 assert state.eth1_data_votes[0].block_hash == c + + +@with_all_phases +@spec_state_test +def test_eth1_data_votes_no_consensus(spec, state): + # Don't run when it will take very, very long to simulate. Minimal configuration suffices. + if spec.SLOTS_PER_ETH1_VOTING_PERIOD > 16: + return + + offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + state_transition_and_sign_block(spec, state, offset_block) + yield 'pre', state + + a = b'\xaa' * 32 + b = b'\xbb' * 32 + + blocks = [] + + for i in range(0, spec.SLOTS_PER_ETH1_VOTING_PERIOD): + block = build_empty_block_for_next_slot(spec, state) + # wait for precisely 50% for A, then start voting B for other 50% + block.body.eth1_data.block_hash = b if i * 2 >= spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + state_transition_and_sign_block(spec, state, block) + blocks.append(block) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_ETH1_VOTING_PERIOD + assert state.eth1_data.block_hash == b'\x00' * 32 + + yield 'blocks', blocks + yield 'post', state From 13b67b4cde6ddb762927ee78645ce38bd8538a76 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:46:26 +0200 Subject: [PATCH 025/110] sign blocks in eth1 vote tests --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index b11e41d66..34409986e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -373,6 +373,7 @@ def test_historical_batch(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state, signed=True) + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) yield 'blocks', [block] @@ -391,6 +392,7 @@ def test_eth1_data_votes_consensus(spec, state): return offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + sign_block(spec, state, offset_block) state_transition_and_sign_block(spec, state, offset_block) yield 'pre', state @@ -404,6 +406,7 @@ def test_eth1_data_votes_consensus(spec, state): block = build_empty_block_for_next_slot(spec, state) # wait for over 50% for A, then start voting B block.body.eth1_data.block_hash = b if i * 2 > spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) @@ -413,6 +416,7 @@ def test_eth1_data_votes_consensus(spec, state): # transition to next eth1 voting period block = build_empty_block_for_next_slot(spec, state) block.body.eth1_data.block_hash = c + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) @@ -433,6 +437,7 @@ def test_eth1_data_votes_no_consensus(spec, state): return offset_block = build_empty_block(spec, state, slot=spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1) + sign_block(spec, state, offset_block) state_transition_and_sign_block(spec, state, offset_block) yield 'pre', state @@ -445,6 +450,7 @@ def test_eth1_data_votes_no_consensus(spec, state): block = build_empty_block_for_next_slot(spec, state) # wait for precisely 50% for A, then start voting B for other 50% block.body.eth1_data.block_hash = b if i * 2 >= spec.SLOTS_PER_ETH1_VOTING_PERIOD else a + sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) blocks.append(block) From 327953852d2034ebf6acf7eb7d39af84314b91b2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 01:47:53 +0200 Subject: [PATCH 026/110] test invalid shard in attestation --- .../block_processing/test_process_attestation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..79af0b202 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -148,6 +148,20 @@ def test_wrong_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +def test_invalid_shard(spec, state): + attestation = get_valid_attestation(spec, state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + # off by one (with respect to valid range) on purpose + attestation.data.crosslink.shard = spec.SHARD_COUNT + + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_new_source_epoch(spec, state): From f75e3dccb2041cec7884266a7a348decd1051074 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:03:43 +0200 Subject: [PATCH 027/110] test old and future target epoch in attestation --- .../test_process_attestation.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 79af0b202..59e99ac0c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -162,6 +162,33 @@ def test_invalid_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +def test_old_target_epoch(spec, state): + assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 + + attestation = get_valid_attestation(spec, state, signed=True) + + state.slot = spec.SLOTS_PER_EPOCH * 2 # target epoch will be too old to handle + + yield from run_attestation_processing(spec, state, attestation, False) + + +@with_all_phases +@spec_state_test +def test_future_target_epoch(spec, state): + assert spec.MIN_ATTESTATION_INCLUSION_DELAY < spec.SLOTS_PER_EPOCH * 2 + + attestation = get_valid_attestation(spec, state) + + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.target_epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_new_source_epoch(spec, state): From 64e15c524b569ce474406e1c6781d95495bcd53e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:27:28 +0200 Subject: [PATCH 028/110] improve intersection test, just 1 index is enough. And add invalid att1/att2 tests --- .../test_process_attester_slashing.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c51f5a8a9..2d8f9d31a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -142,12 +142,39 @@ def test_participants_already_slashed(spec, state): @with_all_phases @spec_state_test -def test_custody_bit_0_and_1(spec, state): +def test_custody_bit_0_and_1_intersect(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.custody_bit_1_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices + attester_slashing.attestation_1.custody_bit_1_indices.append( + attester_slashing.attestation_1.custody_bit_0_indices[0] ) + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_attester_slashing_invalid_att_1(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + assert len(indices) >= 3 + indices[1], indices[2] = indices[2], indices[1] # unsort second and third index + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_attester_slashing_invalid_att_2(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + assert len(indices) >= 3 + indices[1], indices[2] = indices[2], indices[1] # unsort second and third index + sign_indexed_attestation(spec, state, attester_slashing.attestation_2) + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) From 1d6b1cab13dc979f15e1e1bc5b50ae3c3585e795 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:37:20 +0200 Subject: [PATCH 029/110] expected deposit count test --- .../eth2spec/test/sanity/test_blocks.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 34409986e..bb6a6dc06 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -171,6 +171,28 @@ def test_attester_slashing(spec, state): ) +@with_all_phases +@spec_state_test +def test_expected_deposit_in_block(spec, state): + # Make the state expect a deposit, then don't provide it. + state.eth1_data.deposit_count += 1 + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + sign_block(spec, state, block) + bad = False + try: + state_transition_and_sign_block(spec, state, block) + bad = True + except AssertionError: + pass + if bad: + raise AssertionError("expected deposit was not enforced") + + yield 'blocks', [block] + yield 'post', None + + @with_all_phases @spec_state_test def test_deposit_in_block(spec, state): From 55d86b4f13afacf5fec4e48cb544cfd003c7cd31 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 02:48:40 +0200 Subject: [PATCH 030/110] effective balance testing in deposits --- .../block_processing/test_process_deposit.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 8b3d7b413..1d4b3a107 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -49,6 +49,11 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert len(state.balances) == pre_validator_count + 1 assert get_balance(state, validator_index) == pre_balance + deposit.data.amount + effective = min(spec.MAX_EFFECTIVE_BALANCE, + pre_balance + deposit.data.amount) + effective -= effective % spec.EFFECTIVE_BALANCE_INCREMENT + assert state.validators[validator_index].effective_balance == effective + assert state.eth1_deposit_index == state.eth1_data.deposit_count @@ -57,7 +62,20 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef def test_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) - amount = spec.MAX_EFFECTIVE_BALANCE + # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. + amount = spec.MAX_EFFECTIVE_BALANCE - 1 + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_maxed_out(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # just 1 over the limit, effective balance should be set MAX_EFFECTIVE_BALANCE during processing + amount = spec.MAX_EFFECTIVE_BALANCE + 1 deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) yield from run_deposit_processing(spec, state, deposit, validator_index) From 063d94b9c7da65d076f563e243584fa0b8b9ab3d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:13:48 +0200 Subject: [PATCH 031/110] Bugfix transfer tests --- .../block_processing/test_process_transfer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 89246cc51..45c161214 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -107,7 +107,7 @@ def test_active_but_transfer_past_effective_balance(spec, state): def test_incorrect_slot(spec, state): transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -120,7 +120,7 @@ def test_insufficient_balance_for_fee_result_dust(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -146,7 +146,7 @@ def test_insufficient_balance_for_amount_result_dust(spec, state): transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -287,7 +287,7 @@ def test_no_dust_sender(spec, state): ) # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -301,7 +301,7 @@ def test_no_dust_recipient(spec, state): state.balances[transfer.recipient] = 0 # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) @@ -313,6 +313,6 @@ def test_invalid_pubkey(spec, state): state.validators[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer - state.validators[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH yield from run_transfer_processing(spec, state, transfer, False) From e79b47e3c3a24d54f341d3defb691707c0acfb73 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 03:54:07 +0200 Subject: [PATCH 032/110] non-existent transfer participants tests --- .../pyspec/eth2spec/test/helpers/transfers.py | 6 ++-- .../block_processing/test_process_transfer.py | 34 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index acc6a35c5..fa01a3088 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -4,13 +4,15 @@ from eth2spec.utils.bls import bls_sign from eth2spec.utils.ssz.ssz_impl import signing_root -def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, fee=None, signed=False): +def get_valid_transfer(spec, state, slot=None, sender_index=None, + recipient_index=None, amount=None, fee=None, signed=False): if slot is None: slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] - recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] + if recipient_index is None: + recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 45c161214..ad7e6ff60 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -1,7 +1,7 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.helpers.transfers import get_valid_transfer +from eth2spec.test.helpers.transfers import get_valid_transfer, sign_transfer def run_transfer_processing(spec, state, transfer, valid=True): @@ -13,11 +13,6 @@ def run_transfer_processing(spec, state, transfer, valid=True): If ``valid == False``, run expecting ``AssertionError`` """ - proposer_index = spec.get_beacon_proposer_index(state) - pre_transfer_sender_balance = state.balances[transfer.sender] - pre_transfer_recipient_balance = state.balances[transfer.recipient] - pre_transfer_proposer_balance = state.balances[proposer_index] - yield 'pre', state yield 'transfer', transfer @@ -26,6 +21,11 @@ def run_transfer_processing(spec, state, transfer, valid=True): yield 'post', None return + proposer_index = spec.get_beacon_proposer_index(state) + pre_transfer_sender_balance = state.balances[transfer.sender] + pre_transfer_recipient_balance = state.balances[transfer.recipient] + pre_transfer_proposer_balance = state.balances[proposer_index] + spec.process_transfer(state, transfer) yield 'post', state @@ -306,6 +306,28 @@ def test_no_dust_recipient(spec, state): yield from run_transfer_processing(spec, state, transfer, False) +@with_all_phases +@spec_state_test +def test_non_existent_sender(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0) + transfer.sender = len(state.validators) + sign_transfer(spec, state, transfer, 42) # mostly valid signature, but sender won't exist, use bogus key. + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_non_existent_recipient(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + recipient_index=len(state.validators), amount=1, fee=0, signed=True) + + yield from run_transfer_processing(spec, state, transfer, False) + + @with_all_phases @spec_state_test def test_invalid_pubkey(spec, state): From 6266133572494b606e12db1326fa2ac597c215af Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 25 Jun 2019 03:53:40 +0200 Subject: [PATCH 033/110] rename test methods based on suggestion Co-Authored-By: Danny Ryan --- .../block_processing/test_process_attester_slashing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 2d8f9d31a..86b6811a2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -156,7 +156,7 @@ def test_custody_bit_0_and_1_intersect(spec, state): @with_all_phases @spec_state_test -def test_attester_slashing_invalid_att_1(spec, state): +def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) indices = attester_slashing.attestation_1.custody_bit_0_indices @@ -169,7 +169,7 @@ def test_attester_slashing_invalid_att_1(spec, state): @with_all_phases @spec_state_test -def test_attester_slashing_invalid_att_2(spec, state): +def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) indices = attester_slashing.attestation_2.custody_bit_0_indices From c4b88e68e1772bd2a88987f030113309dd4a9c4d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:19:02 +0200 Subject: [PATCH 034/110] different new-deposit tests --- .../block_processing/test_process_deposit.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 1d4b3a107..0f94bd26c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -59,10 +59,10 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef @with_all_phases @spec_state_test -def test_new_deposit(spec, state): +def test_new_deposit_under_max(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) - # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. + # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. amount = spec.MAX_EFFECTIVE_BALANCE - 1 deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) @@ -71,7 +71,19 @@ def test_new_deposit(spec, state): @with_all_phases @spec_state_test -def test_new_deposit_maxed_out(spec, state): +def test_new_deposit_max(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # effective balance will be exactly the same as balance. + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_over_max(spec, state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validators) # just 1 over the limit, effective balance should be set MAX_EFFECTIVE_BALANCE during processing From b4b4e9571da32e6ff13cd550cb1308cfdf703919 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 04:45:53 +0200 Subject: [PATCH 035/110] test activation queue --- .../test_process_registry_updates.py | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index d92220910..6c1319a5b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -30,18 +30,20 @@ def run_process_registry_updates(spec, state, valid=True): yield 'post', state -@with_all_phases -@spec_state_test -def test_activation(spec, state): - index = 0 +def mock_deposit(spec, state, index): assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - - # Mock a new deposit state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + +@with_all_phases +@spec_state_test +def test_activation(spec, state): + index = 0 + mock_deposit(spec, state, index) + for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(spec, state) @@ -49,10 +51,39 @@ def test_activation(spec, state): assert state.validators[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH assert state.validators[index].activation_epoch != spec.FAR_FUTURE_EPOCH - assert spec.is_active_validator( - state.validators[index], - spec.get_current_epoch(state), - ) + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + + +@with_all_phases +@spec_state_test +def test_activation_queue_sorting(spec, state): + mock_activations = 10 + + epoch = spec.get_current_epoch(state) + for i in range(mock_activations): + mock_deposit(spec, state, i) + state.validators[i].activation_eligibility_epoch = epoch + 1 + + # give the last priority over the others + state.validators[mock_activations - 1].activation_eligibility_epoch = epoch + + # make sure we are hitting the churn + churn_limit = spec.get_churn_limit(state) + assert mock_activations > churn_limit + + yield from run_process_registry_updates(spec, state) + + # the first got in as second + assert state.validators[0].activation_epoch != spec.FAR_FUTURE_EPOCH + # the prioritized got in as first + assert state.validators[mock_activations - 1].activation_epoch != spec.FAR_FUTURE_EPOCH + # the second last is at the end of the queue, and did not make the churn, + # hence is not assigned an activation_epoch yet. + assert state.validators[mock_activations - 2].activation_epoch == spec.FAR_FUTURE_EPOCH + # the one at churn_limit - 1 did not make it, it was out-prioritized + assert state.validators[churn_limit - 1].activation_epoch == spec.FAR_FUTURE_EPOCH + # but the the one in front of the above did + assert state.validators[churn_limit - 2].activation_epoch != spec.FAR_FUTURE_EPOCH @with_all_phases From 0e3c2cef5ca56e011ea1ec776e8eac6aa0884f3e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 15:45:57 +0200 Subject: [PATCH 036/110] fix transfer tests, add 2 new tests --- .../block_processing/test_process_transfer.py | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index ad7e6ff60..6903f0666 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -114,9 +114,37 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance_for_fee_result_dust(spec, state): +def test_transfer_clean(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT, fee=0, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer) + + +@with_all_phases +@spec_state_test +def test_transfer_clean_split_to_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT // 2, fee=spec.MIN_DEPOSIT_AMOUNT // 2, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer @@ -142,7 +170,7 @@ def test_insufficient_balance_for_fee_result_full(spec, state): @spec_state_test def test_insufficient_balance_for_amount_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer From b2034a54a07264553f7aad38b838cedd98ff64ec Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 22:22:17 +0200 Subject: [PATCH 037/110] generalize epoch processing testing, add final-processing tests --- .../run_epoch_process_base.py | 44 +++++++++++++++ .../test_process_crosslinks.py | 30 ++--------- .../test_process_final_updates.py | 53 +++++++++++++++++++ .../test_process_registry_updates.py | 31 ++--------- 4 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py new file mode 100644 index 000000000..13e1c3f3b --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -0,0 +1,44 @@ +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block + + +process_calls = ( + 'process_justification_and_finalization' + 'process_crosslinks' + 'process_rewards_and_penalties' + 'process_registry_updates' + 'process_reveal_deadlines' + 'process_challenge_deadlines' + 'process_slashings' + 'process_final_updates' + 'after_process_final_updates' +) + + +def run_epoch_processing_to(spec, state, process_name: str): + """ + Run the epoch processing functions up to ``process_name`` (incl.), yielding: + - pre-state ('pre'), state before calling ``process_name`` + - post-state ('post'), state after calling ``process_name`` + """ + # transition state to slot before state transition + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 + block = build_empty_block_for_next_slot(spec, state) + block.slot = slot + sign_block(spec, state, block) + state_transition_and_sign_block(spec, state, block) + + # cache state before epoch transition + spec.process_slot(state) + + # process components of epoch transition before final-updates + for name in process_calls: + if name == process_name: + break + # only run when present. Later phases introduce more to the epoch-processing. + if hasattr(spec, name): + getattr(spec, name)(state) + + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index d51191efb..a38435b96 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -4,41 +4,19 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.state import ( next_epoch, next_slot, - state_transition_and_sign_block, ) -from eth2spec.test.helpers.block import apply_empty_block, sign_block +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, - build_empty_block_for_next_slot, fill_aggregate_attestation, get_valid_attestation, sign_attestation, ) +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to -def run_process_crosslinks(spec, state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - # cache state before epoch transition - spec.process_slot(state) - - # process components of epoch transition before processing crosslinks - spec.process_justification_and_finalization(state) - - yield 'pre', state - spec.process_crosslinks(state) - yield 'post', state +def run_process_crosslinks(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_crosslinks') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py new file mode 100644 index 000000000..ca2f9eb10 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -0,0 +1,53 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to + + +def run_process_final_updates(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_final_updates') + + +@with_all_phases +@spec_state_test +def test_eth1_vote_reset(spec, state): + # skip ahead to near the end of the voting period + state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 2 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_final_updates(spec, state) + + assert len(state.eth1_data_votes) == 0 + + +@with_all_phases +@spec_state_test +def test_effective_balance_hysteresis(spec, state): + # Set some edge cases for balances + max = spec.MAX_EFFECTIVE_BALANCE + min = spec.EJECTION_BALANCE + inc = spec.EFFECTIVE_BALANCE_INCREMENT + half_inc = inc // 2 + cases = [ + (max, max, max), # as is + (max, max - 1, max - inc), # round down, step lower + (max, max + 1, max), # round down + (max, max - inc, max - inc), # exactly 1 step lower + (max, max - inc - 1, max - (2 * inc)), # just 1 over 1 step lower + (max, max - inc + 1, max - inc), # close to 1 step lower + (min, min + (half_inc * 3), min), # bigger balance, but not high enough + (min, min + (half_inc * 3) + 1, min + inc), # bigger balance, high enough, but small step + (min, min + (half_inc * 4) - 1, min + inc), # bigger balance, high enough, close to double step + (min, min + (half_inc * 4), min + (2 * inc)), # exact two step balance increment + (min, min + (half_inc * 4) + 1, min + (2 * inc)), # over two steps, round down + ] + for i, (pre_eff, bal, _) in enumerate(cases): + state.validators[i].effective_balance = pre_eff + state.balances[i] = bal + + yield from run_process_final_updates(spec, state) + + for i, (_, _, post_eff) in enumerate(cases): + assert state.validators[i].effective_balance == post_eff diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index 6c1319a5b..ae76e95c2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,33 +1,10 @@ -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block -from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block +from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to -def run_process_registry_updates(spec, state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - # cache state before epoch transition - spec.process_slot(state) - - # process components of epoch transition before registry update - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.process_rewards_and_penalties(state) - - yield 'pre', state - spec.process_registry_updates(state) - yield 'post', state +def run_process_registry_updates(spec, state): + yield from run_epoch_processing_to(spec, state, 'process_registry_updates') def mock_deposit(spec, state, index): From c4c9bd32e2af615bae95220c0c9e2492dc5ab67b Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 22:26:35 +0200 Subject: [PATCH 038/110] test_eth1_vote_no_reset --- .../test_process_final_updates.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index ca2f9eb10..0c19f8e31 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -6,6 +6,23 @@ def run_process_final_updates(spec, state): yield from run_epoch_processing_to(spec, state, 'process_final_updates') +@with_all_phases +@spec_state_test +def test_eth1_vote_no_reset(spec, state): + assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH + # skip ahead to near the end of the epoch + state.slot = spec.SLOTS_PER_EPOCH - 2 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_final_updates(spec, state) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH + + @with_all_phases @spec_state_test def test_eth1_vote_reset(spec, state): From aedd281edbcacfa68ff7ec91d4f3f2b0031a61c7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 23:18:19 +0200 Subject: [PATCH 039/110] clean up epoch processing testing --- .../run_epoch_process_base.py | 51 ++++++++--------- .../test_process_final_updates.py | 55 +++++++++++++------ 2 files changed, 60 insertions(+), 46 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index 13e1c3f3b..7e0cffc79 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -1,35 +1,29 @@ -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block -from eth2spec.test.helpers.state import state_transition_and_sign_block + +process_calls = [ + 'process_justification_and_finalization', + 'process_crosslinks', + 'process_rewards_and_penalties', + 'process_registry_updates', + 'process_reveal_deadlines', + 'process_challenge_deadlines', + 'process_slashings', + 'process_final_updates', + 'after_process_final_updates', +] -process_calls = ( - 'process_justification_and_finalization' - 'process_crosslinks' - 'process_rewards_and_penalties' - 'process_registry_updates' - 'process_reveal_deadlines' - 'process_challenge_deadlines' - 'process_slashings' - 'process_final_updates' - 'after_process_final_updates' -) - - -def run_epoch_processing_to(spec, state, process_name: str): +def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): """ - Run the epoch processing functions up to ``process_name`` (incl.), yielding: + Run the epoch processing functions up to ``process_name``. + If ``exclusive`` is True, the process itself will not be ran. + If ``exclusive`` is False (default), this function yields: - pre-state ('pre'), state before calling ``process_name`` - post-state ('post'), state after calling ``process_name`` """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - # cache state before epoch transition - spec.process_slot(state) + # transition state to slot before epoch state transition + spec.process_slots(state, slot) # process components of epoch transition before final-updates for name in process_calls: @@ -39,6 +33,7 @@ def run_epoch_processing_to(spec, state, process_name: str): if hasattr(spec, name): getattr(spec, name)(state) - yield 'pre', state - getattr(spec, process_name)(state) - yield 'post', state + if not exclusive: + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index 0c19f8e31..d1af7d396 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -10,8 +10,8 @@ def run_process_final_updates(spec, state): @spec_state_test def test_eth1_vote_no_reset(spec, state): assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH - # skip ahead to near the end of the epoch - state.slot = spec.SLOTS_PER_EPOCH - 2 + # skip ahead to the end of the epoch + state.slot = spec.SLOTS_PER_EPOCH - 1 for i in range(state.slot + 1): # add a vote for each skipped slot. state.eth1_data_votes.append( spec.Eth1Data(deposit_root=b'\xaa' * 32, @@ -26,8 +26,8 @@ def test_eth1_vote_no_reset(spec, state): @with_all_phases @spec_state_test def test_eth1_vote_reset(spec, state): - # skip ahead to near the end of the voting period - state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 2 + # skip ahead to the end of the voting period + state.slot = spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1 for i in range(state.slot + 1): # add a vote for each skipped slot. state.eth1_data_votes.append( spec.Eth1Data(deposit_root=b'\xaa' * 32, @@ -42,29 +42,48 @@ def test_eth1_vote_reset(spec, state): @with_all_phases @spec_state_test def test_effective_balance_hysteresis(spec, state): + # Prepare state up to the final-updates. + # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. + run_epoch_processing_to(spec, state, 'process_final_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE inc = spec.EFFECTIVE_BALANCE_INCREMENT half_inc = inc // 2 cases = [ - (max, max, max), # as is - (max, max - 1, max - inc), # round down, step lower - (max, max + 1, max), # round down - (max, max - inc, max - inc), # exactly 1 step lower - (max, max - inc - 1, max - (2 * inc)), # just 1 over 1 step lower - (max, max - inc + 1, max - inc), # close to 1 step lower - (min, min + (half_inc * 3), min), # bigger balance, but not high enough - (min, min + (half_inc * 3) + 1, min + inc), # bigger balance, high enough, but small step - (min, min + (half_inc * 4) - 1, min + inc), # bigger balance, high enough, close to double step - (min, min + (half_inc * 4), min + (2 * inc)), # exact two step balance increment - (min, min + (half_inc * 4) + 1, min + (2 * inc)), # over two steps, round down + (max, max, max, "as-is"), + (max, max - 1, max - inc, "round down, step lower"), + (max, max + 1, max, "round down"), + (max, max - inc, max - inc, "exactly 1 step lower"), + (max, max - inc - 1, max - (2 * inc), "just 1 over 1 step lower"), + (max, max - inc + 1, max - inc, "close to 1 step lower"), + (min, min + (half_inc * 3), min, "bigger balance, but not high enough"), + (min, min + (half_inc * 3) + 1, min + inc, "bigger balance, high enough, but small step"), + (min, min + (half_inc * 4) - 1, min + inc, "bigger balance, high enough, close to double step"), + (min, min + (half_inc * 4), min + (2 * inc), "exact two step balance increment"), + (min, min + (half_inc * 4) + 1, min + (2 * inc), "over two steps, round down"), ] - for i, (pre_eff, bal, _) in enumerate(cases): + current_epoch = spec.get_current_epoch(state) + for i, (pre_eff, bal, _, _) in enumerate(cases): + assert spec.is_active_validator(state.validators[i], current_epoch) state.validators[i].effective_balance = pre_eff state.balances[i] = bal + yield 'pre', state + spec.process_final_updates(state) + yield 'post', state + + for i, (_, _, post_eff, name) in enumerate(cases): + assert state.validators[i].effective_balance == post_eff, name + + +@with_all_phases +@spec_state_test +def test_historical_root_accumulator(spec, state): + # skip ahead to near the end of the historical roots period (excl block before epoch processing) + state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 + history_len = len(state.historical_roots) + yield from run_process_final_updates(spec, state) - for i, (_, _, post_eff) in enumerate(cases): - assert state.validators[i].effective_balance == post_eff + assert len(state.historical_roots) == history_len + 1 From c66031f55cf4b24b832f917778730c0aaf8e99dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 00:01:21 +0200 Subject: [PATCH 040/110] fix crosslink tests, fix generalization of epoch processing --- .../epoch_processing/run_epoch_process_base.py | 5 ++++- .../epoch_processing/test_process_crosslinks.py | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index 7e0cffc79..c69160fb0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -23,7 +23,10 @@ def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) # transition state to slot before epoch state transition - spec.process_slots(state, slot) + spec.process_slots(state, slot - 1) + + # start transitioning, do one slot update before the epoch itself. + spec.process_slot(state) # process components of epoch transition before final-updates for name in process_calls: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index a38435b96..7e93675e8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -4,8 +4,8 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.state import ( next_epoch, next_slot, -) -from eth2spec.test.helpers.block import apply_empty_block + state_transition_and_sign_block) +from eth2spec.test.helpers.block import apply_empty_block, build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, fill_aggregate_attestation, @@ -28,6 +28,14 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] +def add_block_to_end_of_epoch(spec, state): + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 + block = build_empty_block_for_next_slot(spec, state) + block.slot = slot + sign_block(spec, state, block) + state_transition_and_sign_block(spec, state, block) + + @with_all_phases @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): @@ -43,6 +51,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) + add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] @@ -66,6 +75,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): crosslink_deltas = spec.get_crosslink_deltas(state) + add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] From 46dc3f39bb4c7d01c4e059c69de437983ac20262 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 00:22:24 +0200 Subject: [PATCH 041/110] detach crosslink tests from extra block --- .../epoch_processing/test_process_crosslinks.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 7e93675e8..058c93733 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -3,9 +3,9 @@ from copy import deepcopy from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.state import ( next_epoch, - next_slot, - state_transition_and_sign_block) -from eth2spec.test.helpers.block import apply_empty_block, build_empty_block_for_next_slot, sign_block + next_slot +) +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, fill_aggregate_attestation, @@ -28,14 +28,6 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] -def add_block_to_end_of_epoch(spec, state): - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(spec, state) - block.slot = slot - sign_block(spec, state, block) - state_transition_and_sign_block(spec, state, block) - - @with_all_phases @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): @@ -51,7 +43,6 @@ def test_single_crosslink_update_from_current_epoch(spec, state): shard = attestation.data.crosslink.shard pre_crosslink = deepcopy(state.current_crosslinks[shard]) - add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] @@ -75,7 +66,6 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): crosslink_deltas = spec.get_crosslink_deltas(state) - add_block_to_end_of_epoch(spec, state) yield from run_process_crosslinks(spec, state) assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] From 24aa0646c095cb2ef6060345a2813991be9e94cc Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 03:17:46 +0200 Subject: [PATCH 042/110] new process-slashings tests, and epoch processing bugfix with transition-to-excl not working when not yielded from --- .../run_epoch_process_base.py | 23 ++-- .../test_process_crosslinks.py | 4 +- .../test_process_final_updates.py | 6 +- .../test_process_registry_updates.py | 4 +- .../test_process_slashings.py | 121 ++++++++++++++++++ 5 files changed, 142 insertions(+), 16 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index c69160fb0..5b2a2ece4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -12,13 +12,9 @@ process_calls = [ ] -def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): +def run_epoch_processing_to(spec, state, process_name: str): """ - Run the epoch processing functions up to ``process_name``. - If ``exclusive`` is True, the process itself will not be ran. - If ``exclusive`` is False (default), this function yields: - - pre-state ('pre'), state before calling ``process_name`` - - post-state ('post'), state after calling ``process_name`` + Processes to the next epoch transition, up to, but not including, the sub-transition named ``process_name`` """ slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) @@ -36,7 +32,14 @@ def run_epoch_processing_to(spec, state, process_name: str, exclusive=False): if hasattr(spec, name): getattr(spec, name)(state) - if not exclusive: - yield 'pre', state - getattr(spec, process_name)(state) - yield 'post', state + +def run_epoch_processing_with(spec, state, process_name: str): + """ + Processes to the next epoch transition, up to and including the sub-transition named ``process_name`` + - pre-state ('pre'), state before calling ``process_name`` + - post-state ('post'), state after calling ``process_name`` + """ + run_epoch_processing_to(spec, state, process_name) + yield 'pre', state + getattr(spec, process_name)(state) + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 058c93733..599fde8e7 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -12,11 +12,11 @@ from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, ) -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with def run_process_crosslinks(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_crosslinks') + yield from run_epoch_processing_with(spec, state, 'process_crosslinks') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py index d1af7d396..58882a44f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py @@ -1,9 +1,11 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, run_epoch_processing_to +) def run_process_final_updates(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_final_updates') + yield from run_epoch_processing_with(spec, state, 'process_final_updates') @with_all_phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index ae76e95c2..19500d4ab 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,10 +1,10 @@ from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_to +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with def run_process_registry_updates(spec, state): - yield from run_epoch_processing_to(spec, state, 'process_registry_updates') + yield from run_epoch_processing_with(spec, state, 'process_registry_updates') def mock_deposit(spec, state, index): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py new file mode 100644 index 000000000..f1a23326b --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -0,0 +1,121 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, run_epoch_processing_to +) + + +def run_process_slashings(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings') + + +def slash_validators(spec, state, indices, out_epochs): + total_slashed_balance = 0 + for i, out_epoch in zip(indices, out_epochs): + v = state.validators[i] + v.slashed = True + spec.initiate_validator_exit(state, i) + v.withdrawable_epoch = out_epoch + total_slashed_balance += v.effective_balance + + state.slashed_balances[ + spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + ] = total_slashed_balance + + +@with_all_phases +@spec_state_test +def test_max_penalties(spec, state): + slashed_count = (len(state.validators) // 3) + 1 + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + total_balance = spec.get_total_active_balance(state) + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + + assert total_balance // 3 <= total_penalties + + yield from run_process_slashings(spec, state) + + for i in slashed_indices: + assert state.balances[i] == 0 + + +@with_all_phases +@spec_state_test +def test_min_penalties(spec, state): + # run_epoch_processing_to(spec, state, 'process_slashings', exclusive=True) + + # Just the bare minimum for this one validator + pre_balance = state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE + # All the other validators get the maximum. + for i in range(1, len(state.validators)): + state.validators[i].effective_balance = state.balances[i] = spec.MAX_EFFECTIVE_BALANCE + + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slash_validators(spec, state, [0], [out_epoch]) + + total_balance = spec.get_total_active_balance(state) + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + + # we are testing the minimum here, i.e. get slashed (effective_balance / MIN_SLASHING_PENALTY_QUOTIENT) + assert total_penalties * 3 / total_balance < 1 / spec.MIN_SLASHING_PENALTY_QUOTIENT + + yield from run_process_slashings(spec, state) + + assert state.balances[0] == pre_balance - (pre_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT) + + +@with_all_phases +@spec_state_test +def test_scaled_penalties(spec, state): + # skip to next epoch + state.slot = spec.SLOTS_PER_EPOCH + + # Also mock some previous slashings, so that we test to have the delta in the penalties computation. + for i in range(spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR): + state.slashed_balances[i] = spec.MAX_EFFECTIVE_BALANCE * 3 + + # Mock the very last one (which is to be used for the delta balance computation) to be different. + # To enforce the client test runner to correctly get this one from the array, not the others. + prev_penalties = state.slashed_balances[ + (spec.get_current_epoch(state) + 1) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + ] = spec.MAX_EFFECTIVE_BALANCE * 2 + + slashed_count = len(state.validators) // 4 + + assert slashed_count > 10 + + # make the balances non-uniform. + # Otherwise it would just be a simple 3/4 balance slashing. Test the per-validator scaled penalties. + for i in range(10): + state.validators[i].effective_balance += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + state.balances[i] += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + + total_balance = spec.get_total_active_balance(state) + + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + + # Process up to the sub-transition, then Hi-jack and get the balances. + # We just want to test the slashings. + # But we are not interested in the other balance changes during the same epoch transition. + run_epoch_processing_to(spec, state, 'process_slashings') + pre_slash_balances = list(state.balances) + + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + yield 'pre', state + spec.process_slashings(state) + yield 'post', state + + total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties -= prev_penalties + + for i in slashed_indices: + v = state.validators[i] + penalty = v.effective_balance * total_penalties * 3 // total_balance + assert state.balances[i] == pre_slash_balances[i] - penalty From 7a418ed682fc3edd4d2693c4084f0e0eb2ea5130 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Jun 2019 23:40:56 +0200 Subject: [PATCH 043/110] test messed up indices in attester slashings --- .../test_process_attester_slashing.py | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 86b6811a2..226d4a561 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -154,9 +154,73 @@ def test_custody_bit_0_and_1_intersect(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) +@always_bls @with_all_phases @spec_state_test -def test_unsorted_att_1(spec, state): +def test_att1_bad_extra_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices.append(options[len(options) // 2]) # add random index, not previously in attestation. + attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad extra index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls +@with_all_phases +@spec_state_test +def test_att1_bad_replaced_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_1.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. + attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad replaced index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls +@with_all_phases +@spec_state_test +def test_att2_bad_extra_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices.append(options[len(options) // 2]) # add random index, not previously in attestation. + attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad extra index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@always_bls +@with_all_phases +@spec_state_test +def test_att2_bad_replaced_index(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + + indices = attester_slashing.attestation_2.custody_bit_0_indices + options = list(set(range(len(state.validators))) - set(indices)) + indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. + attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), + # see if the bad replaced index is spotted, and slashing is aborted. + + yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +@with_all_phases +@spec_state_test +def test_unsorted_att_1_bit0(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) indices = attester_slashing.attestation_1.custody_bit_0_indices @@ -169,7 +233,7 @@ def test_unsorted_att_1(spec, state): @with_all_phases @spec_state_test -def test_unsorted_att_2(spec, state): +def test_unsorted_att_2_bit0(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) indices = attester_slashing.attestation_2.custody_bit_0_indices @@ -178,3 +242,7 @@ def test_unsorted_att_2(spec, state): sign_indexed_attestation(spec, state, attester_slashing.attestation_2) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) + + +# note: unsorted indices for custody bit 0 are to be introduced in phase 1 testing. + From ff2d711d51ccccff9ad60d6ef6f54c9a379d0d84 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:35:01 +0200 Subject: [PATCH 044/110] test block application on same and on previous slot state --- .../eth2spec/test/sanity/test_blocks.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index bb6a6dc06..8150b06f6 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -11,7 +11,39 @@ from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_all_phases, expect_assertion_error + + +@with_all_phases +@spec_state_test +def test_prev_slot_block_transition(spec, state): + # Go to clean slot + spec.process_slots(state, state.slot + 1) + # Make a block for it + block = build_empty_block(spec, state, slot=state.slot, signed=True) + # Transition to next slot, above block will not be invalid on top of new state. + spec.process_slots(state, state.slot + 1) + + yield 'pre', state + expect_assertion_error(lambda: state_transition_and_sign_block(spec, state, block)) + yield 'blocks', [block] + yield 'post', None + + +@with_all_phases +@spec_state_test +def test_same_slot_block_transition(spec, state): + # Same slot on top of pre-state, but move out of slot 0 first. + spec.process_slots(state, state.slot + 1) + + block = build_empty_block(spec, state, slot=state.slot, signed=True) + + yield 'pre', state + + state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [block] + yield 'post', state @with_all_phases From 8445d1d90c05a29bcc8158ca6c7e9c7754b2df2c Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:37:32 +0200 Subject: [PATCH 045/110] fix formatting for lint --- .../phase_0/block_processing/test_process_attester_slashing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 226d4a561..7fcbd2da5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -245,4 +245,3 @@ def test_unsorted_att_2_bit0(spec, state): # note: unsorted indices for custody bit 0 are to be introduced in phase 1 testing. - From f7b3c87715dae19119ba6b2e6d27dcc8b54d9af8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:43:50 +0200 Subject: [PATCH 046/110] check invalid state root --- .../pyspec/eth2spec/test/sanity/test_blocks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 8150b06f6..0a879dac6 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -66,6 +66,22 @@ def test_empty_block_transition(spec, state): assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.ZERO_HASH +@with_all_phases +@spec_state_test +def test_invalid_state_root(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.state_root = b"\xaa" * 32 + sign_block(spec, state, block) + + expect_assertion_error( + lambda: spec.state_transition(state, block, validate_state_root=True)) + + yield 'blocks', [block] + yield 'post', None + + @with_all_phases @spec_state_test def test_skipped_slots(spec, state): From 235c3d6841d5a29895ea0c9465fd447f5338b92e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 00:58:25 +0200 Subject: [PATCH 047/110] re-enable test_empty_epoch_transition_not_finalizing for minimal config --- .../eth2spec/test/sanity/test_blocks.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 0a879dac6..e56baee8c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -123,26 +123,29 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -# @with_all_phases -# @spec_state_test -# def test_empty_epoch_transition_not_finalizing(spec, state): -# # copy for later balance lookups. -# pre_state = deepcopy(state) -# yield 'pre', state +@with_all_phases +@spec_state_test +def test_empty_epoch_transition_not_finalizing(spec, state): + # Don't run for non-minimal configs, it takes very long, and the effect + # of calling finalization/justifcation is just the same as with the minimal configuration. + if spec.SLOTS_PER_EPOCH > 8: + return -# block = build_empty_block_for_next_slot(spec, state) -# block.slot += spec.SLOTS_PER_EPOCH * 5 -# sign_block(spec, state, block, proposer_index=0) + # copy for later balance lookups. + pre_balances = list(state.balances) + yield 'pre', state -# state_transition_and_sign_block(spec, state, block) + spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH * 5)) + block = build_empty_block_for_next_slot(spec, state, signed=True) + state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block] -# yield 'post', state + yield 'blocks', [block] + yield 'post', state -# assert state.slot == block.slot -# assert state.finalized_epoch < spec.get_current_epoch(state) - 4 -# for index in range(len(state.validators)): -# assert get_balance(state, index) < get_balance(pre_state, index) + assert state.slot == block.slot + assert state.finalized_epoch < spec.get_current_epoch(state) - 4 + for index in range(len(state.validators)): + assert state.balances[index] < pre_balances[index] @with_all_phases From dbb697dadd3b027979baaffaafc62d5932b895a0 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 26 Jun 2019 19:40:11 -0400 Subject: [PATCH 048/110] Small update to typing in BLS spec file [uint384] -> Tuple[uint384, uint384] --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 3fe1bcc0e..b901b9345 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -71,7 +71,7 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: +def hash_to_G2(message_hash: Bytes32, domain: uint64) -> Tuple[uint384, uint384]: # Initial candidate x coordinate x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big') From e49519a53b5c1492fc39978316edafeef88769d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 02:50:49 +0200 Subject: [PATCH 049/110] wrong end epoch test --- .../test_process_attestation.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 59e99ac0c..daac7efed 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -85,6 +85,29 @@ def test_success_since_max_epochs_per_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_wrong_end_epoch_with_max_epochs_per_crosslink(spec, state): + for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): + next_epoch(spec, state) + apply_empty_block(spec, state) + + attestation = get_valid_attestation(spec, state) + data = attestation.data + # test logic sanity check: make sure the attestation only includes MAX_EPOCHS_PER_CROSSLINK epochs + assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK + # Now change it to be different + data.crosslink.end_epoch += 1 + + sign_attestation(spec, state, attestation) + + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(spec, state) + apply_empty_block(spec, state) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @always_bls @spec_state_test From bcfe383e2555ee14261c995044ee3cf503432297 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 08:44:44 +0100 Subject: [PATCH 050/110] WIP --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- specs/core/0_beacon-chain.md | 37 +++++++++++-------- specs/light_client/sync_protocol.md | 4 +- .../pyspec/eth2spec/test/helpers/genesis.py | 4 +- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 9f7ca950f..2aa45cde3 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -10,7 +10,7 @@ SHARD_COUNT: 1024 # 2**7 (= 128) TARGET_COMMITTEE_SIZE: 128 # 2**12 (= 4,096) -MAX_INDICES_PER_ATTESTATION: 4096 +MAX_VALIDATORS_PER_COMMITTEE: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 3e3f7ccb4..417f11c94 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -9,7 +9,7 @@ SHARD_COUNT: 8 # [customized] unsecure, but fast TARGET_COMMITTEE_SIZE: 4 # 2**12 (= 4,096) -MAX_INDICES_PER_ATTESTATION: 4096 +MAX_VALIDATORS_PER_COMMITTEE: 4096 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6cf6db0cc..a7d47e108 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,7 +30,7 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - - [`CompactCommitteees`](#CompactCommitteees) + - [`CompactCommittees`](#CompactCommittees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) @@ -69,7 +69,7 @@ - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - [`get_randao_mix`](#get_randao_mix) - - [`get_compact_committee_root`](#get_compact_committee_root) + - [`get_compact_committees_root`](#get_compact_committees_root) - [`generate_seed`](#generate_seed) - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - [`verify_merkle_branch`](#verify_merkle_branch) @@ -361,14 +361,19 @@ class PendingAttestation(Container): proposer_index: ValidatorIndex ``` -#### `CompactCommitteees` +#### `CompactCommittee` ```python -class CompactCommitteees(Container): - data: Vector[Container( - pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] - ), SHARD_COUNT] +class CompactCommittee(Container): + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + +#### `CompactCommittees` + +```python +class CompactCommittees(Container): + data: Vector[CompactCommittee, SHARD_COUNT] ``` #### `Eth1Data` @@ -522,7 +527,7 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - compact_committee_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients + compact_committees_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Committee digests for light clients # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations @@ -753,14 +758,14 @@ def get_randao_mix(state: BeaconState, return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` -### `get_compact_committee_root` +### `get_compact_committees_root` ```python -def get_compact_committee_root(state: BeaconState, epoch: Epoch) -> Hash: +def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committee_data = CompactCommitteees().data + committee_data = CompactCommittees().data for committee_number in range(get_epoch_committee_count(state, epoch)): shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT for index in get_crosslink_committee(state, epoch, shard): @@ -782,7 +787,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_compact_committee_root(state, epoch) + + get_compact_committees_root(state, epoch) + int_to_bytes(epoch, length=32) ) ``` @@ -1190,9 +1195,9 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate compact_committee_roots + # Populate compact_committees_roots for index in range(EPOCHS_PER_HISTORICAL_VECTOR): - state.compact_committee_roots[index] = get_compact_committee_root(state, GENESIS_EPOCH) + state.compact_committees_roots[index] = get_compact_committees_root(state, GENESIS_EPOCH) return state ``` @@ -1546,7 +1551,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.compact_committee_roots[index_root_position] = get_compact_committee_root(state, next_epoch + ACTIVATION_EXIT_DELAY) + state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch + ACTIVATION_EXIT_DELAY) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf5608..a29ed05c8 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -31,7 +31,7 @@ We define an "expansion" of an object as an object where a field in an object th We define two expansions: -* `ExtendedBeaconState`, which is identical to a `BeaconState` except `active_index_roots: List[Bytes32]` is replaced by `active_indices: List[List[ValidatorIndex]]`, where `BeaconState.active_index_roots[i] = hash_tree_root(ExtendedBeaconState.active_indices[i])`. +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `compact_committees_roots: List[Bytes32]` is replaced by `active_indices: List[List[ValidatorIndex]]`, where `BeaconState.compact_committees_roots[i] = hash_tree_root(ExtendedBeaconState.active_indices[i])`. * `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`. ### `get_active_validator_indices` @@ -40,7 +40,7 @@ Note that there is now a new way to compute `get_active_validator_indices`: ```python def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: - return state.active_indices[epoch % ACTIVE_INDEX_ROOTS_LENGTH] + return state.active_indices[epoch % compact_committees_rootS_LENGTH] ``` Note that it takes `state` instead of `state.validators` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index ce0be19bb..c793254c8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,9 +41,9 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( + genesis_compact_committees_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): - state.active_index_roots[index] = genesis_active_index_root + state.compact_committees_roots[index] = genesis_compact_committees_root return state From 02f6ba36f048d4b79b556df69eed612f660d8e42 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 09:51:06 +0100 Subject: [PATCH 051/110] Add Bitvector and Bitlist Bool, Bit -> boolean, bit Fix simple-serialize.md --- scripts/build_spec.py | 6 +- specs/core/0_beacon-chain.md | 4 +- specs/simple-serialize.md | 87 ++++++++++++++----- test_libs/pyspec/eth2spec/debug/decode.py | 4 +- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 8 +- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 19 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++- .../eth2spec/utils/ssz/test_ssz_impl.py | 10 +-- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 ++-- 11 files changed, 127 insertions(+), 59 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 99c5cd69d..0b1512111 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -52,7 +52,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -174,7 +174,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a..7b9aeee4b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -297,7 +297,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bool + slashed: boolean # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -337,7 +337,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data + custody_bit: bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3318fe45b..53c5649ed 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -15,9 +15,9 @@ - [Default values](#default-values) - [Illegal types](#illegal-types) - [Serialization](#serialization) - - [`"uintN"`](#uintn) - - [`"bool"`](#bool) - - [`"null`](#null) + - [`uintN`](#uintn) + - [`boolean`](#boolean) + - [`null`](#null) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) @@ -37,36 +37,45 @@ ## Typing ### Basic types -* `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) -* `"bool"`: `True` or `False` +* `uintN`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) +* `boolean`: `True` or `False` ### Composite types * **container**: ordered heterogeneous collection of values - * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` + * python dataclass notation with key-type pairs, e.g. +```python +class ContainerExample(Container): + foo: uint64 + bar: boolean +``` * **vector**: ordered fixed-length homogeneous collection of values - * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogeneous collection of values - * angle bracket notation `[type]`, e.g. `["uint64"]` + * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` +* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` + * notation `List[type, N]`, e.g. `List[uint64, N]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type_1, type_2, ...)`, e.g. `("null", "uint64")` + * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` +* **Bitvector**: a fixed-length list of `boolean` values + * notation `Bitvector[N]` +* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` + * notation `Bitlist[N]` ### Variable-size and fixed-size -We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". +We recursively define "variable-size" types to be lists, unions, `Bitlist` and all types that contain a variable-size type. All other types are said to be "fixed-size". ### Aliases For convenience we alias: -* `"byte"` to `"uint8"` (this is a basic type) -* `"bytes"` to `["byte"]` (this is *not* a basic type) -* `"bytesN"` to `["byte", N]` (this is *not* a basic type) -* `"null"`: `{}`, i.e. the empty container +* `bit` to `boolean` +* `byte` to `uint8` (this is a basic type) +* `BytesN` to `Vector[byte, N]` (this is *not* a basic type) +* `null`: `{}`, i.e. the empty container ### Default values -The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. Unions default to the first type in the union (with type index zero), which is `"null"` if present in the union. +The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union. #### `is_empty` @@ -74,34 +83,50 @@ An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization -We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. +We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `bytes`. *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. -### `"uintN"` +### `uintN` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -### `"bool"` +### `boolean` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### `"null"` +### `null` ```python return b"" ``` +### `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((N + 7) // 8, "little") +``` + +### `Bitlist[N]` + +Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. + +```python +as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +``` + ### Vectors, containers, lists, unions ```python @@ -142,17 +167,33 @@ We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. +* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type +### Merkleization of `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return merkleize(as_integer.to_bytes((N + 7) // 8, "little")) +``` + +### `Bitlist[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) +``` + + ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c0b53b0ef..c0b977ab3 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, uint, Container, Bytes, List, boolean, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bool)): + if issubclass(typ, (uint, boolean)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 02814e441..670f580b2 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bool + SSZValue, uint, Container, boolean ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bool): + elif isinstance(value, boolean): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index c6efb722b..cdcba343a 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index e533ca5c2..130956235 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b9c7b6d38..a7f6f9da1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,8 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + Bitlist, Bitvector, uint, ) # SSZ Serialization @@ -13,7 +14,7 @@ BYTES_PER_LENGTH_OFFSET = 4 def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bool): + elif isinstance(value, boolean): if value: return b'\x01' else: @@ -39,6 +40,12 @@ def is_empty(obj: SSZValue): def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) + elif isinstance(obj, Bitvector): + as_integer = sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((len(obj) + 7) // 8, "little") + elif isinstance(obj, Bitlist): + as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") elif isinstance(obj, Series): return encode_series(obj) else: @@ -85,6 +92,12 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values + elif isinstance(values, Bitvector): + as_integer = sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitlist): + as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) @@ -134,7 +147,7 @@ def hash_tree_root(obj: SSZValue): else: raise Exception(f"Type not supported: {type(obj)}") - if isinstance(obj, (List, Bytes)): + if isinstance(obj, (List, Bytes, Bitlist)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 58e66ca68..ea07359b2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bool(BasicValue): # can't subclass bool. +class boolean(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) @@ -48,7 +48,7 @@ class Bool(BasicValue): # can't subclass bool. # Alias for Bool -class Bit(Bool): +class bit(boolean): pass @@ -233,7 +233,7 @@ class ParamsMeta(SSZType): return f"{self.__name__}~{self.__class__.__name__}" def __repr__(self): - return self, self.__class__ + return f"{self.__name__}~{self.__class__.__name__}" def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. @@ -280,11 +280,12 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int +class BitElementsType(ElementsType): + elem_type = boolean class Elements(ParamsBase, metaclass=ElementsType): pass - class BaseList(list, Elements): def __init__(self, *args): @@ -310,6 +311,10 @@ class BaseList(list, Elements): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __repr__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __getitem__(self, k) -> SSZValue: if isinstance(k, int): # check if we are just doing a lookup, and not slicing if k < 0: @@ -337,6 +342,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] +class BaseBitfield(BaseList, metaclass=BitElementsType): + elem_type = bool + +class Bitlist(BaseBitfield): + pass + +class Bitvector(BaseBitfield): + pass + class List(BaseList): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 82fb4ec68..33badcf4a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, BytesN, + bit, boolean, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -74,10 +74,10 @@ def merge(a: str, branch: Iterable[str]) -> str: test_data = [ - ("bit F", Bit(False), "00", chunk("00")), - ("bit T", Bit(True), "01", chunk("01")), - ("bool F", Bool(False), "00", chunk("00")), - ("bool T", Bool(True), "01", chunk("01")), + ("bit F", bit(False), "00", chunk("00")), + ("bit T", bit(True), "01", chunk("01")), + ("boolean F", boolean(False), "00", chunk("00")), + ("boolean T", boolean(True), "01", chunk("01")), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 2af742360..f746a29c9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,6 +1,6 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + Elements, bit, boolean, Container, List, Vector, Bytes, BytesN, byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bool, BasicValue) - assert isinstance(Bool, BasicType) + assert issubclass(boolean, BasicValue) + assert isinstance(boolean, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -45,16 +45,16 @@ def test_basic_instances(): assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bool(True), BasicValue) - assert isinstance(Bool(False), BasicValue) - assert isinstance(Bit(True), Bool) - assert isinstance(Bit(False), Bool) + assert isinstance(boolean(True), BasicValue) + assert isinstance(boolean(False), BasicValue) + assert isinstance(bit(True), boolean) + assert isinstance(bit(False), boolean) def test_basic_value_bounds(): max = { - Bool: 2 ** 1, - Bit: 2 ** 1, + boolean: 2 ** 1, + bit: 2 ** 1, uint8: 2 ** (8 * 1), byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), From 23c743570e5b42119f7e7fdb9ffcf8584e98c1d1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:26:45 +0100 Subject: [PATCH 052/110] Add some tests and fix pack --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a7f6f9da1..d6689892d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -96,7 +96,7 @@ def pack(values: Series): as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") elif isinstance(values, Bitlist): - as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 33badcf4a..1522ce200 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -2,6 +2,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, BytesN, + Bitlist, Bitvector, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -78,6 +79,10 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 494984f7d3129d8d3c4798d285824b59dc7f68b2 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:42:14 +0100 Subject: [PATCH 053/110] Fix linting errors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 ++++++ test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d6689892d..7298fb3ca 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -26,7 +26,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bool): + elif issubclass(typ, boolean): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea07359b2..047abd4fe 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -280,12 +280,15 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int + class BitElementsType(ElementsType): elem_type = boolean + class Elements(ParamsBase, metaclass=ElementsType): pass + class BaseList(list, Elements): def __init__(self, *args): @@ -342,12 +345,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] + class BaseBitfield(BaseList, metaclass=BitElementsType): elem_type = bool + class Bitlist(BaseBitfield): pass + class Bitvector(BaseBitfield): pass diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 1522ce200..63f0c835d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,10 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), - ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), - ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), - ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), - ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From d641e941519efc8cacd5e2ddabcaaf1722aa0797 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 11:30:23 +0100 Subject: [PATCH 054/110] Cleanups --- specs/simple-serialize.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 53c5649ed..97b1d560c 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -43,7 +43,7 @@ ### Composite types * **container**: ordered heterogeneous collection of values - * python dataclass notation with key-type pairs, e.g. + * python dataclass notation with key-type pairs, e.g. ```python class ContainerExample(Container): foo: uint64 @@ -53,12 +53,12 @@ class ContainerExample(Container): * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` * **list**: ordered variable-length homogeneous collection of values, with maximum length `N` * notation `List[type, N]`, e.g. `List[uint64, N]` +* **bitvector**: ordered fixed-length collection of `boolean` values + * notation `Bitvector[N]` +* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` + * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` -* **Bitvector**: a fixed-length list of `boolean` values - * notation `Bitvector[N]` -* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` - * notation `Bitlist[N]` ### Variable-size and fixed-size @@ -193,7 +193,6 @@ as_integer = sum([value[i] << i for i in range(len(value))]) return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) ``` - ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. From 67c50cb197824a015ff9946fb6fd5c684e9e9639 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 11:45:35 +0100 Subject: [PATCH 055/110] Changed attestation and custody bitfields --- scripts/build_spec.py | 6 +-- specs/core/0_beacon-chain.md | 41 +++---------------- .../pyspec/eth2spec/debug/random_value.py | 6 +-- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 11 +++++ .../eth2spec/test/helpers/attestations.py | 9 ++-- .../pyspec/eth2spec/test/helpers/bitfields.py | 11 ----- .../test_process_attestation.py | 8 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 +++- 8 files changed, 38 insertions(+), 62 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b1512111..0a41fe5c0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,7 +26,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -53,7 +53,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -175,7 +175,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' - 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc ] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b9aeee4b..13ab4cdae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -80,8 +80,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - - [`get_bitfield_bit`](#get_bitfield_bit) - - [`verify_bitfield`](#verify_bitfield) - [`convert_to_indexed`](#convert_to_indexed) - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) @@ -354,7 +352,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -421,9 +419,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -865,13 +863,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Sequence[ValidatorIndex]: + bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) - assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return sorted([index for i, index in enumerate(committee) if bitfield[i]]) ``` ### `int_to_bytes` @@ -912,34 +909,6 @@ def get_domain(state: BeaconState, return bls_domain(domain_type, fork_version) ``` -### `get_bitfield_bit` - -```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -``` - -### `verify_bitfield` - -```python -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -``` - ### `convert_to_indexed` ```python diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index cdcba343a..95a3ae970 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -3,7 +3,7 @@ from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, - Vector, BytesN + Vector, BytesN, Bitlist, Bitvector ) # in bytes @@ -83,12 +83,12 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif issubclass(typ, Vector): + elif issubclass(typ, Vector) or issubclass(typ, Bitvector): return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ) - elif issubclass(typ, List): + elif issubclass(typ, List) or issubclass(typ, Bitlist): length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 130956235..ccca17385 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -18,7 +18,14 @@ def translate_typ(typ) -> ssz.BaseSedes: elif issubclass(typ, spec_ssz.Vector): return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): + # TODO: Make py-ssz List support the new fixed length list return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitlist): + # TODO: Once Bitlist implemented in py-ssz, use appropriate type + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitvector): + # TODO: Once Bitvector implemented in py-ssz, use appropriate type + return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): @@ -68,6 +75,10 @@ def translate_value(value, typ): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.Bitlist): + return typ(value) + elif issubclass(typ, spec_ssz.Bitvector): + return typ(value) elif issubclass(typ, spec_ssz.BytesN): return typ(value) elif issubclass(typ, spec_ssz.Bytes): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7eb..2e637d42f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -1,10 +1,10 @@ from typing import List -from eth2spec.test.helpers.bitfields import set_bitfield_bit from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import Bitlist def build_attestation_data(spec, state, slot, shard): @@ -69,9 +69,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\x00' * bitfield_length - custody_bitfield = b'\x00' * bitfield_length + aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( aggregation_bitfield=aggregation_bitfield, data=attestation_data, @@ -138,7 +137,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + attestation.aggregation_bitfield[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py deleted file mode 100644 index 50e5b6cba..000000000 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ /dev/null @@ -1,11 +0,0 @@ -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..ee76ab23d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -10,6 +10,7 @@ from eth2spec.test.helpers.state import ( next_slot, ) from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_attestation_processing(spec, state, attestation, valid=True): @@ -281,7 +282,10 @@ def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + custody_bitfield = deepcopy(attestation.aggregation_bitfield) + custody_bitfield.append(False) + + attestation.custody_bitfield = custody_bitfield sign_attestation(spec, state, attestation) @@ -307,7 +311,7 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 047abd4fe..d0aef1b44 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -351,11 +351,15 @@ class BaseBitfield(BaseList, metaclass=BitElementsType): class Bitlist(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return False class Bitvector(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return True class List(BaseList): From becb7a032ab4ef1cbc2fe0e3d12ae381250f6175 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:14:36 +0100 Subject: [PATCH 056/110] justification_bitfield -> Bitvector[4] --- specs/core/0_beacon-chain.md | 16 ++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 13ab4cdae..c1f1a7bcf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -523,7 +523,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: uint64 # Bit set for every recent justified epoch + justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,38 +1291,38 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 + state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 1) + state.justification_bitfield[1] = True current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 0) + state.justification_bitfield[0] = True # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if all(bitfield[1:4]) and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if all(bitfield[1:3]) and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if all(bitfield[0:3]) and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if all(bitfield[0:2]) and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d0aef1b44..6ce2b1538 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -355,12 +355,19 @@ class Bitlist(BaseBitfield): def is_fixed_size(cls): return False + @classmethod + def default(cls): + return cls() + class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + + @classmethod + def default(cls): + return cls(0 for _ in range(cls.length)) class List(BaseList): From 80c680e6147ffc3971856802b4e54d2b776c6862 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:41:22 +0100 Subject: [PATCH 057/110] Phase 1 to Bitvector/Bitlist --- specs/core/1_custody-game.md | 12 +++++++++++- specs/core/1_shard-data-chains.md | 5 ++--- specs/light_client/sync_protocol.md | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 07f6ec698..17f599fcd 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,6 +272,16 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` +### `get_bitfield_bit` + +```python +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +``` + ### `get_custody_chunk_bit` ```python @@ -566,7 +576,7 @@ def process_bit_challenge(state: BeaconState, chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d1c86eca2..b2a5ea9ea 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bytes[PLACEHOLDER] + aggregation_bitfield: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -230,10 +230,9 @@ def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data persistent_committee = get_persistent_committee(state, data.shard, data.slot) - assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: + if attestation.aggregation_bitfield[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf5608..a1b5777cf 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': 'bytes', + 'shard_bitfield': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if get_bitfield_bit(proof.shard_bitfield, index) is True + if proof.shard_bitfield[index] ]) assert bls_verify( pubkey=group_public_key, From f57387cc83f16e2ec6b19ee544964254786fa7fa Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 13:09:48 +0100 Subject: [PATCH 058/110] Justification bitvector length to constant --- specs/core/0_beacon-chain.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c1f1a7bcf..4d3fe9773 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,6 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -523,7 +524,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch + justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,21 +1292,21 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) + state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[1] = True + state.justification_bitfield[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[0] = True + state.justification_bitfield[0] = 0b1 # Process finalizations bitfield = state.justification_bitfield From a5154da1ff76e8b768e64987471820f958c391d1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 15:40:40 +0200 Subject: [PATCH 059/110] suggestion to implement bitfield like --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 7298fb3ca..f0ee944bd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,6 +128,8 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 + elif issubclass(typ, Bitfield): + return (typ.length + 7) // 8 // 32 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..53ab42743 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -281,10 +281,6 @@ class ElementsType(ParamsMeta): length: int -class BitElementsType(ElementsType): - elem_type = boolean - - class Elements(ParamsBase, metaclass=ElementsType): pass @@ -346,11 +342,16 @@ class BaseList(list, Elements): return self[len(self) - 1] -class BaseBitfield(BaseList, metaclass=BitElementsType): - elem_type = bool +class BitElementsType(ElementsType): + elem_type: SSZType = boolean + length: int -class Bitlist(BaseBitfield): +class Bitfield(BaseList, metaclass=BitElementsType): + pass + + +class Bitlist(Bitfield): @classmethod def is_fixed_size(cls): return False @@ -360,15 +361,29 @@ class Bitlist(BaseBitfield): return cls() -class Bitvector(BaseBitfield): +class Bitvector(Bitfield): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) + + @classmethod + def value_check(cls, value): + # check length limit strictly + return len(value) == cls.length and super().value_check(value) + @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From b574a581094b8a00b360de573bee6a949b44650a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 14:45:17 +0100 Subject: [PATCH 060/110] Remove not working py-ssz decoder tests --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 26ee6e913..b362d503b 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,9 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: + # TODO: Once has Bitfields and Bitvectors, add back + # spec.BeaconState and spec.BeaconBlock + for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, From 8ed638bb84ae3dd88e6658f7bfd5eeda874b97f9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 15:21:04 +0100 Subject: [PATCH 061/110] Linter fixes --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/phase_0/block_processing/test_process_attestation.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index b362d503b..c707c840a 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitfields and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index ee76ab23d..d17d93e6d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -311,7 +311,8 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..85f5e0bdb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,11 +364,12 @@ class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From afd86f71de09e556a7a4cc5d82a38b8d514e0e93 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:31:33 +0100 Subject: [PATCH 062/110] Fixes in ssz impl --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index f0ee944bd..53075845b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -129,7 +129,7 @@ def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 elif issubclass(typ, Bitfield): - return (typ.length + 7) // 8 // 32 + return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): From 93ce1688629f45f92f7a261a958e99351299606a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:47:48 +0100 Subject: [PATCH 063/110] More linting fixes --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 3 ++- specs/core/1_custody-game.md | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0a41fe5c0..1f5fe1ee6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, uint64, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d3fe9773..f596da520 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,7 +1292,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( + *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 17f599fcd..9ad00516e 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -574,7 +574,6 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) - assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) From 853f5fc3f0a8ff671ec9bf946d5cf2a84e1c4f9e Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 19:05:27 +0100 Subject: [PATCH 064/110] Apply Danny's suggestions --- specs/core/0_beacon-chain.md | 45 ++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a7d47e108..ba2cf771b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -30,10 +30,10 @@ - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - - [`CompactCommittees`](#CompactCommittees) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) + - [`CompactCommittee`](#compactcommittee) - [`BeaconBlockHeader`](#beaconblockheader) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) @@ -361,21 +361,6 @@ class PendingAttestation(Container): proposer_index: ValidatorIndex ``` -#### `CompactCommittee` - -```python -class CompactCommittee(Container): - pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] - compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] -``` - -#### `CompactCommittees` - -```python -class CompactCommittees(Container): - data: Vector[CompactCommittee, SHARD_COUNT] -``` - #### `Eth1Data` ```python @@ -403,6 +388,14 @@ class DepositData(Container): signature: BLSSignature ``` +#### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[Bytes48, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + #### `BeaconBlockHeader` ```python @@ -765,16 +758,18 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committee_data = CompactCommittees().data + committees = Vector[CompactCommittee, SHARD_COUNT]() + start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): - shard = (get_epoch_start_shard(state, epoch) + committee_number) % SHARD_COUNT + shard = (start_shard + committee_number) % SHARD_COUNT for index in get_crosslink_committee(state, epoch, shard): - validator = validators[index] - committee_data[shard].pubkeys.append(validator.pubkey) - # `index` (top 7 bytes) + `slashed` (8th bit) + `effective_balance` (bottom 7 bits) - compact_validator = index << 8 + validator.slashed << 7 + validator.effective_balance // GWEI_PER_ETH - committee_data[shard].compact_validators.append(compact_validator) - return hash_tree_root(committee_data) + validator = state.validators[index] + committees[shard].pubkeys.append(validator.pubkey) + compact_balance = validator.effective_balance // EFFECTIVE_BALANCE_INCREMENT + # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) + compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) + committees[shard].compact_validators.append(compact_validator) + return hash_tree_root(committees) ``` ### `generate_seed` @@ -787,7 +782,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - get_compact_committees_root(state, epoch) + + state.compact_committees_roots[epoch] + int_to_bytes(epoch, length=32) ) ``` From 7adf07ea5f1fca3696cc52cfa6e7127aa7e80883 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 22:58:44 +0100 Subject: [PATCH 065/110] A few more tests for Bitvector/Bitlist --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 63f0c835d..88ccc838c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -85,6 +85,16 @@ test_data = [ "c5c2", chunk("c5c2")), ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("long bitvector", Bitvector[512](1 for i in range(512)), + "ff" * 64, h("ff" * 32, "ff" * 32)), + ("long bitlist", Bitlist[512](1), + "03", h(h(chunk("01"), chunk("")), chunk("01"))), + ("long bitlist", Bitlist[512](1 for i in range(512)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), chunk("0002"))), + ("odd bitvector", Bitvector[513](1 for i in range(513)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk("")))), + ("odd bitlist", Bitlist[513](1 for i in range(513)), + "ff" * 64 + "03", h(h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk(""))), chunk("0102"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 384fa8854a92666694aabd5b4c923d2968438148 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 00:19:55 +0200 Subject: [PATCH 066/110] justification/finalization testing groundwork --- ..._process_justification_and_finalization.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py new file mode 100644 index 000000000..95f00edd4 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -0,0 +1,74 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with +) + + +def run_process_just_and_fin(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_justification_and_finalization') + + +def get_committee_size(spec, state, slot): + epoch = spec.slot_to_epoch(slot) + epoch_start_shard = spec.get_epoch_start_shard(state, epoch) + committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT + committee_index = (shard + spec.SHARD_COUNT - spec.get_epoch_start_shard(state, epoch)) % spec.SHARD_COUNT + committee_count = spec.get_epoch_committee_count(state, epoch) + indices = spec.get_active_validator_indices(state, epoch) + start = (len(indices) * committee_index) // committee_count + end = (len(indices) * (committee_index + 1)) // committee_count + size = end - start + return size + + +def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): + # we must be at the end of the epoch + assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 + + previous_epoch = spec.get_previous_epoch(state) + current_epoch = spec.get_current_epoch(state) + + if current_epoch == target_epoch: + attestations = state.current_epoch_attestations + elif previous_epoch == target_epoch: + attestations = state.previous_epoch_attestations + else: + raise Exception(f"cannot target epoch ${target_epoch} from epoch ${current_epoch}") + + total = 0 + while total < att_count: + for i in range(spec.SLOTS_PER_EPOCH): + size = get_committee_size(spec, state, state.slot + i) + # Create a bitfield filled with the given count per attestation, + # exactly on the right-most part of the committee field. + attesting_count = int(size * att_ratio) + aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') + + attestations.append(spec.PendingAttestation( + aggregation_bitfield=aggregation_bitfield, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source_epoch=0, + source_root=b'\xbb' * 32, + target_root=b'\xbb' * 32, + crosslink=spec.Crosslink() + ), + inclusion_delay=0, + )) + total += 1 + + +@with_all_phases +@spec_state_test +def test_rule_1(spec, state): + previous_epoch = spec.get_previous_epoch(state) + current_epoch = spec.get_current_epoch(state) + + # TODO + # add_mock_attestations(spec, state, ...) + # get indices attesting e.g. current_epoch_attestations + # set their balances + # yield from run_process_just_and_fin(spec, state) + # check finalization + From 990cc55db74628c045ba9919a65c2d9f5d8d0692 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 27 Jun 2019 16:32:10 -0600 Subject: [PATCH 067/110] fix committee typing error --- motes.md | 42 ++++++++++++++++++++++++++++++++++++ notes.txt | 1 + specs/core/0_beacon-chain.md | 4 ++-- 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 motes.md create mode 100644 notes.txt diff --git a/motes.md b/motes.md new file mode 100644 index 000000000..e01ba19b7 --- /dev/null +++ b/motes.md @@ -0,0 +1,42 @@ +* `BLS_WITHDRAWAL_PREFIX` + * Why int rather than bytes? +* `MIN_SEED_LOOKAHEAD` + * Is this actually tunable? + * If so, what are the reprecussions? +* `ACTIVATION_EXIT_DELAY` + * Reaquaint with purpose +* AttesterSlashings + * `MAX_ATTESTER_SLASHINGS` is 1. + * Are there scenarios in which validators can create more effective slashable + messages than can be included on chain? For example, Validators split up to + create double attestations for checkpoints but different (junk) crosslink + data to prevent them from being aggregatable to the fullest + * Max is for block size, no? +* Signature domains + * Max 4byte ints +* `Version` not defined in one of the lists of custom types (2!!). ensure in spec +* `PendingAttestation` + * Don't think `proposer_index` is actually necessary here because effective + balance is stable until end of epoch so can do dynamic lookups +* is_genesis_trigger + * only run at ends of blocks to preserve invariant that eth1data.deposit_root + is the deposit root at the _end_ of an eth1 block +* `Attestation` + * why bitfields not together? +* `Transfer` + * replay mechanism... say the slot gets missed and you sign another transfer + * in a fork you could include both transfers +* `get_previous_epoch` + * do a once over on the genesis stuff +* `get_epoch_start_shard` + * checking next hinges upon the fact that the validator set for the next + epoch is 100% known at the current epoch. Ensure this is the case +* `get_block_root_at_slot` .. `generate_seed` can be bade into one line + function signatures +* `get_shuffled_index` + * I think it should be maybe `assert index_count <= VALIDATOR_REGISTRY_LIMIT` + * is the `2**40` special for security of alg? probably. + + +pubkey/privkey g1 vs g2 + diff --git a/notes.txt b/notes.txt new file mode 100644 index 000000000..421ad7750 --- /dev/null +++ b/notes.txt @@ -0,0 +1 @@ +* `BLS_WITHDRAWAL_PREFIX` -- diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ba2cf771b..842b16b98 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -758,7 +758,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: """ Return the compact committee root for the current epoch. """ - committees = Vector[CompactCommittee, SHARD_COUNT]() + committees = [CompactCommittee() for _ in range(SHARD_COUNT)] start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): shard = (start_shard + committee_number) % SHARD_COUNT @@ -769,7 +769,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) committees[shard].compact_validators.append(compact_validator) - return hash_tree_root(committees) + return hash_tree_root(Vector[CompactCommittee, SHARD_COUNT](committees)) ``` ### `generate_seed` From 237b41df5b41a37972c66c51d5acd551134189b5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:18:54 +0100 Subject: [PATCH 068/110] Slice notation for justification_bitfield shift --- specs/core/0_beacon-chain.md | 5 +++-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f596da520..0ea24a255 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,8 +1292,9 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( - *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] + state.justification_bitfield[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 53ab42743..ba773b443 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -324,12 +324,18 @@ class BaseList(list, Elements): return super().__getitem__(k) def __setitem__(self, k, v): - if k < 0: - raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self): - raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self)})") - super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + if type(k) == slice: + if k.start < 0 or k.stop > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds slice {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) + else: + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) From 2677d233a830b07a7f6f60392a1a54b888d3ac60 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:31:37 +0100 Subject: [PATCH 069/110] Some more (shorter) Bitvector and Bitlist tests --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 88ccc838c..637d9c5c4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,6 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TTFTFTFF", Bitvector[8](1, 1, 0, 1, 0, 1, 0, 0), "2b", chunk("2b")), + ("bitlist TTFTFTFF", Bitlist[8](1, 1, 0, 1, 0, 1, 0, 0), "2b01", h(chunk("2b"), chunk("08"))), + ("bitvector FTFT", Bitvector[4](0, 1, 0, 1), "0a", chunk("0a")), + ("bitlist FTFT", Bitlist[4](0, 1, 0, 1), "1a", h(chunk("0a"), chunk("04"))), + ("bitvector FTF", Bitvector[3](0, 1, 0), "02", chunk("02")), + ("bitlist FTF", Bitlist[3](0, 1, 0), "0a", h(chunk("02"), chunk("03"))), ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), From 196ac4202595a716a7acba91e7a437d80cc1a4ad Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:23:22 +0100 Subject: [PATCH 070/110] Cleanup naming --- specs/core/0_beacon-chain.md | 49 +++++++++---------- specs/core/1_custody-game.md | 26 +++++----- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 20 ++++---- specs/validator/beacon_node_oapi.yaml | 8 +-- .../pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/fork_choice/test_on_attestation.py | 2 +- .../eth2spec/test/helpers/attestations.py | 12 ++--- .../test_process_attestation.py | 18 +++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +-- 12 files changed, 79 insertions(+), 80 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9ce7976b5..d41173424 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -190,7 +190,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | +| `JUSTIFICATION_BITS_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -360,7 +360,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -427,9 +427,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + custody_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -527,7 +527,7 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Finality - justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint @@ -867,12 +867,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, data: AttestationData, - bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: + bits: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bits``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) - return sorted([index for i, index in enumerate(committee) if bitfield[i]]) + return sorted([index for i, index in enumerate(committee) if bits[i]]) ``` ### `int_to_bytes` @@ -920,8 +920,8 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ Convert ``attestation`` to (almost) indexed-verifiable form. """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) assert set(custody_bit_1_indices).issubset(attesting_indices) custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] @@ -1146,7 +1146,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1254,7 +1254,7 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: - output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` @@ -1294,36 +1294,35 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ - state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] - state.justification_bitfield[0] = 0b0 + state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) - state.justification_bitfield[1] = 0b1 + state.justification_bits[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) - state.justification_bitfield[0] = 0b1 + state.justification_bits[0] = 0b1 # Process finalizations - bitfield = state.justification_bitfield + bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if all(bitfield[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if all(bitfield[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if all(bitfield[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if all(bitfield[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` @@ -1379,7 +1378,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations - if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) + if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward @@ -1661,7 +1660,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=data, - aggregation_bitfield=attestation.aggregation_bitfield, + aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), ) @@ -1680,7 +1679,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] - + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9ad00516e..883476c58 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,14 +272,14 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` -### `get_bitfield_bit` +### `get_bit` ```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: +def get_bit(serialization: bytes, i: int) -> int: """ - Extract the bit in ``bitfield`` at position ``i``. + Extract the bit in ``serialization`` at position ``i``. """ - return (bitfield[i // 8] >> (i % 8)) % 2 + return (serialization[i // 8] >> (i % 8)) % 2 ``` ### `get_custody_chunk_bit` @@ -287,17 +287,17 @@ def get_bitfield_bit(bitfield: bytes, i: int) -> int: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return bool(get_bitfield_bit(hash(key + chunk), 0)) + return bool(get_bit(hash(key + chunk), 0)) ``` ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: +def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: aggregated_bits = bytearray([0] * 32) - for i in range(0, len(chunk_bitfield), 32): + for i in range(0, len(chunk_bits), 32): for j in range(32): - aggregated_bits[j] ^= chunk_bitfield[i + j] + aggregated_bits[j] ^= chunk_bits[i + j] return hash(aggregated_bits) ``` @@ -489,7 +489,7 @@ def process_chunk_challenge(state: BeaconState, responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) assert challenge.responder_index in attesters # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: @@ -546,7 +546,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert challenge.responder_index in attesters # A validator can be the challenger for at most one challenge at a time @@ -575,8 +575,8 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] - assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) + custody_bit = attestation.custody_bits[attesters.index(challenge.responder_index)] + assert custody_bit != get_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, @@ -670,7 +670,7 @@ def process_bit_challenge_response(state: BeaconState, ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b2a5ea9ea..432820b7c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bitlist[PLACEHOLDER] + aggregation_bits: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -232,7 +232,7 @@ def verify_shard_attestation_signature(state: BeaconState, persistent_committee = get_persistent_committee(state, data.shard, data.slot) pubkeys = [] for i, index in enumerate(persistent_committee): - if attestation.aggregation_bitfield[i]: + if attestation.aggregation_bits[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a1b5777cf..3f5a37289 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': Bitlist[PLACEHOLDER], + 'shard_bits': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bits[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if proof.shard_bitfield[index] + if proof.shard_bits[index] ]) assert bls_verify( pubkey=group_public_key, @@ -196,4 +196,4 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bits) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66..813b8bcfe 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -44,8 +44,8 @@ - [Crosslink vote](#crosslink-vote) - [Construct attestation](#construct-attestation) - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) + - [Aggregation bits](#aggregation-bits) + - [Custody bits](#custody-bits) - [Aggregate signature](#aggregate-signature) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) @@ -322,19 +322,19 @@ Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-c Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). -##### Aggregation bitfield +##### Aggregation bits -* Let `aggregation_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. * Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bitfield[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bitfield = aggregation_bitfield`. +* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. +* Set `attestation.aggregation_bits = aggregation_bits`. -*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. -##### Custody bitfield +##### Custody bits -* Let `custody_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bitfield = custody_bitfield`. +* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Set `attestation.custody_bits = custody_bits`. *Note*: This is a stub for Phase 0. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 74be21fac..4da8f7933 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -415,16 +415,16 @@ components: type: object description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: - aggregation_bitfield: + aggregation_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Attester aggregation bitfield." - custody_bitfield: + description: "Attester aggregation bits." + custody_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield." + description: "Custody bits." signature: type: string format: byte diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index c707c840a..ea1f1d47f 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitlists and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 1a67e01d5..5adb022a6 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -118,5 +118,5 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid - attestation.custody_bitfield[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] + attestation.custody_bits[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index a184a5f5e..6fdf1538b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -67,12 +67,12 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) - custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=attestation_data, - custody_bitfield=custody_bitfield, + custody_bits=custody_bits, ) fill_aggregate_attestation(spec, state, attestation) if signed: @@ -105,7 +105,7 @@ def sign_attestation(spec, state, attestation): participants = spec.get_attesting_indices( state, attestation.data, - attestation.aggregation_bitfield, + attestation.aggregation_bits, ) attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) @@ -135,7 +135,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield[i] = True + attestation.aggregation_bits[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 1ea7c92cf..916724001 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -275,14 +275,14 @@ def test_bad_crosslink_end_epoch(spec, state): @with_all_phases @spec_state_test -def test_inconsistent_bitfields(spec, state): +def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bitfield = deepcopy(attestation.aggregation_bitfield) - custody_bitfield.append(False) + custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits.append(False) - attestation.custody_bitfield = custody_bitfield + attestation.custody_bits = custody_bits sign_attestation(spec, state, attestation) @@ -291,11 +291,11 @@ def test_inconsistent_bitfields(spec, state): @with_phases(['phase0']) @spec_state_test -def test_non_empty_custody_bitfield(spec, state): +def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + attestation.custody_bits = deepcopy(attestation.aggregation_bits) sign_attestation(spec, state, attestation) @@ -304,12 +304,12 @@ def test_non_empty_custody_bitfield(spec, state): @with_all_phases @spec_state_test -def test_empty_aggregation_bitfield(spec, state): +def test_empty_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( - *([0b0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bits))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 53075845b..d5855a755 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,7 +128,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 - elif issubclass(typ, Bitfield): + elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ba773b443..91d3ff4f2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -353,11 +353,11 @@ class BitElementsType(ElementsType): length: int -class Bitfield(BaseList, metaclass=BitElementsType): +class Bits(BaseList, metaclass=BitElementsType): pass -class Bitlist(Bitfield): +class Bitlist(Bits): @classmethod def is_fixed_size(cls): return False @@ -367,7 +367,7 @@ class Bitlist(Bitfield): return cls() -class Bitvector(Bitfield): +class Bitvector(Bits): @classmethod def extract_args(cls, *args): From 6f9d37485d624cb96188714db495cdb07ace9aa6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:34:01 +0100 Subject: [PATCH 071/110] Cleanups --- specs/core/0_beacon-chain.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d41173424..9de024909 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,20 +1294,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] - state.justification_bits[0] = 0b0 - previous_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, previous_epoch) - ) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, - root=get_block_root(state, previous_epoch)) + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - current_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, current_epoch) - ) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From e36593b155d3fdffdb8ebe451236d5dfb54e22c9 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:35:50 +0100 Subject: [PATCH 072/110] Add comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9de024909..96b8981fe 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) From c764202a5768003bfaf354e90e591f66c48c9679 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 28 Jun 2019 21:35:26 +0800 Subject: [PATCH 073/110] Slashing penalty calculation change (#1217) If the exit queue is very long, then a validator may take many months to exit. With the code as currently written, however, self-slashing is a potentially lucrative route to get one's money out faster, because one can exit in 36 days. This PR changes it so that slashing can only extend your withdrawal time, not contract it. Also, instead of the slashed balances used to calculate one's slashing penalty being those in `[withdrawal - 54 days ... withdrawal - 18 days]`, we now run the penalization algorithm once every 36 days that a validator is slashed but not withdrawn, so that it covers the 36-day period where the validator was actually slashed. It also moves the minimum slashing penalty to the `slash_validator` function so that it is only applied once. We also simplify the `slashed_balances` logic to be per-epoch. --- configs/constant_presets/mainnet.yaml | 4 +- configs/constant_presets/minimal.yaml | 4 +- specs/core/0_beacon-chain.md | 46 ++++------- specs/core/1_custody-game.md | 2 +- .../test_process_attester_slashing.py | 82 ++++++++++++++++--- 5 files changed, 92 insertions(+), 46 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 9f7ca950f..38f10c8fc 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -76,7 +76,7 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # 2**16 (= 65,536) epochs ~0.8 years EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +EPOCHS_PER_SLASHINGS_VECTOR: 8192 # 2**24 (= 16,777,216) historical roots, ~26,131 years HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -88,7 +88,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 3e3f7ccb4..3aa4a6b71 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -77,7 +77,7 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # [customized] smaller state EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +EPOCHS_PER_SLASHINGS_VECTOR: 64 # 2**24 (= 16,777,216) historical roots HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -89,7 +89,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1b7cda0a4..61ef8b742 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -234,7 +234,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | -| `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | | `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | @@ -243,7 +243,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `BASE_REWARD_FACTOR` | `2**6` (= 64) | -| `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | +| `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | @@ -520,7 +520,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1097,21 +1097,22 @@ def slash_validator(state: BeaconState, """ Slash the validator with index ``slashed_index``. """ - current_epoch = get_current_epoch(state) + epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - state.validators[slashed_index].slashed = True - state.validators[slashed_index].withdrawable_epoch = Epoch(current_epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR) - slashed_balance = state.validators[slashed_index].effective_balance - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] += slashed_balance + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + # Apply proposer and whistleblower rewards proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index - whistleblowing_reward = Gwei(slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT) - proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) - decrease_balance(state, slashed_index, whistleblowing_reward) + increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) ``` ## Genesis @@ -1174,7 +1175,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1493,18 +1494,9 @@ def process_registry_updates(state: BeaconState) -> None: def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) - - # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_at_end = state.slashed_balances[epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_penalties = total_at_end - total_at_start - for index, validator in enumerate(state.validators): - if validator.slashed and epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 == validator.withdrawable_epoch: - penalty = max( - validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, - validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT - ) + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + penalty = validator.effective_balance * min(sum(state.slashings) * 3, total_balance) // total_balance decrease_balance(state, ValidatorIndex(index), penalty) ``` @@ -1532,10 +1524,8 @@ def process_final_updates(state: BeaconState) -> None: get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) - # Set total slashed balances - state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - ) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 47d1578c5..c29033fe3 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -453,7 +453,7 @@ def process_early_derived_secret_reveal(state: BeaconState, # Apply penalty proposer_index = get_beacon_proposer_index(state) whistleblower_index = reveal.masker_index - whistleblowing_reward = Gwei(penalty // WHISTLEBLOWING_REWARD_QUOTIENT) + whistleblowing_reward = Gwei(penalty // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index e2b50ea0b..e78e1a866 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -25,31 +25,56 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', None return - slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - pre_slashed_balance = get_balance(state, slashed_index) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) + pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} + pre_withdrawalable_epochs = { + slashed_index: state.validators[slashed_index].withdrawable_epoch + for slashed_index in slashed_indices + } + + total_proposer_rewards = sum( + balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT + for balance in pre_slashings.values() + ) # Process slashing spec.process_attester_slashing(state, attester_slashing) - slashed_validator = state.validators[slashed_index] + for slashed_index in slashed_indices: + pre_withdrawalable_epoch = pre_withdrawalable_epochs[slashed_index] + slashed_validator = state.validators[slashed_index] - # Check slashing - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # Check slashing + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + if pre_withdrawalable_epoch < spec.FAR_FUTURE_EPOCH: + expected_withdrawable_epoch = max( + pre_withdrawalable_epoch, + spec.get_current_epoch(state) + spec.EPOCHS_PER_SLASHINGS_VECTOR + ) + assert slashed_validator.withdrawable_epoch == expected_withdrawable_epoch + else: + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + assert get_balance(state, slashed_index) < pre_slashings[slashed_index] - if slashed_index != proposer_index: - # lost whistleblower reward - assert get_balance(state, slashed_index) < pre_slashed_balance + if proposer_index not in slashed_indices: # gained whistleblower reward - assert get_balance(state, proposer_index) > pre_proposer_balance + assert get_balance(state, proposer_index) == pre_proposer_balance + total_proposer_rewards else: # gained rewards for all slashings, which may include others. And only lost that of themselves. - # Netto at least 0, if more people where slashed, a balance increase. - assert get_balance(state, slashed_index) >= pre_slashed_balance + expected_balance = ( + pre_proposer_balance + + total_proposer_rewards + - pre_slashings[proposer_index] // spec.MIN_SLASHING_PENALTY_QUOTIENT + ) + + assert get_balance(state, proposer_index) == expected_balance yield 'post', state @@ -82,6 +107,37 @@ def test_success_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_recent(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_long_ago(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + state.validators[index].withdrawable_epoch = spec.get_current_epoch(state) + 2 + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + @with_all_phases @always_bls @spec_state_test From 2739767a7147f6b2efd6537189f3f3437b885058 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 14:43:44 +0100 Subject: [PATCH 074/110] Hardened Eth 1.0 voting strategy (#1218) --- specs/validator/0_beacon-chain-validator.md | 31 +++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66..44b5e2a3c 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -221,19 +221,26 @@ epoch_signature = bls_sign( ##### Eth1 Data -`block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. +The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. -* Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.eth1_data.block_hash`. - * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. - * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. -* If `D` is empty: - * Let `block_hash` be the block hash of the `ETH1_FOLLOW_DISTANCE`'th ancestor of the head of the canonical Eth 1.0 chain. - * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` - * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. -* If `D` is nonempty: - * Let `best_vote_data` be the `eth1_data` member of `D` that has the highest vote count (`D.count(eth1_data)`), breaking ties by favoring block hashes with higher associated block height. -* Set `block.eth1_data = best_vote_data`. +Let `get_eth1_data(distance: int) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: + +```python +def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: + new_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, 2 * ETH1_FOLLOW_DISTANCE)] + all_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, previous_eth1_distance)] + + valid_votes = [] + for slot, vote in enumerate(state.eth1_data_votes): + period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_square_root(SLOTS_PER_ETH1_VOTING_PERIOD) + if vote in new_eth1_data or (period_tail and vote in all_eth1_data): + valid_votes.append(vote) + + return max(valid_votes, + key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)), # Tiebreak by smallest distance + default=get_eth1_data(ETH1_FOLLOW_DISTANCE), + ) +``` ##### Signature From 05842f83718644184ce997277e7817bada07d568 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 15:26:02 +0100 Subject: [PATCH 075/110] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 96b8981fe..b9c3f5f0c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,14 +1294,15 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift + state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5ff13dd81a95d7852c2afa65dd3851d5642c17dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:07:36 +0200 Subject: [PATCH 076/110] be explicit about limiting for HTR and chunk padding --- specs/simple-serialize.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 97b1d560c..3fba59fba 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -49,13 +49,13 @@ class ContainerExample(Container): foo: uint64 bar: boolean ``` -* **vector**: ordered fixed-length homogeneous collection of values +* **vector**: ordered fixed-length homogeneous collection, with `N` values * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` -* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` +* **list**: ordered variable-length homogeneous collection, limited to `N` values * notation `List[type, N]`, e.g. `List[uint64, N]` -* **bitvector**: ordered fixed-length collection of `boolean` values +* **bitvector**: ordered fixed-length collection of `boolean` values, with `N` bits * notation `Bitvector[N]` -* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` +* **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` @@ -161,22 +161,31 @@ return serialized_type_index + serialized_bytes Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). +Note that deserialization requires hardening against invalid inputs. A non-exhaustive list: +- Offsets: out of order, out of range, mismatching minimum element size +- Scope: Extra unused bytes, not aligned with element size. +- More elements than a list limit allows. Part of enforcing consensus. + ## Merkleization We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. -* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) +* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + There merkleization depends on the input length: + - `0` chunks: A chunk filled with zeroes. + - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - `1+` chunks: pad to the next power of 2, merkleize as binary tree. + - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From dcb0244a4f99f50dfa82884a81bd14e1eabb316e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 10:19:59 -0500 Subject: [PATCH 077/110] get_attesting_indices set instead of sorted (#1225) --- specs/core/0_beacon-chain.md | 14 +++--- specs/core/1_custody-game.md | 94 ++++++++++++------------------------ 2 files changed, 39 insertions(+), 69 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 61ef8b742..abca72ff8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -866,13 +866,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Sequence[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Set[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the set of attesting indices corresponding to ``data`` and ``bitfield``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return set(index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1) ``` ### `int_to_bytes` @@ -950,12 +950,12 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) - assert set(custody_bit_1_indices).issubset(attesting_indices) - custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] + assert custody_bit_1_indices.issubset(attesting_indices) + custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) return IndexedAttestation( - custody_bit_0_indices=custody_bit_0_indices, - custody_bit_1_indices=custody_bit_1_indices, + custody_bit_0_indices=sorted(custody_bit_0_indices), + custody_bit_1_indices=sorted(custody_bit_1_indices), data=attestation.data, signature=attestation.signature, ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index c29033fe3..9b17b39ed 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -36,7 +36,7 @@ - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) + - [`get_reveal_period`](#get_reveal_period) - [`replace_empty_or_append`](#replace_empty_or_append) - [Per-block processing](#per-block-processing) - [Operations](#operations) @@ -224,7 +224,7 @@ Add the following fields to the end of the specified container objects. Fields w class Validator(Container): # next_custody_reveal_period is initialised to the custody period # (of the particular validator) in which the validator is activated - # = get_validators_custody_reveal_period(...) + # = get_reveal_period(...) next_custody_reveal_period: uint64 max_reveal_lateness: uint64 ``` @@ -299,17 +299,12 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI return Epoch(next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING) ``` -### `get_validators_custody_reveal_period` +### `get_reveal_period` ```python -def get_validators_custody_reveal_period(state: BeaconState, - validator_index: ValidatorIndex, - epoch: Epoch=None) -> int: +def get_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: ''' - This function returns the reveal period for a given validator. - If no epoch is supplied, the current epoch is assumed. - Note: This function implicitly requires that validators are not removed from the - validator set in fewer than EPOCHS_PER_CUSTODY_PERIOD epochs + Return the reveal period for a given validator. ''' epoch = get_current_epoch(state) if epoch is None else epoch return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD @@ -340,17 +335,15 @@ Verify that `len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS`. For each `reveal` in `block.body.custody_key_reveals`, run the following function: ```python -def process_custody_key_reveal(state: BeaconState, - reveal: CustodyKeyReveal) -> None: +def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> None: """ Process ``CustodyKeyReveal`` operation. Note that this function mutates ``state``. """ - revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_reveal_period, reveal.revealed_index) - assert revealer.next_custody_reveal_period < get_validators_custody_reveal_period(state, reveal.revealed_index) + assert revealer.next_custody_reveal_period < get_reveal_period(state, reveal.revealed_index) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -368,11 +361,11 @@ def process_custody_key_reveal(state: BeaconState, ) # Decrement max reveal lateness if response is timely - if revealer.next_custody_reveal_period == get_validators_custody_reveal_period(state, reveal.revealer_index) - 2: + if revealer.next_custody_reveal_period == get_reveal_period(state, reveal.revealer_index) - 2: revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT revealer.max_reveal_lateness = max( revealer.max_reveal_lateness, - get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period + get_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period ) # Process reveal @@ -394,13 +387,11 @@ Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_S For each `reveal` in `block.body.early_derived_secret_reveals`, run the following function: ```python -def process_early_derived_secret_reveal(state: BeaconState, - reveal: EarlyDerivedSecretReveal) -> None: +def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerivedSecretReveal) -> None: """ Process ``EarlyDerivedSecretReveal`` operation. Note that this function mutates ``state``. """ - revealed_validator = state.validators[reveal.revealed_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS @@ -470,8 +461,7 @@ Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALL For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: ```python -def process_chunk_challenge(state: BeaconState, - challenge: CustodyChunkChallenge) -> None: +def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge @@ -514,59 +504,41 @@ Verify that `len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGE For each `challenge` in `block.body.custody_bit_challenges`, run the following function: ```python -def process_bit_challenge(state: BeaconState, - challenge: CustodyBitChallenge) -> None: +def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: + attestation = challenge.attestation + epoch = slot_to_epoch(attestation.data.slot) + shard = attestation.data.crosslink.shard # Verify challenge signature challenger = state.validators[challenge.challenger_index] - assert bls_verify( - pubkey=challenger.pubkey, - message_hash=signing_root(challenge), - signature=challenge.signature, - domain=get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)), - ) + domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) + assert bls_verify(challenger.pubkey, signing_root(challenge), challenge.signature, domain) + # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) - - # Verify the attestation - attestation = challenge.attestation + # Verify attestation validate_indexed_attestation(state, convert_to_indexed(state, attestation)) - # Verify the attestation is eligible for challenging + # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] - assert (slot_to_epoch(attestation.data.slot) + responder.max_reveal_lateness <= - get_validators_custody_reveal_period(state, challenge.responder_index)) - - # Verify the responder participated in the attestation + assert epoch + responder.max_reveal_lateness <= get_reveal_period(state, challenge.responder_index) + # Verify responder participated in the attestation attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - - # A validator can be the challenger for at most one challenge at a time + # Verifier challenger is not already challenging for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - - # Verify the responder is a valid custody key + # Verify the responder custody key epoch_to_sign = get_randao_epoch_for_custody_period( - get_validators_custody_reveal_period( - state, - challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot)), - challenge.responder_index + get_reveal_period(state, challenge.responder_index, epoch), + challenge.responder_index, ) - assert bls_verify( - pubkey=responder.pubkey, - message_hash=hash_tree_root(epoch_to_sign), - signature=challenge.responder_key, - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=epoch_to_sign, - ), - ) - + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + committee = get_crosslink_committee(state, epoch, shard) + custody_bit = get_bitfield_bit(attestation.custody_bitfield, committee.index(challenge.responder_index)) assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( @@ -581,7 +553,6 @@ def process_bit_challenge(state: BeaconState, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) state.custody_challenge_index += 1 - # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` @@ -593,8 +564,7 @@ Verify that `len(block.body.custody_responses) <= MAX_CUSTODY_RESPONSES`. For each `response` in `block.body.custody_responses`, run the following function: ```python -def process_custody_response(state: BeaconState, - response: CustodyResponse) -> None: +def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: @@ -682,7 +652,7 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validators): deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - if get_validators_custody_reveal_period(state, ValidatorIndex(index)) > deadline: + if get_reveal_period(state, ValidatorIndex(index)) > deadline: slash_validator(state, ValidatorIndex(index)) ``` From 128bbbc665c8eabe10640c019ccb0765c5be8f11 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:27:59 +0200 Subject: [PATCH 078/110] fix slicing, and support partial slice bounds --- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b9c3f5f0c..887ae6802 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 91d3ff4f2..d87a22399 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -325,7 +325,7 @@ class BaseList(list, Elements): def __setitem__(self, k, v): if type(k) == slice: - if k.start < 0 or k.stop > len(self): + if (k.start is not None and k.start < 0) or (k.stop is not None and k.stop > len(self)): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds slice {k} (to {v}, bound: {len(self)})") super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) From 25db39755047ce6ae28332d90080c502ce899423 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:34:31 +0200 Subject: [PATCH 079/110] fix line length lint problem in checkpoint --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 887ae6802..ba5b591d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1298,11 +1298,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5a8f3e495a8be99d6f6ae821c141e1b881d44d92 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:10:17 -0600 Subject: [PATCH 080/110] set committees root for next epoch rather tahn ACTIVaTION_EXIT_DELAY in the future --- specs/core/0_beacon-chain.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 842b16b98..8f55addbc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -698,6 +698,9 @@ def get_shard_delta(state: BeaconState, epoch: Epoch) -> int: ```python def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: + """ + Return the start shard of the 0th committee in an epoch. + """ assert epoch <= get_current_epoch(state) + 1 check_epoch = Epoch(get_current_epoch(state) + 1) shard = Shard((state.start_shard + get_shard_delta(state, get_current_epoch(state))) % SHARD_COUNT) @@ -767,7 +770,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: committees[shard].pubkeys.append(validator.pubkey) compact_balance = validator.effective_balance // EFFECTIVE_BALANCE_INCREMENT # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance` (bottom 15 bits) - compact_validator = uint64(index << 16 + validator.slashed << 15 + compact_balance) + compact_validator = uint64((index << 16) + (validator.slashed << 15) + compact_balance) committees[shard].compact_validators.append(compact_validator) return hash_tree_root(Vector[CompactCommittee, SHARD_COUNT](committees)) ``` @@ -782,7 +785,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - state.compact_committees_roots[epoch] + + state.compact_committees_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + int_to_bytes(epoch, length=32) ) ``` @@ -1546,7 +1549,7 @@ def process_final_updates(state: BeaconState) -> None: state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR - state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch + ACTIVATION_EXIT_DELAY) + state.compact_committees_roots[index_root_position] = get_compact_committees_root(state, next_epoch) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] From b40e2284a0f601dd6f6ac80f833d4036c244ecfb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:20:24 -0600 Subject: [PATCH 081/110] use active index root for generate seed mix in --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f55addbc..80977c8f7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -785,7 +785,7 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) + - state.compact_committees_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] + + hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, epoch))) + int_to_bytes(epoch, length=32) ) ``` From 9993a2879629b7e27f97086ff3d5d70cc3da6a2f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 11:26:05 -0600 Subject: [PATCH 082/110] lint --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 80977c8f7..490ac0c96 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -764,7 +764,7 @@ def get_compact_committees_root(state: BeaconState, epoch: Epoch) -> Hash: committees = [CompactCommittee() for _ in range(SHARD_COUNT)] start_shard = get_epoch_start_shard(state, epoch) for committee_number in range(get_epoch_committee_count(state, epoch)): - shard = (start_shard + committee_number) % SHARD_COUNT + shard = Shard((start_shard + committee_number) % SHARD_COUNT) for index in get_crosslink_committee(state, epoch, shard): validator = state.validators[index] committees[shard].pubkeys.append(validator.pubkey) @@ -1535,7 +1535,7 @@ def process_slashings(state: BeaconState) -> None: ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 + next_epoch = Shard(current_epoch + 1) # Reset eth1 data votes if (state.slot + 1) % SLOTS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] From fa84c496592d722b544c775065b9258cd22dbd73 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:23:34 +0100 Subject: [PATCH 083/110] Update specs/core/0_beacon-chain.md Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cb50e2f86..6f28c0e3d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1295,7 +1295,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:-1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: From 6a2d2c84a8f3fa3ea846d17dff5d10ebfb3508e6 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 20:49:57 +0100 Subject: [PATCH 084/110] Bitlist for attestation doc --- specs/validator/0_beacon-chain-validator.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ad01476e9..7d5630c7b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -331,17 +331,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bits = aggregation_bits`. +* Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bits = custody_bits`. +* Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0. From 4dcb47e393ca68ab9e32af8ba913eaf829d6cb69 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:06 +0100 Subject: [PATCH 085/110] Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 916724001..b114c85e9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -279,7 +279,7 @@ def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits = attestation.aggregation_bits[:] custody_bits.append(False) attestation.custody_bits = custody_bits From be04eb2673563b6250cc5637777badc76913ec5f Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:16 +0100 Subject: [PATCH 086/110] Change copy style, and remove deepcopy import Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index b114c85e9..41207fdf4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, @@ -295,7 +293,7 @@ def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bits = deepcopy(attestation.aggregation_bits) + attestation.custody_bits = attestation.aggregation_bits[:] sign_attestation(spec, state, attestation) From 4f31207b7f2cd4ba2bf21976b6e8bc6644e24202 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 22:45:20 +0200 Subject: [PATCH 087/110] reword merkleize with limit / length --- specs/simple-serialize.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3fba59fba..7076b6410 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -171,21 +171,21 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. - There merkleization depends on the input length: - - `0` chunks: A chunk filled with zeroes. - - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. - - `1+` chunks: pad to the next power of 2, merkleize as binary tree. - - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). +* `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` +* `merkleize(data, pad_for)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + The merkleization depends on the effective input, which can be padded: if `pad_for=L`, then pad the `data` with zeroed chunks to `next_pow_of_two(L)` (virtually for memory efficiency). + Then, merkleize the chunks (empty input is padded to 1 zero chunk): + - If `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - If `> 1` chunks: pad to `next_pow_of_two(len(chunks))`, merkleize as binary tree. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). +* `mix_in_length(merkleize(pack(value), pad_for=(N * elem_size / BYTES_PER_CHUNK)), len(value))` if `value` is a list of basic objects. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_for=N), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From d5c2ecb6f73a3c2ea673332d45739704b6af9b29 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 15:44:26 -0600 Subject: [PATCH 088/110] remove local notes files --- motes.md | 42 ------------------------------------------ notes.txt | 1 - 2 files changed, 43 deletions(-) delete mode 100644 motes.md delete mode 100644 notes.txt diff --git a/motes.md b/motes.md deleted file mode 100644 index e01ba19b7..000000000 --- a/motes.md +++ /dev/null @@ -1,42 +0,0 @@ -* `BLS_WITHDRAWAL_PREFIX` - * Why int rather than bytes? -* `MIN_SEED_LOOKAHEAD` - * Is this actually tunable? - * If so, what are the reprecussions? -* `ACTIVATION_EXIT_DELAY` - * Reaquaint with purpose -* AttesterSlashings - * `MAX_ATTESTER_SLASHINGS` is 1. - * Are there scenarios in which validators can create more effective slashable - messages than can be included on chain? For example, Validators split up to - create double attestations for checkpoints but different (junk) crosslink - data to prevent them from being aggregatable to the fullest - * Max is for block size, no? -* Signature domains - * Max 4byte ints -* `Version` not defined in one of the lists of custom types (2!!). ensure in spec -* `PendingAttestation` - * Don't think `proposer_index` is actually necessary here because effective - balance is stable until end of epoch so can do dynamic lookups -* is_genesis_trigger - * only run at ends of blocks to preserve invariant that eth1data.deposit_root - is the deposit root at the _end_ of an eth1 block -* `Attestation` - * why bitfields not together? -* `Transfer` - * replay mechanism... say the slot gets missed and you sign another transfer - * in a fork you could include both transfers -* `get_previous_epoch` - * do a once over on the genesis stuff -* `get_epoch_start_shard` - * checking next hinges upon the fact that the validator set for the next - epoch is 100% known at the current epoch. Ensure this is the case -* `get_block_root_at_slot` .. `generate_seed` can be bade into one line - function signatures -* `get_shuffled_index` - * I think it should be maybe `assert index_count <= VALIDATOR_REGISTRY_LIMIT` - * is the `2**40` special for security of alg? probably. - - -pubkey/privkey g1 vs g2 - diff --git a/notes.txt b/notes.txt deleted file mode 100644 index 421ad7750..000000000 --- a/notes.txt +++ /dev/null @@ -1 +0,0 @@ -* `BLS_WITHDRAWAL_PREFIX` -- From 3a60f64b9275f0dea931bf240eba2d1f083a22e8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 01:22:29 +0200 Subject: [PATCH 089/110] refactor finalization test helper func --- ..._process_justification_and_finalization.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 95f00edd4..f0fe35e8b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -1,3 +1,4 @@ +import math from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with @@ -8,41 +9,46 @@ def run_process_just_and_fin(spec, state): yield from run_epoch_processing_with(spec, state, 'process_justification_and_finalization') -def get_committee_size(spec, state, slot): +def get_shards_for_slot(spec, state, slot): epoch = spec.slot_to_epoch(slot) epoch_start_shard = spec.get_epoch_start_shard(state, epoch) committees_per_slot = spec.get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT - committee_index = (shard + spec.SHARD_COUNT - spec.get_epoch_start_shard(state, epoch)) % spec.SHARD_COUNT - committee_count = spec.get_epoch_committee_count(state, epoch) - indices = spec.get_active_validator_indices(state, epoch) + return [shard + i for i in range(committees_per_slot)] + + +def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices): + committee_index = (shard + spec.SHARD_COUNT - epoch_start_shard) % spec.SHARD_COUNT start = (len(indices) * committee_index) // committee_count end = (len(indices) * (committee_index + 1)) // committee_count size = end - start - return size + return size, -def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): +def add_mock_attestations(spec, state, epoch, att_count, att_ratio): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 previous_epoch = spec.get_previous_epoch(state) current_epoch = spec.get_current_epoch(state) - if current_epoch == target_epoch: + if current_epoch == epoch: attestations = state.current_epoch_attestations - elif previous_epoch == target_epoch: + elif previous_epoch == epoch: attestations = state.previous_epoch_attestations else: - raise Exception(f"cannot target epoch ${target_epoch} from epoch ${current_epoch}") + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") + committee_count = spec.get_epoch_committee_count(state, epoch) + indices = spec.get_active_validator_indices(state, epoch) + epoch_start_shard = spec.get_epoch_start_shard(state, epoch) total = 0 - while total < att_count: - for i in range(spec.SLOTS_PER_EPOCH): - size = get_committee_size(spec, state, state.slot + i) + for i in range(spec.SLOTS_PER_EPOCH): + for shard in get_shards_for_slot(spec, state, state.slot + i): + size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = int(size * att_ratio) + attesting_count = math.ceil(size * att_ratio) aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') attestations.append(spec.PendingAttestation( @@ -54,16 +60,20 @@ def add_mock_attestations(spec, state, target_epoch, att_count, att_ratio): target_root=b'\xbb' * 32, crosslink=spec.Crosslink() ), - inclusion_delay=0, + inclusion_delay=1, )) total += 1 + if total >= att_count: + return + + raise Exception(f"could not fill state with {att_count} attestations for epoch {epoch}") @with_all_phases @spec_state_test def test_rule_1(spec, state): - previous_epoch = spec.get_previous_epoch(state) - current_epoch = spec.get_current_epoch(state) + # previous_epoch = spec.get_previous_epoch(state) + # current_epoch = spec.get_current_epoch(state) # TODO # add_mock_attestations(spec, state, ...) @@ -71,4 +81,4 @@ def test_rule_1(spec, state): # set their balances # yield from run_process_just_and_fin(spec, state) # check finalization - + pass From 518db42de75b9714347cd4ad2d549691f28afe6b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:19:30 +0200 Subject: [PATCH 090/110] fix attestation tests to work with checkpoints --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index a1f87f059..f083a07f4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -205,7 +205,7 @@ def test_future_target_epoch(spec, state): state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.target_epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle + attestation.data.target.epoch = spec.get_current_epoch(state) + 1 # target epoch will be too new to handle sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation, False) From efd9d173d786c87ad4daf4710615cd739ee2d584 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:19:57 +0200 Subject: [PATCH 091/110] update attester slashings processing tests --- .../test_process_slashings.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py index f1a23326b..7be23a04d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -17,8 +17,8 @@ def slash_validators(spec, state, indices, out_epochs): v.withdrawable_epoch = out_epoch total_slashed_balance += v.effective_balance - state.slashed_balances[ - spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR + state.slashings[ + spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHINGS_VECTOR ] = total_slashed_balance @@ -26,13 +26,13 @@ def slash_validators(spec, state, indices, out_epochs): @spec_state_test def test_max_penalties(spec, state): slashed_count = (len(state.validators) // 3) + 1 - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slashed_indices = list(range(slashed_count)) slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) total_balance = spec.get_total_active_balance(state) - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties = sum(state.slashings) assert total_balance // 3 <= total_penalties @@ -44,28 +44,30 @@ def test_max_penalties(spec, state): @with_all_phases @spec_state_test -def test_min_penalties(spec, state): - # run_epoch_processing_to(spec, state, 'process_slashings', exclusive=True) - +def test_small_penalty(spec, state): # Just the bare minimum for this one validator - pre_balance = state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE + state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE # All the other validators get the maximum. for i in range(1, len(state.validators)): state.validators[i].effective_balance = state.balances[i] = spec.MAX_EFFECTIVE_BALANCE - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slash_validators(spec, state, [0], [out_epoch]) total_balance = spec.get_total_active_balance(state) - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_penalties = sum(state.slashings) - # we are testing the minimum here, i.e. get slashed (effective_balance / MIN_SLASHING_PENALTY_QUOTIENT) - assert total_penalties * 3 / total_balance < 1 / spec.MIN_SLASHING_PENALTY_QUOTIENT + assert total_balance // 3 > total_penalties - yield from run_process_slashings(spec, state) + run_epoch_processing_to(spec, state, 'process_slashings') + pre_slash_balances = list(state.balances) + yield 'pre', state + spec.process_slashings(state) + yield 'post', state - assert state.balances[0] == pre_balance - (pre_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT) + assert state.balances[0] == pre_slash_balances[0] - (state.validators[0].effective_balance + * 3 * total_penalties // total_balance) @with_all_phases @@ -75,14 +77,13 @@ def test_scaled_penalties(spec, state): state.slot = spec.SLOTS_PER_EPOCH # Also mock some previous slashings, so that we test to have the delta in the penalties computation. - for i in range(spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR): - state.slashed_balances[i] = spec.MAX_EFFECTIVE_BALANCE * 3 - - # Mock the very last one (which is to be used for the delta balance computation) to be different. - # To enforce the client test runner to correctly get this one from the array, not the others. - prev_penalties = state.slashed_balances[ - (spec.get_current_epoch(state) + 1) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR - ] = spec.MAX_EFFECTIVE_BALANCE * 2 + base = spec.EJECTION_BALANCE + incr = spec.EFFECTIVE_BALANCE_INCREMENT + # Just add some random slashings. non-zero slashings are at least the minimal effective balance. + state.slashings[0] = base + (incr * 12) + state.slashings[4] = base + (incr * 3) + state.slashings[5] = base + (incr * 6) + state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) slashed_count = len(state.validators) // 4 @@ -90,13 +91,17 @@ def test_scaled_penalties(spec, state): # make the balances non-uniform. # Otherwise it would just be a simple 3/4 balance slashing. Test the per-validator scaled penalties. + diff = spec.MAX_EFFECTIVE_BALANCE - base + increments = diff // incr for i in range(10): - state.validators[i].effective_balance += spec.EFFECTIVE_BALANCE_INCREMENT * 4 - state.balances[i] += spec.EFFECTIVE_BALANCE_INCREMENT * 4 + state.validators[i].effective_balance = base + (incr * (i % increments)) + assert state.validators[i].effective_balance <= spec.MAX_EFFECTIVE_BALANCE + # add/remove some, see if balances different than the effective balances are picked up + state.balances[i] = state.validators[i].effective_balance + i - 5 total_balance = spec.get_total_active_balance(state) - out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2) + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) slashed_indices = list(range(slashed_count)) @@ -112,8 +117,7 @@ def test_scaled_penalties(spec, state): spec.process_slashings(state) yield 'post', state - total_penalties = state.slashed_balances[spec.get_current_epoch(state) % spec.EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_penalties -= prev_penalties + total_penalties = sum(state.slashings) for i in slashed_indices: v = state.validators[i] From afb34c71e6aeaa3b088b46b685f03a9dfaba8a9b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 03:22:14 +0200 Subject: [PATCH 092/110] fix broken block test for checkpoint use --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index e56baee8c..019563978 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -143,7 +143,7 @@ def test_empty_epoch_transition_not_finalizing(spec, state): yield 'post', state assert state.slot == block.slot - assert state.finalized_epoch < spec.get_current_epoch(state) - 4 + assert state.finalized_checkpoint.epoch < spec.get_current_epoch(state) - 4 for index in range(len(state.validators)): assert state.balances[index] < pre_balances[index] From cfbdee709bd408572d10c70f2a7adb3dd72220aa Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 05:04:25 +0200 Subject: [PATCH 093/110] finalization testing --- ..._process_justification_and_finalization.py | 226 ++++++++++++++++-- 1 file changed, 204 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index f0fe35e8b..0afa4f307 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -22,10 +22,10 @@ def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) start = (len(indices) * committee_index) // committee_count end = (len(indices) * (committee_index + 1)) // committee_count size = end - start - return size, + return size -def add_mock_attestations(spec, state, epoch, att_count, att_ratio): +def add_mock_attestations(spec, state, epoch, att_ratio, source, target): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -42,9 +42,9 @@ def add_mock_attestations(spec, state, epoch, att_count, att_ratio): committee_count = spec.get_epoch_committee_count(state, epoch) indices = spec.get_active_validator_indices(state, epoch) epoch_start_shard = spec.get_epoch_start_shard(state, epoch) - total = 0 - for i in range(spec.SLOTS_PER_EPOCH): - for shard in get_shards_for_slot(spec, state, state.slot + i): + epoch_start_slot = spec.get_epoch_start_slot(epoch) + for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): + for shard in get_shards_for_slot(spec, state, slot): size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -55,30 +55,212 @@ def add_mock_attestations(spec, state, epoch, att_count, att_ratio): aggregation_bitfield=aggregation_bitfield, data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, - source_epoch=0, - source_root=b'\xbb' * 32, - target_root=b'\xbb' * 32, + source=source, + target=source, crosslink=spec.Crosslink() ), inclusion_delay=1, )) - total += 1 - if total >= att_count: - return - raise Exception(f"could not fill state with {att_count} attestations for epoch {epoch}") + +def finalize_on_234(spec, state, epoch, support): + assert epoch > 4 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c4 + state.current_justified_checkpoint = c3 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[3:4] = [1, 1] # mock 3rd and 4th latest epochs as justified + # mock the 2nd latest epoch as justifiable, with 4th as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + att_ratio=support, + source=c4, + target=c2) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c2 # changed to 2nd latest + assert state.finalized_checkpoint == c4 # finalized old previous justified epoch + else: + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c3 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_23(spec, state, epoch, support): + assert epoch > 3 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c3 + state.current_justified_checkpoint = c3 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[2] = 1 # mock 3rd latest epoch as justified + # mock the 2nd latest epoch as justifiable, with 3rd as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + att_ratio=support, + source=c3, + target=c2) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c2 # changed to 2nd latest + assert state.finalized_checkpoint == c3 # finalized old previous justified epoch + else: + assert state.previous_justified_checkpoint == c3 # changed to old current + assert state.current_justified_checkpoint == c3 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_123(spec, state, epoch, support): + assert epoch > 3 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210x -- justification bitfield indices + # 011*. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c3 + state.current_justified_checkpoint = c2 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[1:2] = [1, 1] # mock 2rd and 3th latest epochs as justified + # mock the 1st latest epoch as justifiable, with 3rd as source + add_mock_attestations(spec, state, + epoch=epoch - 1, + att_ratio=support, + source=c3, + target=c1) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c1 # changed to 1st latest + assert state.finalized_checkpoint == c2 # finalized old current + else: + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c2 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized + + +def finalize_on_12(spec, state, epoch, support): + assert epoch > 2 + state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch + + # 43210 -- epochs ago + # 3210 -- justification bitfield indices + # 001*. -- justification bitfield contents, . = this epoch, * is being justified now + # checkpoints for the epochs ago: + # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) + # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + + old_finalized = state.finalized_checkpoint + state.previous_justified_checkpoint = c2 + state.current_justified_checkpoint = c2 + bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + bits[3] = 1 # mock 3rd latest epoch as justified + # mock the 1st latest epoch as justifiable, with 2nd as source + add_mock_attestations(spec, state, + epoch=epoch - 1, + att_ratio=support, + source=c2, + target=c1) + + # process! + yield from run_process_just_and_fin(spec, state) + + if support >= (2 / 3): + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c1 # changed to 1st latest + assert state.finalized_checkpoint == c2 # finalized previous justified epoch + else: + assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.current_justified_checkpoint == c2 # still old current + assert state.finalized_checkpoint == old_finalized # no new finalized @with_all_phases @spec_state_test -def test_rule_1(spec, state): - # previous_epoch = spec.get_previous_epoch(state) - # current_epoch = spec.get_current_epoch(state) +def test_234_ok_support(spec, state): + yield from finalize_on_234(spec, state, 5, 1.0) - # TODO - # add_mock_attestations(spec, state, ...) - # get indices attesting e.g. current_epoch_attestations - # set their balances - # yield from run_process_just_and_fin(spec, state) - # check finalization - pass + +@with_all_phases +@spec_state_test +def test_234_poor_support(spec, state): + yield from finalize_on_234(spec, state, 5, 0.6) + + +@with_all_phases +@spec_state_test +def test_23_ok_support(spec, state): + yield from finalize_on_23(spec, state, 4, 1.0) + + +@with_all_phases +@spec_state_test +def test_23_poor_support(spec, state): + yield from finalize_on_23(spec, state, 4, 0.6) + + +@with_all_phases +@spec_state_test +def test_123_ok_support(spec, state): + yield from finalize_on_123(spec, state, 4, 1.0) + + +@with_all_phases +@spec_state_test +def test_123_poor_support(spec, state): + yield from finalize_on_123(spec, state, 4, 0.6) + + +@with_all_phases +@spec_state_test +def test_12_ok_support(spec, state): + yield from finalize_on_12(spec, state, 3, 1.0) + + +@with_all_phases +@spec_state_test +def test_12_poor_support(spec, state): + yield from finalize_on_12(spec, state, 3, 0.6) + + +# TODO: bring ratios closer to 2/3 for edge case testing. From f484b1e98c44564c33672d49da23a42ed38ccc22 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:18:22 -0600 Subject: [PATCH 094/110] some fixes to finality_12 --- .../test_process_justification_and_finalization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 0afa4f307..b1d94c4fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -189,12 +189,14 @@ def finalize_on_12(spec, state, epoch, support): # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0] = 1 # mock latest epoch as justified # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, From 1885e285e3b57a51450910e77e19cf6d505b34c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:28:36 -0600 Subject: [PATCH 095/110] fix source/target epochs in test_12 --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index b1d94c4fb..afb25fdae 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From bc5e947efc01141909aaf952167d6b20e6eb9d63 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:38:14 -0600 Subject: [PATCH 096/110] aggregation_bitfield - bits --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index afb25fdae..879c982f0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -49,10 +49,10 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. attesting_count = math.ceil(size * att_ratio) - aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') + aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, From 022f1e7108a2d6971c147b22a3fef2b0aacebbc9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:59:20 -0600 Subject: [PATCH 097/110] fix source/target --- .../test_process_justification_and_finalization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 879c982f0..246f20c9e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -56,7 +56,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, - target=source, + target=target, crosslink=spec.Crosslink() ), inclusion_delay=1, @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From 129fd6297e1218f0655dda2bf8eb2b49a75bd387 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 00:03:06 -0600 Subject: [PATCH 098/110] add shard to mock crosslink to separate attestations from eachother --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 246f20c9e..adcf6ea1d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -57,7 +57,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): beacon_block_root=b'\xaa' * 32, source=source, target=target, - crosslink=spec.Crosslink() + crosslink=spec.Crosslink(shard=shard) ), inclusion_delay=1, )) From b05bebf45c1bfe31fdf9ee195283827be4f145e0 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 12:23:03 +0200 Subject: [PATCH 099/110] Fix list slicing --- ..._process_justification_and_finalization.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index adcf6ea1d..3e2f6e967 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -74,13 +74,15 @@ def finalize_on_234(spec, state, epoch, support): c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3:4] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -109,16 +111,16 @@ def finalize_on_23(spec, state, epoch, support): # 3210x -- justification bitfield indices # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[2] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1] = 1 # mock 2nd latest epoch as justified # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -147,16 +149,18 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[1:2] = [1, 1] # mock 2rd and 3th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -185,8 +189,6 @@ def finalize_on_12(spec, state, epoch, support): # 3210 -- justification bitfield indices # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root From f9ca7c97c9bc4d994ba88ee721afcfc04336673b Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:01:59 +0200 Subject: [PATCH 100/110] Fix 123 finalisation --- .../test_process_justification_and_finalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 3e2f6e967..42ef73084 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -149,6 +149,7 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) @@ -157,7 +158,7 @@ def finalize_on_123(spec, state, epoch, support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c3 + state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified From 2eca6ef09d6d94656863907ba2fed5213788f213 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:10:09 +0200 Subject: [PATCH 101/110] Corrects justification comments --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 42ef73084..02439f664 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -82,7 +82,7 @@ def finalize_on_234(spec, state, epoch, support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, From 0680d8cc536f72a863d9e9ec1975a2fd5ff1729c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 14:35:07 +0200 Subject: [PATCH 102/110] Makes justification ratios more marginal --- ..._process_justification_and_finalization.py | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 02439f664..406bdbf7f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -25,7 +25,7 @@ def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) return size -def add_mock_attestations(spec, state, epoch, att_ratio, source, target): +def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -48,7 +48,8 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * att_ratio) + attesting_count = math.ceil(size * 2 /3) + attesting_count = attesting_count if sufficient_support else attesting_count - 1 aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( @@ -63,7 +64,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): )) -def finalize_on_234(spec, state, epoch, support): +def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -86,14 +87,14 @@ def finalize_on_234(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c4, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch @@ -103,7 +104,7 @@ def finalize_on_234(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_23(spec, state, epoch, support): +def finalize_on_23(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -124,14 +125,14 @@ def finalize_on_23(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c3, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch @@ -141,7 +142,7 @@ def finalize_on_23(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_123(spec, state, epoch, support): +def finalize_on_123(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -165,14 +166,14 @@ def finalize_on_123(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c3, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current @@ -182,7 +183,7 @@ def finalize_on_123(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_12(spec, state, epoch, support): +def finalize_on_12(spec, state, epoch, sufficient_support): assert epoch > 2 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -203,14 +204,14 @@ def finalize_on_12(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c2, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch @@ -223,49 +224,46 @@ def finalize_on_12(spec, state, epoch, support): @with_all_phases @spec_state_test def test_234_ok_support(spec, state): - yield from finalize_on_234(spec, state, 5, 1.0) + yield from finalize_on_234(spec, state, 5, True) @with_all_phases @spec_state_test def test_234_poor_support(spec, state): - yield from finalize_on_234(spec, state, 5, 0.6) + yield from finalize_on_234(spec, state, 5, False) @with_all_phases @spec_state_test def test_23_ok_support(spec, state): - yield from finalize_on_23(spec, state, 4, 1.0) + yield from finalize_on_23(spec, state, 4, True) @with_all_phases @spec_state_test def test_23_poor_support(spec, state): - yield from finalize_on_23(spec, state, 4, 0.6) + yield from finalize_on_23(spec, state, 4, False) @with_all_phases @spec_state_test def test_123_ok_support(spec, state): - yield from finalize_on_123(spec, state, 4, 1.0) + yield from finalize_on_123(spec, state, 4, True) @with_all_phases @spec_state_test def test_123_poor_support(spec, state): - yield from finalize_on_123(spec, state, 4, 0.6) + yield from finalize_on_123(spec, state, 4, False) @with_all_phases @spec_state_test def test_12_ok_support(spec, state): - yield from finalize_on_12(spec, state, 3, 1.0) + yield from finalize_on_12(spec, state, 3, True) @with_all_phases @spec_state_test def test_12_poor_support(spec, state): - yield from finalize_on_12(spec, state, 3, 0.6) - - -# TODO: bring ratios closer to 2/3 for edge case testing. + yield from finalize_on_12(spec, state, 3, False) From 4ed7af7bacb2602d97ea67596b8b21d4b019b150 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 15:48:39 +0200 Subject: [PATCH 103/110] mock attestation refactor --- ..._process_justification_and_finalization.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 406bdbf7f..66e2095b4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,3 @@ -import math from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with @@ -39,29 +38,37 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") - committee_count = spec.get_epoch_committee_count(state, epoch) - indices = spec.get_active_validator_indices(state, epoch) - epoch_start_shard = spec.get_epoch_start_shard(state, epoch) + total_balance = spec.get_total_active_balance(state) + remaining_balance = total_balance * 2 // 3 + epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): - size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * 2 /3) - attesting_count = attesting_count if sufficient_support else attesting_count - 1 - aggregation_bits = [i < attesting_count for i in range(size)] - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + aggregation_bits = [0] * len(committee) + for v in range(len(committee) * 2 // 3 + 1): + if remaining_balance > 0: + remaining_balance -= state.validators[v].effective_balance + aggregation_bits[v] = 1 + elif not sufficient_support: + aggregation_bits[v - 1] = 0 + break + else: + break + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): From 0c29c5125f0e890e5bdd7d9e14afd166d0b6b260 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 17:10:12 +0200 Subject: [PATCH 104/110] Finnish refactor --- .../test_process_justification_and_finalization.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 66e2095b4..50a0a0cab 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -16,14 +16,6 @@ def get_shards_for_slot(spec, state, slot): return [shard + i for i in range(committees_per_slot)] -def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices): - committee_index = (shard + spec.SHARD_COUNT - epoch_start_shard) % spec.SHARD_COUNT - start = (len(indices) * committee_index) // committee_count - end = (len(indices) * (committee_index + 1)) // committee_count - size = end - start - return size - - def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 From 5d633bfdf36c5b4a9e069e302cecd70bbd782b13 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 18:14:17 +0200 Subject: [PATCH 105/110] bugfix attestation creation so that it works on mainnet with multiple committees per slot, and improve bitfield index descriptions --- ..._process_justification_and_finalization.py | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 50a0a0cab..d043d8d97 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -36,6 +36,11 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): + # Check if we already have had sufficient balance. (and undone if we don't want it). + # If so, do not create more attestations. (we do not have empty pending attestations normally anyway) + if remaining_balance < 0: + return + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -45,22 +50,23 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support if remaining_balance > 0: remaining_balance -= state.validators[v].effective_balance aggregation_bits[v] = 1 - elif not sufficient_support: - aggregation_bits[v - 1] = 0 - break else: break - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + # remove just one attester to make the marginal support insufficient + if not sufficient_support: + aggregation_bits[aggregation_bits.index(1)] = 0 + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): @@ -82,7 +88,7 @@ def finalize_on_234(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -93,12 +99,11 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -108,7 +113,8 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) @@ -120,7 +126,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1] = 1 # mock 2nd latest epoch as justified + state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -131,12 +137,11 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -146,7 +151,8 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) @@ -158,10 +164,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c4 + state.previous_justified_checkpoint = c4 # not c3, otherwise finalize 23 would trigger. state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified + state.justification_bits[0:2] = [1, 1] # mock 2nd and 3rd latest epochs as justified (indices are pre-shift) # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -172,12 +178,11 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -187,7 +192,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210 -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) @@ -199,7 +205,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0] = 1 # mock latest epoch as justified + state.justification_bits[0] = 1 # mock 2nd latest epoch as justified (this is pre-shift) # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -210,12 +216,11 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized From bc8df3cba3ccef7ce28b202720d6f698971e5126 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 12:04:56 -0500 Subject: [PATCH 106/110] minor typo Co-Authored-By: Alex Stokes --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a29ed05c8..cb7b6ce10 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -40,7 +40,7 @@ Note that there is now a new way to compute `get_active_validator_indices`: ```python def get_active_validator_indices(state: ExtendedBeaconState, epoch: Epoch) -> List[ValidatorIndex]: - return state.active_indices[epoch % compact_committees_rootS_LENGTH] + return state.active_indices[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` Note that it takes `state` instead of `state.validators` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. From 6b81e747b25a0b478099c594abd640317d6f6767 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 29 Jun 2019 18:50:23 +0100 Subject: [PATCH 107/110] Fix comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 619bd3c01..b5e36bcf0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1167,7 +1167,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate compact_committees_roots genesis_committee_root = get_compact_committees_root(state, GENESIS_EPOCH) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.compact_committees_roots[index] = genesis_committee_root From 302b3afe2ae2dcb0287096dec8b8121dca5d248f Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 21:37:54 +0200 Subject: [PATCH 108/110] rename/fix roots in justification tests for consistency --- .../test_process_justification_and_finalization.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index d043d8d97..c76692a95 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -77,8 +77,8 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root @@ -117,7 +117,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root @@ -155,10 +155,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root @@ -197,7 +197,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From afb33ddc5bff4a8d2ad4b456fd33587838054ba4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 21:39:10 +0200 Subject: [PATCH 109/110] fix typo in justification wording --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 019563978..b2eb19244 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -127,7 +127,7 @@ def test_empty_epoch_transition(spec, state): @spec_state_test def test_empty_epoch_transition_not_finalizing(spec, state): # Don't run for non-minimal configs, it takes very long, and the effect - # of calling finalization/justifcation is just the same as with the minimal configuration. + # of calling finalization/justification is just the same as with the minimal configuration. if spec.SLOTS_PER_EPOCH > 8: return From 36dd977b85ef1c21e35f691cd071b30900709759 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 22:27:05 +0200 Subject: [PATCH 110/110] fix finalize on double justification in 123 rule --- ..._process_justification_and_finalization.py | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index c76692a95..1744d388c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -60,7 +60,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support attestations.append(spec.PendingAttestation( aggregation_bits=aggregation_bits, data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, + beacon_block_root=b'\xff' * 32, # irrelevant to testing source=source, target=target, crosslink=spec.Crosslink(shard=shard) @@ -69,6 +69,20 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support )) +def get_checkpoints(spec, epoch): + c1 = None if epoch < 1 else spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) + c2 = None if epoch < 2 else spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c3 = None if epoch < 3 else spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) + c4 = None if epoch < 4 else spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) + c5 = None if epoch < 5 else spec.Checkpoint(epoch=epoch - 5, root=b'\xee' * 32) + return c1, c2, c3, c4, c5 + + +def put_checkpoints_in_block_roots(spec, state, checkpoints): + for c in checkpoints: + state.block_roots[spec.get_epoch_start_slot(c.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c.root + + def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -77,12 +91,8 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices # 11*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + c1, c2, c3, c4, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3, c4]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c4 @@ -117,16 +127,14 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + c1, c2, c3, _, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (indices are pre-shift) + state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (index is pre-shift) # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -147,7 +155,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): def finalize_on_123(spec, state, epoch, sufficient_support): - assert epoch > 3 + assert epoch > 5 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago @@ -155,19 +163,20 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xdd' * 32) - c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xcc' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) - state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root - state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root + c1, c2, c3, c4, c5 = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2, c3, c4, c5]) old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c4 # not c3, otherwise finalize 23 would trigger. - state.current_justified_checkpoint = c2 + state.previous_justified_checkpoint = c5 + state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0:2] = [1, 1] # mock 2nd and 3rd latest epochs as justified (indices are pre-shift) + state.justification_bits[1] = 1 # mock 3rd latest epochs as justified (index is pre-shift) + # mock the 2nd latest epoch as justifiable, with 5th as source + add_mock_attestations(spec, state, + epoch=epoch - 2, + source=c5, + target=c2, + sufficient_support=sufficient_support) # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -178,12 +187,12 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) - assert state.previous_justified_checkpoint == c2 # changed to old current + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: assert state.current_justified_checkpoint == c1 # changed to 1st latest - assert state.finalized_checkpoint == c2 # finalized old current + assert state.finalized_checkpoint == c3 # finalized old current else: - assert state.current_justified_checkpoint == c2 # still old current + assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -196,10 +205,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # 3210x -- justification bitfield indices (post shift) # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xaa' * 32) - state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root - state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root + c1, c2, _, _, _ = get_checkpoints(spec, epoch) + put_checkpoints_in_block_roots(spec, state, [c1, c2]) old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c2 @@ -252,13 +259,13 @@ def test_23_poor_support(spec, state): @with_all_phases @spec_state_test def test_123_ok_support(spec, state): - yield from finalize_on_123(spec, state, 4, True) + yield from finalize_on_123(spec, state, 6, True) @with_all_phases @spec_state_test def test_123_poor_support(spec, state): - yield from finalize_on_123(spec, state, 4, False) + yield from finalize_on_123(spec, state, 6, False) @with_all_phases