From c5143ccefaddd9929bfb68401c6c00e1afa75e6d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:48:10 -0600 Subject: [PATCH 01/53] modify fork choice to utilize epochs as first class citizens --- specs/core/0_fork-choice.md | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6b7e4bdbe..3bcb6b889 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -63,17 +63,30 @@ class Target(object): root: Hash ``` +#### `RootSlot` + +```python +@dataclass +class RootSlot(object): + slot: Slot + root: Hash +``` + + + #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[Hash, BeaconState] = field(default_factory=dict) + states: Dict[RootSlot, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) justified_root: Hash = ZERO_HASH + justified_epoch: Epoch = 0 finalized_root: Hash = ZERO_HASH + finalized_epoch: Epoch = 0 ``` #### `get_genesis_store` @@ -98,8 +111,8 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[store.justified_root] - active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root @@ -112,8 +125,12 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_root + justified_slot = get_epoch_start_slot(store.justified_epoch) while True: - children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == head and store.blocks[root].slot > justified_slot + ] if len(children) == 0: return head # Sort by latest attesting balance with ties broken lexicographically @@ -137,30 +154,58 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check that block is later than the finalized epoch slot + assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time pre_state = deepcopy(store.states[block.parent_root]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) - # Add new state to the store - store.states[signing_root(block)] = state - # Update justified block root - if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + # Add new state for this block to the store + store.states[RootSlot(signing_root(block), block.slot)] = state + # Update justified block root and epoch + previous_justified_epoch = store.justified_epoch + justified_root_slot = None + if state.current_justified_epoch > store.justified_epoch: store.justified_root = state.current_justified_root - elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + state.justified_epoch = state.current_justified_epoch + elif state.previous_justified_epoch > store.justified_epoch: store.justified_root = state.previous_justified_root - # Update finalized block root - if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + state.justified_epoch = state.previous_justified_epoch + + # Store justified state + if previous_justified_epoch != store.justified_epoch: + justified_slot = get_epoch_start_slot(store.justified_epoch) + justified_root_slot = RootSlot(store.justified_root, justified_slot) + if justified_root_slot not in store.states: + base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] + store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) + + # Update finalized block root and epoch, and store finalized state + if state.finalized_epoch > state.finalized_epoch: store.finalized_root = state.finalized_root + store.finalized_epoch = state.finalized_epoch + finalized_slot = get_epoch_start_slot(store.finalized_epoch) + finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) + if finalized_root_slot not in store.states: + base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] + store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - state = store.states[get_head(store)] + # cannot calculate the current shuffling if have not seen the target + assert attestation.data.target_root in store.blocks + + # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees + state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) + + # update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) From fb9a5f0bc5e2358fbb2a57f3324b83ca2f82ed07 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 20 Jun 2019 14:57:53 -0600 Subject: [PATCH 02/53] one more rootslot fix --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3bcb6b889..668b20714 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -157,7 +157,8 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check that block is later than the finalized epoch slot assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) # Check block slot against Unix time - pre_state = deepcopy(store.states[block.parent_root]) + parent_block = store.blocks[block.parent_root] + pre_state = deepcopy(store.states[RootSlot(signing_root(parent_block), parent_block.slot)]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) From 1b66a1a2bd297d33f666deede5b403388197073a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 12:55:55 -0600 Subject: [PATCH 03/53] rework forkchoice to use Checkpoints --- specs/core/0_fork-choice.md | 104 +++++++++++++++--------------------- 1 file changed, 42 insertions(+), 62 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index ee318412d..1755c6df3 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -12,7 +12,7 @@ - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [`Target`](#target) + - [`Checkpoint`](#checkpoint) - [`Store`](#store) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) @@ -55,39 +55,27 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Target` +#### `Checkpoint` ```python @dataclass -class Target(object): +class Checkpoint(object): epoch: Epoch root: Hash ``` -#### `RootSlot` - -```python -@dataclass -class RootSlot(object): - slot: Slot - root: Hash -``` - - - #### `Store` ```python @dataclass class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) - states: Dict[RootSlot, BeaconState] = field(default_factory=dict) - time: int = 0 - latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Hash = ZERO_HASH - justified_epoch: Epoch = 0 - finalized_root: Hash = ZERO_HASH - finalized_epoch: Epoch = 0 + block_states: Dict[Hash, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + time: int + latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -96,12 +84,15 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) + justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) + finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( blocks={root: genesis_block}, - states={root: genesis_state}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, - justified_root=root, - finalized_root=root, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, ) ``` @@ -118,7 +109,7 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: - state = store.states[RootSlot(store.justified_root, get_epoch_start_slot(store.justified_epoch)] + state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices @@ -131,8 +122,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: ```python def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice - head = store.justified_root - justified_slot = get_epoch_start_slot(store.justified_epoch) + head = store.justified_checkpoint.root + justified_slot = get_epoch_start_slot(store.justified_checkpoint.epoch) while True: children = [ root for root in store.blocks.keys() @@ -159,63 +150,52 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.states[RootSlot(signing_root(parent_block), parent_block.slot)].copy() + pre_state = store.block_states[parent_block.root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot - assert blocks.slot > get_epoch_start_slot(store.finalized_epoch) + assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state for this block to the store - store.states[RootSlot(signing_root(block), block.slot)] = state - # Update justified block root and epoch - previous_justified_epoch = store.justified_epoch - justified_root_slot = None - if state.current_justified_epoch > store.justified_epoch: - store.justified_root = state.current_justified_root - state.justified_epoch = state.current_justified_epoch + store.block_states[signing_root(block)] = state + + # Update justified checkpoint + if state.current_justified_epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) elif state.previous_justified_epoch > store.justified_epoch: - store.justified_root = state.previous_justified_root - state.justified_epoch = state.previous_justified_epoch + store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) - # Store justified state - if previous_justified_epoch != store.justified_epoch: - justified_slot = get_epoch_start_slot(store.justified_epoch) - justified_root_slot = RootSlot(store.justified_root, justified_slot) - if justified_root_slot not in store.states: - base_state = store.states[RootSlot(store.justified_root, store.blocks[store.justified_root].slot)] - store.states[justified_root_slot] = process_slots(deepcopy(base_state), justified_slot) - - # Update finalized block root and epoch, and store finalized state + # Update finalized checkpoint if state.finalized_epoch > state.finalized_epoch: - store.finalized_root = state.finalized_root - store.finalized_epoch = state.finalized_epoch - finalized_slot = get_epoch_start_slot(store.finalized_epoch) - finalized_root_slot = RootSlot(store.finalized_root, finalized_slot) - if finalized_root_slot not in store.states: - base_state = store.states[RootSlot(store.finalized_root, store.blocks[store.finalized_root].slot)] - store.states[finalized_root_slot] = process_slots(deepcopy(base_state), finalized_slot) + store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - # cannot calculate the current shuffling if have not seen the target - assert attestation.data.target_root in store.blocks + target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) - # get state at the `target_root`/`target_epoch` to validate attestation and calculate the committees - state = store.states[(attestation.data.target_root, get_epoch_start_slot(attestation.data.target_epoch))] + # Cannot calculate the current shuffling if have not seen the target + assert target_checkpoint.root in store.blocks + # Store target checkpoint state if not yet seen + if target_checkpoint not in store.checkpoint_states: + base_state = store.block_states[target_checkpoint.root].copy() + store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + + # Get state at the `target_checkpoint` to validate attestation and calculate the committees + state = store.checkpoint_states[target_checkpoint] indexed_attestation = convert_to_indexed(state, attestation) validate_indexed_attestation(state, indexed_attestation) - # update latest targets + # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) + if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target_checkpoint ``` From de5b9cc823dedd9673f3599bf8e209a363cf0dd7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 16:33:43 -0600 Subject: [PATCH 04/53] convert _root/_epoch tuples to Checkpoint obj in beacon spec --- specs/core/0_beacon-chain.md | 92 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5081dbf74..cf1d36862 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -26,6 +26,7 @@ - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - [`Validator`](#validator) + - [`Checkpoint`](#checkpoint) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) @@ -303,6 +304,14 @@ class Fork(Container): epoch: Epoch # Epoch of latest fork ``` +#### `Checkpoint` + +```python +class Checkpoint(Container): + epoch: Epoch + root: Hash +``` + #### `Validator` ```python @@ -337,10 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_epoch: Epoch - source_root: Hash - target_epoch: Epoch - target_root: Hash + source_checkpoint: Checkpoint + target_checkpoint: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -534,14 +541,11 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Justification - previous_justified_epoch: Epoch # Previous epoch snapshot - previous_justified_root: Hash # Previous epoch snapshot - current_justified_epoch: Epoch - current_justified_root: Hash + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint justification_bitfield: uint64 # Bit set for every recent justified epoch # Finality - finalized_epoch: Epoch - finalized_root: Hash + finalized_checkpoint: Checkpoint ``` ## Helper functions @@ -715,9 +719,10 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - committee_count = get_epoch_committee_count(state, data.target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + target_epoch = data.target_checkpoint.epoch + committee_count = get_epoch_committee_count(state, target_epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -876,12 +881,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, - attestation_data: AttestationData, + data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) + committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1001,7 +1006,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), ) ``` @@ -1014,9 +1019,12 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_epoch == data_2.target_epoch) or + (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or # Surround vote - (data_1.source_epoch < data_2.source_epoch and data_2.target_epoch < data_1.target_epoch) + ( + data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and + data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch + ) ) ``` @@ -1274,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_root == get_block_root(state, epoch) + if a.data.target_checkpoint.root == get_block_root(state, epoch) ] ``` @@ -1326,46 +1334,39 @@ def process_justification_and_finalization(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) current_epoch = get_current_epoch(state) - old_previous_justified_epoch = state.previous_justified_epoch - old_current_justified_epoch = state.current_justified_epoch + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint # Process justifications - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root + state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = previous_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_epoch = current_epoch - state.current_justified_root = get_block_root(state, state.current_justified_epoch) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_previous_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: - state.finalized_epoch = old_current_justified_epoch - state.finalized_root = get_block_root(state, state.finalized_epoch) + state.finalized_checkpoint = old_current_justified_checkpoint ``` #### Crosslinks @@ -1428,7 +1429,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: rewards[index] += Gwei(max_attester_reward * MIN_ATTESTATION_INCLUSION_DELAY // attestation.inclusion_delay) # Inactivity penalty - finality_delay = previous_epoch - state.finalized_epoch + finality_delay = previous_epoch - state.finalized_checkpoint.epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in eligible_validator_indices: @@ -1493,7 +1494,7 @@ def process_registry_updates(state: BeaconState) -> None: activation_queue = sorted([ index for index, validator in enumerate(state.validators) if validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and - validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch) + validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_checkpoint.epoch) ], key=lambda index: state.validators[index].activation_eligibility_epoch) # Dequeued validators for activation up to churn limit (without resetting activation epoch) for index in activation_queue[:get_churn_limit(state)]: @@ -1691,9 +1692,10 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data + target_epoch = data.target_checkpoint.epoch assert data.crosslink.shard < SHARD_COUNT - assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1705,19 +1707,19 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if data.target_epoch == get_current_epoch(state): - ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) + if target_epoch == get_current_epoch(state): + ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) + ffg_source = state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG data, crosslink data, and signature - assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) + # Check FFG source, crosslink data, and signature + assert ffg_source == data.source_checkpoint assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From a5b22e13b8b4bf8247b2e130b12917eee026e0f6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 22 Jun 2019 16:56:16 +0200 Subject: [PATCH 05/53] Resolves make masker sign mask --- specs/core/1_custody-game.md | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 13 +++++++++---- .../test_process_early_derived_secret_reveal.py | 16 +++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07..b051d13c6 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -204,7 +204,7 @@ class EarlyDerivedSecretReveal(Container): # Index of the validator who revealed (whistleblower) masker_index: ValidatorIndex # Mask used to hide the actual reveal signature (prevent reveal from being stolen) - mask: Bytes32 + mask: Hash ``` ### Phase 0 container updates diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcd..0167edc6a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -10,6 +10,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): if epoch is None: epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + # Generate the secret that is being revealed reveal = bls_sign( message_hash=spec.hash_tree_root(epoch), privkey=privkeys[revealed_index], @@ -19,8 +20,11 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + # Generate the mask (any random 32 bytes will do) + mask = reveal[:32] + # Generate masker's signature on the mask + masker_signature = bls_sign( + message_hash=mask, privkey=privkeys[masker_index], domain=spec.get_domain( state=state, @@ -28,11 +32,12 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) + masked_reveal = bls_aggregate_signatures([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, epoch=epoch, - reveal=reveal, + reveal=masked_reveal, masker_index=masker_index, mask=mask, ) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 87297d443..831ad35a5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,7 +1,13 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.context import with_all_phases_except, spec_state_test, expect_assertion_error +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, + always_bls, + never_bls, +) def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): @@ -36,6 +42,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_success(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) @@ -44,6 +51,7 @@ def test_success(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_current_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -52,6 +60,7 @@ def test_reveal_from_current_epoch(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_reveal_from_past_epoch(spec, state): next_epoch(spec, state) @@ -62,6 +71,7 @@ def test_reveal_from_past_epoch(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -73,6 +83,7 @@ def test_reveal_with_custody_padding(spec, state): @with_all_phases_except(['phase0']) +@always_bls @spec_state_test def test_reveal_with_custody_padding_minus_one(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -84,6 +95,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_double_reveal(spec, state): randao_key_reveal1 = get_valid_early_derived_secret_reveal( @@ -108,6 +120,7 @@ def test_double_reveal(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -117,6 +130,7 @@ def test_revealer_is_slashed(spec, state): @with_all_phases_except(['phase0']) +@never_bls @spec_state_test def test_far_future_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( From 8a732fbbcfbfae529cc34458681a965e135a36f8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 22 Jun 2019 12:00:26 -0600 Subject: [PATCH 06/53] pass on checkpoints working in testing --- specs/core/0_beacon-chain.md | 12 +++---- .../eth2spec/test/helpers/attestations.py | 22 ++++++------- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 31 ++++++++--------- .../test_process_attester_slashing.py | 10 +++--- .../test_process_crosslinks.py | 4 +-- .../pyspec/eth2spec/test/test_finality.py | 33 ++++++++----------- 7 files changed, 52 insertions(+), 62 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf1d36862..838178ea4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1344,28 +1344,28 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7eb..d2ce6e29b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -24,11 +24,11 @@ def build_attestation_data(spec, state, slot, shard): epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root + source_epoch = state.previous_justified_checkpoint.epoch + source_root = state.previous_justified_checkpoint.root else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root + source_epoch = state.current_justified_checkpoint.epoch + source_root = state.current_justified_checkpoint.root if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): parent_crosslink = state.current_crosslinks[shard] @@ -37,10 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=spec.slot_to_epoch(slot), - target_root=epoch_boundary_root, + source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), + target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -64,8 +62,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_epoch, - attestation_data.crosslink.shard + attestation_data.target_checkpoint.epoch, + attestation_data.crosslink.shard, ) committee_size = len(crosslink_committee) @@ -126,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, + message_epoch=attestation_data.target_checkpoint.epoch, ) ) @@ -134,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 9fd34520c..e339421e8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 + attestation_2.data.target_checkpoint.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..9a718b071 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_epoch == spec.get_current_epoch(state): + if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -119,16 +119,16 @@ def test_after_epoch_slots(spec, state): @spec_state_test def test_old_source_epoch(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 - state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.current_justified_epoch = 4 + state.finalized_checkpoint.epoch = 2 + state.previous_justified_checkpoint.epoch = 3 + state.current_justified_checkpoint.epoch = 4 attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_epoch == state.previous_justified_epoch + assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_epoch -= 1 + attestation.data.source_checkpoint.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_epoch += 1 + attestation.data.source_checkpoint.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = attestation.data.target_root + attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root sign_attestation(spec, state, attestation) @@ -180,21 +180,18 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_epoch = 3 - state.previous_justified_root = b'\x01' * 32 - - state.current_justified_epoch = 4 - state.current_justified_root = b'\xff' * 32 + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: - assert state.current_justified_root != state.previous_justified_root - assert attestation.data.source_root == state.previous_justified_root + assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root + assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_root = state.current_justified_root + attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -207,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_root = b'\x42' * 32 + attestation.data.source_checkpoint.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index c51f5a8a9..71915f1d0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -68,12 +68,14 @@ def test_success_surround(spec, state): next_epoch(spec, state) apply_empty_block(spec, state) - state.current_justified_epoch += 1 + state.current_justified_checkpoint.epoch += 1 attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 - attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 + attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 + attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -120,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_epoch += 1 + attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index d51191efb..482085fd1 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_epoch, + attestation.data.target_checkpoint.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_epoch, + attestation_2.data.target_checkpoint.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c8..e405312eb 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -14,25 +14,22 @@ def check_finality(spec, previous_justified_changed, finalized_changed): if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root + assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch + assert state.current_justified_checkpoint.root != prev_state.current_justified_checkpoint.root else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root + assert state.current_justified_checkpoint == prev_state.current_justified_checkpoint if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root + assert state.previous_justified_checkpoint.epoch > prev_state.previous_justified_checkpoint.epoch + assert state.previous_justified_checkpoint.root != prev_state.previous_justified_checkpoint.root else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root + assert state.previous_justified_checkpoint == prev_state.previous_justified_checkpoint if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root + assert state.finalized_checkpoint.epoch > prev_state.finalized_checkpoint.epoch + assert state.finalized_checkpoint.root != prev_state.finalized_checkpoint.root else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root + assert state.finalized_checkpoint == prev_state.finalized_checkpoint def next_epoch_with_attestations(spec, @@ -82,8 +79,7 @@ def test_finality_rule_4(spec, state): elif epoch >= 3: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -113,8 +109,7 @@ def test_finality_rule_1(spec, state): elif epoch == 2: # finalized by rule 1 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -144,8 +139,7 @@ def test_finality_rule_2(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) # finalized by rule 2 check_finality(spec, state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root + assert state.finalized_checkpoint == prev_state.previous_justified_checkpoint blocks += new_blocks @@ -196,8 +190,7 @@ def test_finality_rule_3(spec, state): blocks += new_blocks # rule 3 check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root + assert state.finalized_checkpoint == prev_state.current_justified_checkpoint yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state From 181a2a876a369051a8a6db3b0c972d13cdb81202 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:49:53 +0200 Subject: [PATCH 07/53] Cleanups --- specs/core/0_beacon-chain.md | 38 ++++++++----------- .../eth2spec/test/helpers/attestations.py | 10 ++--- .../test/helpers/attester_slashings.py | 2 +- .../test_process_attestation.py | 16 ++++---- .../test_process_attester_slashing.py | 6 +-- .../test_process_crosslinks.py | 4 +- 6 files changed, 35 insertions(+), 41 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 838178ea4..d8d3d6a76 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -346,8 +346,8 @@ class AttestationData(Container): # LMD GHOST vote beacon_block_root: Hash # FFG vote - source_checkpoint: Checkpoint - target_checkpoint: Checkpoint + source: Checkpoint + target: Checkpoint # Crosslink vote crosslink: Crosslink ``` @@ -719,10 +719,9 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: - target_epoch = data.target_checkpoint.epoch - committee_count = get_epoch_committee_count(state, target_epoch) - offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, target_epoch)) % SHARD_COUNT - return Slot(get_epoch_start_slot(target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) + committee_count = get_epoch_committee_count(state, data.target.epoch) + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target.epoch)) % SHARD_COUNT + return Slot(get_epoch_start_slot(data.target.epoch) + offset // (committee_count // SLOTS_PER_EPOCH)) ``` ### `get_block_root_at_slot` @@ -886,7 +885,7 @@ def get_attesting_indices(state: BeaconState, """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, data.target_checkpoint.epoch, data.crosslink.shard) + committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1006,7 +1005,7 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_checkpoint.epoch), + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target.epoch), ) ``` @@ -1019,12 +1018,9 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa """ return ( # Double vote - (data_1 != data_2 and data_1.target_checkpoint.epoch == data_2.target_checkpoint.epoch) or + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or # Surround vote - ( - data_1.source_checkpoint.epoch < data_2.source_checkpoint.epoch and - data_2.target_checkpoint.epoch < data_1.target_checkpoint.epoch - ) + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) ) ``` @@ -1282,7 +1278,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.target_checkpoint.root == get_block_root(state, epoch) + if a.data.target.root == get_block_root(state, epoch) ] ``` @@ -1344,13 +1340,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1692,10 +1688,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` operation. """ data = attestation.data - target_epoch = data.target_checkpoint.epoch - assert data.crosslink.shard < SHARD_COUNT - assert target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH @@ -1707,7 +1701,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_index=get_beacon_proposer_index(state), ) - if target_epoch == get_current_epoch(state): + if data.target.epoch == get_current_epoch(state): ffg_source = state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) @@ -1717,9 +1711,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: state.previous_epoch_attestations.append(pending_attestation) # Check FFG source, crosslink data, and signature - assert ffg_source == data.source_checkpoint + assert ffg_source == data.source assert data.crosslink.start_epoch == parent_crosslink.end_epoch - assert data.crosslink.end_epoch == min(target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d2ce6e29b..d717bdc0d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -37,8 +37,8 @@ def build_attestation_data(spec, state, slot, shard): return spec.AttestationData( beacon_block_root=block_root, - source_checkpoint=spec.Checkpoint(epoch=source_epoch, root=source_root), - target_checkpoint=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), + source=spec.Checkpoint(epoch=source_epoch, root=source_root), + target=spec.Checkpoint(epoch=spec.slot_to_epoch(slot), root=epoch_boundary_root), crosslink=spec.Crosslink( shard=shard, start_epoch=parent_crosslink.end_epoch, @@ -62,7 +62,7 @@ def get_valid_attestation(spec, state, slot=None, signed=False): crosslink_committee = spec.get_crosslink_committee( state, - attestation_data.target_checkpoint.epoch, + attestation_data.target.epoch, attestation_data.crosslink.shard, ) @@ -124,7 +124,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_checkpoint.epoch, + message_epoch=attestation_data.target.epoch, ) ) @@ -132,7 +132,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi def fill_aggregate_attestation(spec, state, attestation): crosslink_committee = spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index e339421e8..9c68b7bbe 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -7,7 +7,7 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): attestation_1 = get_valid_attestation(spec, state, signed=signed_1) attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_checkpoint.root = b'\x01' * 32 + attestation_2.data.target.root = b'\x01' * 32 if signed_2: sign_attestation(spec, state, attestation_2) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 9a718b071..d6cb615f8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target_checkpoint.epoch == spec.get_current_epoch(state): + if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 @@ -125,10 +125,10 @@ def test_old_source_epoch(spec, state): attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch - assert attestation.data.source_checkpoint.epoch == state.previous_justified_checkpoint.epoch + assert attestation.data.source.epoch == state.previous_justified_checkpoint.epoch # Now go beyond that, it will be invalid - attestation.data.source_checkpoint.epoch -= 1 + attestation.data.source.epoch -= 1 sign_attestation(spec, state, attestation) @@ -154,7 +154,7 @@ def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.epoch += 1 + attestation.data.source.epoch += 1 sign_attestation(spec, state, attestation) @@ -167,7 +167,7 @@ def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = attestation.data.target_checkpoint.root + attestation.data.source.root = attestation.data.target.root sign_attestation(spec, state, attestation) @@ -188,10 +188,10 @@ def test_invalid_current_source_root(spec, state): # Test logic sanity checks: assert state.current_justified_checkpoint.root != state.previous_justified_checkpoint.root - assert attestation.data.source_checkpoint.root == state.previous_justified_checkpoint.root + assert attestation.data.source.root == state.previous_justified_checkpoint.root # Make attestation source root invalid: should be previous justified, not current one - attestation.data.source_checkpoint.root = state.current_justified_checkpoint.root + attestation.data.source.root = state.current_justified_checkpoint.root sign_attestation(spec, state, attestation) @@ -204,7 +204,7 @@ def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_checkpoint.root = b'\x42' * 32 + attestation.data.source.root = b'\x42' * 32 sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 71915f1d0..e2b50ea0b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -74,8 +74,8 @@ def test_success_surround(spec, state): attestation_2 = attester_slashing.attestation_2 # set attestion1 to surround attestation 2 - attestation_1.data.source_checkpoint.epoch = attestation_2.data.source_checkpoint.epoch - 1 - attestation_1.data.target_checkpoint.epoch = attestation_2.data.target_checkpoint.epoch + 1 + attestation_1.data.source.epoch = attestation_2.data.source.epoch - 1 + attestation_1.data.target.epoch = attestation_2.data.target.epoch + 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -122,7 +122,7 @@ def test_same_data(spec, state): def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - attester_slashing.attestation_1.data.target_checkpoint.epoch += 1 + attester_slashing.attestation_1.data.target.epoch += 1 sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 482085fd1..090858a2e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -96,7 +96,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( state, - attestation.data.target_checkpoint.epoch, + attestation.data.target.epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -148,7 +148,7 @@ def test_double_late_crosslink(spec, state): # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( state, - attestation_2.data.target_checkpoint.epoch, + attestation_2.data.target.epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 From e6e90c7736e69759c59dd6cf6fa2efc9c11e1d3c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:51:04 +0200 Subject: [PATCH 08/53] Minor reorg --- specs/core/0_beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d8d3d6a76..ae99a807a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -540,11 +540,10 @@ class BeaconState(Container): # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] - # Justification + # Finality + justification_bitfield: uint64 # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint - justification_bitfield: uint64 # Bit set for every recent justified epoch - # Finality finalized_checkpoint: Checkpoint ``` From 98692bf9d69198c8079d4af9458e5b4e13243cb2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:52:37 +0200 Subject: [PATCH 09/53] Fix ToC --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ae99a807a..a40bef6d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -25,8 +25,8 @@ - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - - [`Validator`](#validator) - [`Checkpoint`](#checkpoint) + - [`Validator`](#validator) - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) From 2342c787c96b034e9c2db08445b15fdc3a7f8b50 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 22:54:07 +0200 Subject: [PATCH 10/53] Cleanup --- specs/core/0_beacon-chain.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a40bef6d1..30876ae29 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -878,9 +878,7 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, - data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. """ From 1e2bb08a747abc3b9ef4c5434c074bc078360a53 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:09:09 +0200 Subject: [PATCH 11/53] Cosmetic fix for consistency --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 30876ae29..c062bbf78 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1267,7 +1267,7 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - assert epoch in (get_current_epoch(state), get_previous_epoch(state)) + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` From bb0a492fea2f4e2e3a036999325d6d8141b7b4b4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 23 Jun 2019 11:18:24 +0200 Subject: [PATCH 12/53] Cleanups --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c062bbf78..68a121cd4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1699,20 +1699,21 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ) if data.target.epoch == get_current_epoch(state): - ffg_source = state.current_justified_checkpoint + assert data.source == state.current_justified_checkpoint parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: - ffg_source = state.previous_justified_checkpoint + assert data.source == state.previous_justified_checkpoint parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) - # Check FFG source, crosslink data, and signature - assert ffg_source == data.source + # Check crosslink against expected parent crosslink + assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] + + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` From d9644f518b6ce8a9269b08c18942d5ff1d6773f1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:08:13 +0200 Subject: [PATCH 13/53] mask is hash() in tests Co-Authored-By: dankrad --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 0167edc6a..8037755bc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -21,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) # Generate the mask (any random 32 bytes will do) - mask = reveal[:32] + mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( message_hash=mask, From 139d0f56f10972d6677b70683702f819b5c2fe02 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 24 Jun 2019 16:26:21 +0200 Subject: [PATCH 14/53] Finishes moving mask to hash() --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 8037755bc..9ca770c11 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.hash_function import hash def get_valid_early_derived_secret_reveal(spec, state, epoch=None): @@ -20,7 +21,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): message_epoch=epoch, ), ) - # Generate the mask (any random 32 bytes will do) + # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask masker_signature = bls_sign( From 0e362d36b1eda660d013d32002acf7fce0c8ff91 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 17:18:22 -0600 Subject: [PATCH 15/53] pr feedback --- specs/core/0_fork-choice.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1755c6df3..1818de6c4 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -179,23 +179,31 @@ def on_block(store: Store, block: BeaconBlock) -> None: ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target_checkpoint = Checkpoint(attestation.data.target_epoch, attestation.data, target_root) + target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) # Cannot calculate the current shuffling if have not seen the target - assert target_checkpoint.root in store.blocks + assert target.root in store.blocks + + # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen - if target_checkpoint not in store.checkpoint_states: - base_state = store.block_states[target_checkpoint.root].copy() - store.checkpoint_states[target_checkpoint] = process_slots(base_state, get_epoch_start_slot(target_checkpoint.epoch)) + if target not in store.checkpoint_states: + base_state = store.block_states[target.root].copy() + store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + target_state = store.checkpoint_states[target] - # Get state at the `target_checkpoint` to validate attestation and calculate the committees - state = store.checkpoint_states[target_checkpoint] - indexed_attestation = convert_to_indexed(state, attestation) - validate_indexed_attestation(state, indexed_attestation) + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + attestation_slot = get_attestation_data_slot(target_state, attestation.data) + assert store.time >= (attestation_slot + 1) * SECONDS_PER_SLOT + + # Get state at the `target` to validate attestation and calculate the committees + indexed_attestation = convert_to_indexed(target_state, attestation) + validate_indexed_attestation(target_state, indexed_attestation) # Update latest targets for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target_checkpoint.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target_checkpoint + if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = target ``` From 751738f411842c3cf1ac7d3ac9ec828d754912b0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:01:15 -0600 Subject: [PATCH 16/53] enhance fork choice testing --- specs/core/0_fork-choice.md | 24 ++-- .../test/fork_choice/test_on_attestation.py | 134 ++++++++++++++++++ .../test/fork_choice/test_on_block.py | 95 +++++++++++++ .../pyspec/eth2spec/test/test_fork_choice.py | 59 -------- 4 files changed, 241 insertions(+), 71 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1818de6c4..3ab041837 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -58,7 +58,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A #### `Checkpoint` ```python -@dataclass +@dataclass(eq=True, frozen=True) class Checkpoint(object): epoch: Epoch root: Hash @@ -69,13 +69,13 @@ class Checkpoint(object): ```python @dataclass class Store(object): + time: int + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - time: int latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint ``` #### `get_genesis_store` @@ -87,12 +87,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) return Store( - blocks={root: genesis_block}, - block_states={root: genesis_state}, - checkpoint_states={justified_checkpoint: genesis_state.copy()}, time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, + blocks={root: genesis_block}, + block_states={root: genesis_state}, + checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -150,13 +150,13 @@ def on_tick(store: Store, time: int) -> None: def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues parent_block = store.blocks[block.parent_root] - pre_state = store.block_states[parent_block.root].copy() + pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[get_epoch_start_slot(store.finalized_checkpoint)].slot) == store.finalized_checkpoint.root + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -167,7 +167,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Update justified checkpoint if state.current_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_epoch: + elif state.previous_justified_epoch > store.justified_checkpoint.epoch: store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) # Update finalized checkpoint @@ -185,11 +185,11 @@ def on_attestation(store: Store, attestation: Attestation) -> None: assert target.root in store.blocks # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. - assert store.time >= pre_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT + base_state = store.block_states[target.root].copy() + assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - base_state = store.block_states[target.root].copy() store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) target_state = store.checkpoint_states[target] diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py new file mode 100644 index 000000000..fd42a6b29 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -0,0 +1,134 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +def run_on_attestation(spec, state, store, attestation, valid=True): + if not valid: + try: + spec.on_attestation(store, attestation) + except: + return + else: + assert False + + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Checkpoint( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_target_not_in_store(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # move to next epoch to make block new target + state.slot += spec.SLOTS_PER_EPOCH + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # do not add block to store + + next_slot(spec, state) + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_future_epoch(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + # store block in store + spec.on_block(store, block) + next_slot(spec, state) + + # move state forward but not store + attestation_slot = block.slot + spec.SLOTS_PER_EPOCH + state.slot = attestation_slot + + attestation = get_valid_attestation(spec, state, slot=state.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_same_slot(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 1 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + run_on_attestation(spec, state, store, attestation, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation_invalid_attestation(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + store = spec.get_genesis_store(state) + time = 3 * spec.SECONDS_PER_SLOT + spec.on_tick(store, time) + + block = build_empty_block_for_next_slot(spec, state, signed=True) + + spec.on_block(store, block) + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + # make attestation invalid + attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] + run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py new file mode 100644 index 000000000..ef25317b6 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -0,0 +1,95 @@ +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot + + +def run_on_block(spec, state, store, block, valid=True): + if not valid: + try: + spec.on_block(store, block) + except: + return + else: + assert False + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + +@with_all_phases +@with_state +@bls_switch +def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block) + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + + run_on_block(spec, state, store, block) + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_future_block(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + + # do not tick time + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_bad_parent_root(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + block.parent_root = b'\x45' * 32 + run_on_block(spec, state, store, block, False) + + +@with_all_phases +@with_state +@bls_switch +def test_on_block_before_finalized(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + + # Fail receiving block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + run_on_block(spec, state, store, block, False) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 4706f0eaf..8b1378917 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,60 +1 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root -from eth2spec.test.context import with_all_phases, with_state, bls_switch - -from eth2spec.test.helpers.block import build_empty_block_for_next_slot -from eth2spec.test.helpers.attestations import get_valid_attestation -from eth2spec.test.helpers.state import next_slot - - -@with_all_phases -@with_state -@bls_switch -def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - - # Initialization - store = spec.get_genesis_store(state) - blocks = [] - time = 100 - spec.on_tick(store, time) - assert store.time == time - - # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH - blocks.append(block) - - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - - # TODO: add tests for justified_root and finalized_root - - -@with_all_phases -@with_state -@bls_switch -def test_on_attestation(spec, state): - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - - next_slot(spec, state) - - attestation = get_valid_attestation(spec, state, slot=1) - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, - ) - ) From d9b97578c0f73fb0ccd03a5727596eaec4847654 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:09:57 -0600 Subject: [PATCH 17/53] lint --- specs/core/0_fork-choice.md | 12 ++++++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 4 ++-- .../eth2spec/test/fork_choice/test_on_block.py | 9 ++++++--- test_libs/pyspec/eth2spec/test/test_fork_choice.py | 1 - 4 files changed, 16 insertions(+), 10 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/test_fork_choice.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3ab041837..71920d2bc 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -149,14 +149,17 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Make a copy of the state to avoid mutability issues - parent_block = store.blocks[block.parent_root] + assert block.parent_root in store.block_states pre_state = store.block_states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block - assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == store.finalized_checkpoint.root + assert ( + get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) == + store.finalized_checkpoint.root + ) # Check that block is later than the finalized epoch slot assert block.slot > get_epoch_start_slot(store.finalized_checkpoint.epoch) # Check the block is valid and compute the post-state @@ -184,13 +187,14 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks - # Attestations cannot be from future epochs. If they are, their consideration must be delayed until the are in the past. + # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrivesr base_state = store.block_states[target.root].copy() assert store.time >= base_state.genesis_time + get_epoch_start_slot(target.epoch) * SECONDS_PER_SLOT # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - store.checkpoint_states[target] = process_slots(base_state, get_epoch_start_slot(target.epoch)) + process_slots(base_state, get_epoch_start_slot(target.epoch)) + store.checkpoint_states[target] = base_state target_state = store.checkpoint_states[target] # Attestations can only affect the fork choice of subsequent slots. diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index fd42a6b29..cf0e7c9cb 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -11,7 +11,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if not valid: try: spec.on_attestation(store, attestation) - except: + except AssertionError: return else: assert False @@ -131,4 +131,4 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid attestation.custody_bitfield = b'\xf0' + attestation.custody_bitfield[1:] - run_on_attestation(spec, state, store, attestation, False) \ No newline at end of file + run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index ef25317b6..b18752f8c 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -9,7 +9,7 @@ def run_on_block(spec, state, store, block, valid=True): if not valid: try: spec.on_block(store, block) - except: + except AssertionError: return else: assert False @@ -88,8 +88,11 @@ def test_on_block_before_finalized(spec, state): time = 100 spec.on_tick(store, time) - store.finalized_checkpoint = spec.Checkpoint(epoch=store.finalized_checkpoint.epoch + 2, root=store.finalized_checkpoint.root) + store.finalized_checkpoint = spec.Checkpoint( + epoch=store.finalized_checkpoint.epoch + 2, + root=store.finalized_checkpoint.root + ) # Fail receiving block of `GENESIS_SLOT + 1` slot block = build_empty_block_for_next_slot(spec, state) - run_on_block(spec, state, store, block, False) \ No newline at end of file + run_on_block(spec, state, store, block, False) diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py deleted file mode 100644 index 8b1378917..000000000 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ /dev/null @@ -1 +0,0 @@ - From b8c0985e6063db80a99ec1b87127cf1106761034 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 24 Jun 2019 21:43:05 -0600 Subject: [PATCH 18/53] merge in fork choice. tests pass --- specs/core/0_beacon-chain.md | 7 ++--- specs/core/0_fork-choice.md | 27 +++++++------------ .../test/fork_choice/test_on_attestation.py | 4 +-- .../test_process_attestation.py | 4 +-- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 91e79e98f..1c9d02c33 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1337,13 +1337,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bitfield |= (1 << 1) current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bitfield |= (1 << 0) # Process finalizations @@ -1352,7 +1353,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch+ 2 == current_epoch: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source if (bitfield >> 0) % 8 == 0b111 and old_current_justified_checkpoint.epoch + 2 == current_epoch: diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc..77f8e407d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -55,15 +55,6 @@ The head block root associated with a `store` is defined as `get_head(store)`. A ### Helpers -#### `Checkpoint` - -```python -@dataclass(eq=True, frozen=True) -class Checkpoint(object): - epoch: Epoch - root: Hash -``` - #### `Store` ```python @@ -84,8 +75,8 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - justified_checkpoint = Checkpoint(GENESIS_EPOCH, root) - finalized_checkpoint = Checkpoint(GENESIS_EPOCH, root) + justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) + finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) return Store( time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, @@ -168,21 +159,21 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.block_states[signing_root(block)] = state # Update justified checkpoint - if state.current_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.current_justified_epoch, state.current_justified_root) - elif state.previous_justified_epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = Checkpoint(state.previous_justified_epoch, state.previous_justified_root) + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + elif state.previous_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.previous_justified_checkpoint # Update finalized checkpoint - if state.finalized_epoch > state.finalized_epoch: - store.finalized_checkpoint = Checkpoint(state.finalized_epoch, state.finalized_root) + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint ``` #### `on_attestation` ```python def on_attestation(store: Store, attestation: Attestation) -> None: - target = Checkpoint(attestation.data.target_epoch, attestation.data.target_root) + target = attestation.data.target # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb..20c482d4f 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -21,8 +21,8 @@ def run_on_attestation(spec, state, store, attestation, valid=True): assert ( store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == spec.Checkpoint( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + epoch=attestation.data.target.epoch, + root=attestation.data.target.root, ) ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index d6cb615f8..5ceab4058 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -180,8 +180,8 @@ def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 state.finalized_epoch = 2 - state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01'*32) - state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32'*32) + state.previous_justified_checkpoint = spec.Checkpoint(epoch=3, root=b'\x01' * 32) + state.current_justified_checkpoint = spec.Checkpoint(epoch=4, root=b'\x32' * 32) attestation = get_valid_attestation(spec, state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 846ca649aafece56d7aaf09bb16aa3bc5a76725b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 10:31:39 -0600 Subject: [PATCH 19/53] properly construct genesis latest block header in tests --- .../eth2spec/test/fork_choice/test_on_attestation.py | 12 ------------ .../eth2spec/test/fork_choice/test_on_block.py | 11 +---------- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 4 +++- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index cf0e7c9cb..48f4bf46f 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,5 +1,3 @@ -from eth2spec.utils.ssz.ssz_impl import hash_tree_root - from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -31,8 +29,6 @@ def run_on_attestation(spec, state, store, attestation, valid=True): @with_state @bls_switch def test_on_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -52,8 +48,6 @@ def test_on_attestation(spec, state): @with_state @bls_switch def test_on_attestation_target_not_in_store(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -74,8 +68,6 @@ def test_on_attestation_target_not_in_store(spec, state): @with_state @bls_switch def test_on_attestation_future_epoch(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -98,8 +90,6 @@ def test_on_attestation_future_epoch(spec, state): @with_state @bls_switch def test_on_attestation_same_slot(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 1 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) @@ -117,8 +107,6 @@ def test_on_attestation_same_slot(spec, state): @with_state @bls_switch def test_on_attestation_invalid_attestation(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - store = spec.get_genesis_store(state) time = 3 * spec.SECONDS_PER_SLOT spec.on_tick(store, time) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index b18752f8c..158ee1a58 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -1,7 +1,6 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.test.context import with_all_phases, with_state, bls_switch - from eth2spec.test.helpers.block import build_empty_block_for_next_slot @@ -22,8 +21,6 @@ def run_on_block(spec, state, store, block, valid=True): @with_state @bls_switch def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -48,8 +45,6 @@ def test_basic(spec, state): @with_state @bls_switch def test_on_block_future_block(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) @@ -64,8 +59,6 @@ def test_on_block_future_block(spec, state): @with_state @bls_switch def test_on_block_bad_parent_root(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 @@ -81,8 +74,6 @@ def test_on_block_bad_parent_root(spec, state): @with_state @bls_switch def test_on_block_before_finalized(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - # Initialization store = spec.get_genesis_store(state) time = 100 diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d818908..49e4feb7d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -27,7 +27,9 @@ def create_genesis_state(spec, num_validators): deposit_root=deposit_root, deposit_count=num_validators, block_hash=spec.ZERO_HASH, - )) + ), + latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), + ) # We "hack" in the initial validators, # as it is much faster than creating and processing genesis deposits for every single test case. From 228195d89d464e5c4af7c4223808ffd33b96efda Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 11:48:55 -0600 Subject: [PATCH 20/53] get head tests --- specs/core/0_fork-choice.md | 9 +- .../test/fork_choice/test_get_head.py | 118 ++++++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 71920d2bc..f74f487a1 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -91,7 +91,7 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, blocks={root: genesis_block}, - block_states={root: genesis_state}, + block_states={root: genesis_state.copy()}, checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` @@ -110,10 +110,11 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) + active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( - state.validator_registry[i].effective_balance for i in active_indices - if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_targets and + get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) )) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py new file mode 100644 index 000000000..6ac46ba6c --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -0,0 +1,118 @@ +from eth2spec.test.context import with_all_phases, with_state, bls_switch +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block + + +def add_block_to_store(spec, store, block): + pre_state = store.block_states[block.parent_root] + block_time = pre_state.genesis_time + block.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.signing_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) + + +@with_all_phases +@with_state +@bls_switch +def test_genesis(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + +@with_all_phases +@with_state +@bls_switch +def test_chain_no_attestations(spec, state): + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # On receiving a block of `GENESIS_SLOT + 1` slot + block_1 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_1) + add_block_to_store(spec, store, block_1) + + # On receiving a block of next epoch + block_2 = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block_2) + add_block_to_store(spec, store, block_2) + + assert spec.get_head(store) == spec.signing_root(block_2) + + +@with_all_phases +@with_state +@bls_switch +def test_split_tie_breaker_no_attestations(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # block at slot 1 + block_1_state = genesis_state.copy() + block_1 = build_empty_block_for_next_slot(spec, block_1_state) + state_transition_and_sign_block(spec, block_1_state, block_1) + add_block_to_store(spec, store, block_1) + + # additional block at slot 1 + block_2_state = genesis_state.copy() + block_2 = build_empty_block_for_next_slot(spec, block_2_state) + block_2.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, block_2_state, block_2) + add_block_to_store(spec, store, block_2) + + highest_root = max(spec.signing_root(block_1), spec.signing_root(block_2)) + + assert spec.get_head(store) == highest_root + + +@with_all_phases +@with_state +@bls_switch +def test_shorter_chain_but_heavier_weight(spec, state): + genesis_state = state.copy() + + # Initialization + store = spec.get_genesis_store(state) + genesis_block = spec.BeaconBlock(state_root=state.hash_tree_root()) + assert spec.get_head(store) == spec.signing_root(genesis_block) + + # build longer tree + long_state = genesis_state.copy() + for i in range(3): + long_block = build_empty_block_for_next_slot(spec, long_state) + state_transition_and_sign_block(spec, long_state, long_block) + add_block_to_store(spec, store, long_block) + + # build short tree + short_state = genesis_state.copy() + short_block = build_empty_block_for_next_slot(spec, short_state) + short_block.body.graffiti = b'\x42' * 32 + state_transition_and_sign_block(spec, short_state, short_block) + add_block_to_store(spec, store, short_block) + + short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) + add_attestation_to_store(spec, store, short_attestation) + + assert spec.get_head(store) == spec.signing_root(short_block) From c64289677f0a5372dd1d8b6990ccc6510e0a14bc Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 25 Jun 2019 14:42:37 -0600 Subject: [PATCH 21/53] fix gethead tests --- specs/core/0_fork-choice.md | 22 ++++++++++++++----- .../test/fork_choice/test_on_attestation.py | 6 ++--- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f74f487a1..55ff91a81 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -64,6 +64,16 @@ class Checkpoint(object): root: Hash ``` +#### `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Hash +``` + + #### `Store` ```python @@ -75,7 +85,7 @@ class Store(object): blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_targets: Dict[ValidatorIndex, Checkpoint] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) ``` #### `get_genesis_store` @@ -113,8 +123,8 @@ def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_targets and - get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root) + if (i in store.latest_messages and + get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -207,8 +217,8 @@ def on_attestation(store: Store, attestation: Attestation) -> None: indexed_attestation = convert_to_indexed(target_state, attestation) validate_indexed_attestation(target_state, indexed_attestation) - # Update latest targets + # Update latest messages for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: - if i not in store.latest_targets or target.epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = target + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root) ``` diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 48f4bf46f..0e52e7fd0 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -17,10 +17,10 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.convert_to_indexed(state, attestation) spec.on_attestation(store, attestation) assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Checkpoint( + store.latest_messages[indexed_attestation.custody_bit_0_indices[0]] == + spec.LatestMessage( epoch=attestation.data.target_epoch, - root=attestation.data.target_root, + root=attestation.data.beacon_block_root, ) ) From dbb697dadd3b027979baaffaafc62d5932b895a0 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 26 Jun 2019 19:40:11 -0400 Subject: [PATCH 22/53] Small update to typing in BLS spec file [uint384] -> Tuple[uint384, uint384] --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 3fe1bcc0e..b901b9345 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -71,7 +71,7 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: +def hash_to_G2(message_hash: Bytes32, domain: uint64) -> Tuple[uint384, uint384]: # Initial candidate x coordinate x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big') From 02f6ba36f048d4b79b556df69eed612f660d8e42 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 09:51:06 +0100 Subject: [PATCH 23/53] Add Bitvector and Bitlist Bool, Bit -> boolean, bit Fix simple-serialize.md --- scripts/build_spec.py | 6 +- specs/core/0_beacon-chain.md | 4 +- specs/simple-serialize.md | 87 ++++++++++++++----- test_libs/pyspec/eth2spec/debug/decode.py | 4 +- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 8 +- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 19 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++- .../eth2spec/utils/ssz/test_ssz_impl.py | 10 +-- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 ++-- 11 files changed, 127 insertions(+), 59 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 99c5cd69d..0b1512111 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -52,7 +52,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -174,7 +174,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a..7b9aeee4b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -297,7 +297,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bool + slashed: boolean # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -337,7 +337,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data + custody_bit: bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3318fe45b..53c5649ed 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -15,9 +15,9 @@ - [Default values](#default-values) - [Illegal types](#illegal-types) - [Serialization](#serialization) - - [`"uintN"`](#uintn) - - [`"bool"`](#bool) - - [`"null`](#null) + - [`uintN`](#uintn) + - [`boolean`](#boolean) + - [`null`](#null) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) @@ -37,36 +37,45 @@ ## Typing ### Basic types -* `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) -* `"bool"`: `True` or `False` +* `uintN`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) +* `boolean`: `True` or `False` ### Composite types * **container**: ordered heterogeneous collection of values - * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` + * python dataclass notation with key-type pairs, e.g. +```python +class ContainerExample(Container): + foo: uint64 + bar: boolean +``` * **vector**: ordered fixed-length homogeneous collection of values - * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogeneous collection of values - * angle bracket notation `[type]`, e.g. `["uint64"]` + * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` +* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` + * notation `List[type, N]`, e.g. `List[uint64, N]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type_1, type_2, ...)`, e.g. `("null", "uint64")` + * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` +* **Bitvector**: a fixed-length list of `boolean` values + * notation `Bitvector[N]` +* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` + * notation `Bitlist[N]` ### Variable-size and fixed-size -We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". +We recursively define "variable-size" types to be lists, unions, `Bitlist` and all types that contain a variable-size type. All other types are said to be "fixed-size". ### Aliases For convenience we alias: -* `"byte"` to `"uint8"` (this is a basic type) -* `"bytes"` to `["byte"]` (this is *not* a basic type) -* `"bytesN"` to `["byte", N]` (this is *not* a basic type) -* `"null"`: `{}`, i.e. the empty container +* `bit` to `boolean` +* `byte` to `uint8` (this is a basic type) +* `BytesN` to `Vector[byte, N]` (this is *not* a basic type) +* `null`: `{}`, i.e. the empty container ### Default values -The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. Unions default to the first type in the union (with type index zero), which is `"null"` if present in the union. +The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union. #### `is_empty` @@ -74,34 +83,50 @@ An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization -We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. +We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `bytes`. *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. -### `"uintN"` +### `uintN` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -### `"bool"` +### `boolean` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### `"null"` +### `null` ```python return b"" ``` +### `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((N + 7) // 8, "little") +``` + +### `Bitlist[N]` + +Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. + +```python +as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +``` + ### Vectors, containers, lists, unions ```python @@ -142,17 +167,33 @@ We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. +* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type +### Merkleization of `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return merkleize(as_integer.to_bytes((N + 7) // 8, "little")) +``` + +### `Bitlist[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) +``` + + ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c0b53b0ef..c0b977ab3 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, uint, Container, Bytes, List, boolean, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bool)): + if issubclass(typ, (uint, boolean)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 02814e441..670f580b2 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bool + SSZValue, uint, Container, boolean ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bool): + elif isinstance(value, boolean): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index c6efb722b..cdcba343a 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index e533ca5c2..130956235 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b9c7b6d38..a7f6f9da1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,8 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + Bitlist, Bitvector, uint, ) # SSZ Serialization @@ -13,7 +14,7 @@ BYTES_PER_LENGTH_OFFSET = 4 def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bool): + elif isinstance(value, boolean): if value: return b'\x01' else: @@ -39,6 +40,12 @@ def is_empty(obj: SSZValue): def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) + elif isinstance(obj, Bitvector): + as_integer = sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((len(obj) + 7) // 8, "little") + elif isinstance(obj, Bitlist): + as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") elif isinstance(obj, Series): return encode_series(obj) else: @@ -85,6 +92,12 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values + elif isinstance(values, Bitvector): + as_integer = sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitlist): + as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) @@ -134,7 +147,7 @@ def hash_tree_root(obj: SSZValue): else: raise Exception(f"Type not supported: {type(obj)}") - if isinstance(obj, (List, Bytes)): + if isinstance(obj, (List, Bytes, Bitlist)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 58e66ca68..ea07359b2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bool(BasicValue): # can't subclass bool. +class boolean(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) @@ -48,7 +48,7 @@ class Bool(BasicValue): # can't subclass bool. # Alias for Bool -class Bit(Bool): +class bit(boolean): pass @@ -233,7 +233,7 @@ class ParamsMeta(SSZType): return f"{self.__name__}~{self.__class__.__name__}" def __repr__(self): - return self, self.__class__ + return f"{self.__name__}~{self.__class__.__name__}" def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. @@ -280,11 +280,12 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int +class BitElementsType(ElementsType): + elem_type = boolean class Elements(ParamsBase, metaclass=ElementsType): pass - class BaseList(list, Elements): def __init__(self, *args): @@ -310,6 +311,10 @@ class BaseList(list, Elements): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __repr__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __getitem__(self, k) -> SSZValue: if isinstance(k, int): # check if we are just doing a lookup, and not slicing if k < 0: @@ -337,6 +342,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] +class BaseBitfield(BaseList, metaclass=BitElementsType): + elem_type = bool + +class Bitlist(BaseBitfield): + pass + +class Bitvector(BaseBitfield): + pass + class List(BaseList): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 82fb4ec68..33badcf4a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, BytesN, + bit, boolean, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -74,10 +74,10 @@ def merge(a: str, branch: Iterable[str]) -> str: test_data = [ - ("bit F", Bit(False), "00", chunk("00")), - ("bit T", Bit(True), "01", chunk("01")), - ("bool F", Bool(False), "00", chunk("00")), - ("bool T", Bool(True), "01", chunk("01")), + ("bit F", bit(False), "00", chunk("00")), + ("bit T", bit(True), "01", chunk("01")), + ("boolean F", boolean(False), "00", chunk("00")), + ("boolean T", boolean(True), "01", chunk("01")), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 2af742360..f746a29c9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,6 +1,6 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + Elements, bit, boolean, Container, List, Vector, Bytes, BytesN, byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bool, BasicValue) - assert isinstance(Bool, BasicType) + assert issubclass(boolean, BasicValue) + assert isinstance(boolean, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -45,16 +45,16 @@ def test_basic_instances(): assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bool(True), BasicValue) - assert isinstance(Bool(False), BasicValue) - assert isinstance(Bit(True), Bool) - assert isinstance(Bit(False), Bool) + assert isinstance(boolean(True), BasicValue) + assert isinstance(boolean(False), BasicValue) + assert isinstance(bit(True), boolean) + assert isinstance(bit(False), boolean) def test_basic_value_bounds(): max = { - Bool: 2 ** 1, - Bit: 2 ** 1, + boolean: 2 ** 1, + bit: 2 ** 1, uint8: 2 ** (8 * 1), byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), From 23c743570e5b42119f7e7fdb9ffcf8584e98c1d1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:26:45 +0100 Subject: [PATCH 24/53] Add some tests and fix pack --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a7f6f9da1..d6689892d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -96,7 +96,7 @@ def pack(values: Series): as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") elif isinstance(values, Bitlist): - as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 33badcf4a..1522ce200 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -2,6 +2,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, BytesN, + Bitlist, Bitvector, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -78,6 +79,10 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 494984f7d3129d8d3c4798d285824b59dc7f68b2 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:42:14 +0100 Subject: [PATCH 25/53] Fix linting errors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 ++++++ test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d6689892d..7298fb3ca 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -26,7 +26,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bool): + elif issubclass(typ, boolean): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea07359b2..047abd4fe 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -280,12 +280,15 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int + class BitElementsType(ElementsType): elem_type = boolean + class Elements(ParamsBase, metaclass=ElementsType): pass + class BaseList(list, Elements): def __init__(self, *args): @@ -342,12 +345,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] + class BaseBitfield(BaseList, metaclass=BitElementsType): elem_type = bool + class Bitlist(BaseBitfield): pass + class Bitvector(BaseBitfield): pass diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 1522ce200..63f0c835d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,10 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), - ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), - ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), - ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), - ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From d641e941519efc8cacd5e2ddabcaaf1722aa0797 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 11:30:23 +0100 Subject: [PATCH 26/53] Cleanups --- specs/simple-serialize.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 53c5649ed..97b1d560c 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -43,7 +43,7 @@ ### Composite types * **container**: ordered heterogeneous collection of values - * python dataclass notation with key-type pairs, e.g. + * python dataclass notation with key-type pairs, e.g. ```python class ContainerExample(Container): foo: uint64 @@ -53,12 +53,12 @@ class ContainerExample(Container): * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` * **list**: ordered variable-length homogeneous collection of values, with maximum length `N` * notation `List[type, N]`, e.g. `List[uint64, N]` +* **bitvector**: ordered fixed-length collection of `boolean` values + * notation `Bitvector[N]` +* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` + * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` -* **Bitvector**: a fixed-length list of `boolean` values - * notation `Bitvector[N]` -* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` - * notation `Bitlist[N]` ### Variable-size and fixed-size @@ -193,7 +193,6 @@ as_integer = sum([value[i] << i for i in range(len(value))]) return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) ``` - ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. From 67c50cb197824a015ff9946fb6fd5c684e9e9639 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 11:45:35 +0100 Subject: [PATCH 27/53] Changed attestation and custody bitfields --- scripts/build_spec.py | 6 +-- specs/core/0_beacon-chain.md | 41 +++---------------- .../pyspec/eth2spec/debug/random_value.py | 6 +-- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 11 +++++ .../eth2spec/test/helpers/attestations.py | 9 ++-- .../pyspec/eth2spec/test/helpers/bitfields.py | 11 ----- .../test_process_attestation.py | 8 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 +++- 8 files changed, 38 insertions(+), 62 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b1512111..0a41fe5c0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,7 +26,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -53,7 +53,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -175,7 +175,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' - 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc ] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b9aeee4b..13ab4cdae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -80,8 +80,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - - [`get_bitfield_bit`](#get_bitfield_bit) - - [`verify_bitfield`](#verify_bitfield) - [`convert_to_indexed`](#convert_to_indexed) - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) @@ -354,7 +352,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -421,9 +419,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -865,13 +863,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Sequence[ValidatorIndex]: + bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) - assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return sorted([index for i, index in enumerate(committee) if bitfield[i]]) ``` ### `int_to_bytes` @@ -912,34 +909,6 @@ def get_domain(state: BeaconState, return bls_domain(domain_type, fork_version) ``` -### `get_bitfield_bit` - -```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -``` - -### `verify_bitfield` - -```python -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -``` - ### `convert_to_indexed` ```python diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index cdcba343a..95a3ae970 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -3,7 +3,7 @@ from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, - Vector, BytesN + Vector, BytesN, Bitlist, Bitvector ) # in bytes @@ -83,12 +83,12 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif issubclass(typ, Vector): + elif issubclass(typ, Vector) or issubclass(typ, Bitvector): return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ) - elif issubclass(typ, List): + elif issubclass(typ, List) or issubclass(typ, Bitlist): length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 130956235..ccca17385 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -18,7 +18,14 @@ def translate_typ(typ) -> ssz.BaseSedes: elif issubclass(typ, spec_ssz.Vector): return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): + # TODO: Make py-ssz List support the new fixed length list return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitlist): + # TODO: Once Bitlist implemented in py-ssz, use appropriate type + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitvector): + # TODO: Once Bitvector implemented in py-ssz, use appropriate type + return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): @@ -68,6 +75,10 @@ def translate_value(value, typ): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.Bitlist): + return typ(value) + elif issubclass(typ, spec_ssz.Bitvector): + return typ(value) elif issubclass(typ, spec_ssz.BytesN): return typ(value) elif issubclass(typ, spec_ssz.Bytes): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7eb..2e637d42f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -1,10 +1,10 @@ from typing import List -from eth2spec.test.helpers.bitfields import set_bitfield_bit from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import Bitlist def build_attestation_data(spec, state, slot, shard): @@ -69,9 +69,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\x00' * bitfield_length - custody_bitfield = b'\x00' * bitfield_length + aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( aggregation_bitfield=aggregation_bitfield, data=attestation_data, @@ -138,7 +137,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + attestation.aggregation_bitfield[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py deleted file mode 100644 index 50e5b6cba..000000000 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ /dev/null @@ -1,11 +0,0 @@ -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..ee76ab23d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -10,6 +10,7 @@ from eth2spec.test.helpers.state import ( next_slot, ) from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_attestation_processing(spec, state, attestation, valid=True): @@ -281,7 +282,10 @@ def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + custody_bitfield = deepcopy(attestation.aggregation_bitfield) + custody_bitfield.append(False) + + attestation.custody_bitfield = custody_bitfield sign_attestation(spec, state, attestation) @@ -307,7 +311,7 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 047abd4fe..d0aef1b44 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -351,11 +351,15 @@ class BaseBitfield(BaseList, metaclass=BitElementsType): class Bitlist(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return False class Bitvector(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return True class List(BaseList): From becb7a032ab4ef1cbc2fe0e3d12ae381250f6175 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:14:36 +0100 Subject: [PATCH 28/53] justification_bitfield -> Bitvector[4] --- specs/core/0_beacon-chain.md | 16 ++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 13ab4cdae..c1f1a7bcf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -523,7 +523,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: uint64 # Bit set for every recent justified epoch + justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,38 +1291,38 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 + state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 1) + state.justification_bitfield[1] = True current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 0) + state.justification_bitfield[0] = True # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if all(bitfield[1:4]) and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if all(bitfield[1:3]) and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if all(bitfield[0:3]) and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if all(bitfield[0:2]) and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d0aef1b44..6ce2b1538 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -355,12 +355,19 @@ class Bitlist(BaseBitfield): def is_fixed_size(cls): return False + @classmethod + def default(cls): + return cls() + class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + + @classmethod + def default(cls): + return cls(0 for _ in range(cls.length)) class List(BaseList): From 80c680e6147ffc3971856802b4e54d2b776c6862 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:41:22 +0100 Subject: [PATCH 29/53] Phase 1 to Bitvector/Bitlist --- specs/core/1_custody-game.md | 12 +++++++++++- specs/core/1_shard-data-chains.md | 5 ++--- specs/light_client/sync_protocol.md | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 07f6ec698..17f599fcd 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,6 +272,16 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` +### `get_bitfield_bit` + +```python +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +``` + ### `get_custody_chunk_bit` ```python @@ -566,7 +576,7 @@ def process_bit_challenge(state: BeaconState, chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d1c86eca2..b2a5ea9ea 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bytes[PLACEHOLDER] + aggregation_bitfield: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -230,10 +230,9 @@ def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data persistent_committee = get_persistent_committee(state, data.shard, data.slot) - assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: + if attestation.aggregation_bitfield[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf5608..a1b5777cf 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': 'bytes', + 'shard_bitfield': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if get_bitfield_bit(proof.shard_bitfield, index) is True + if proof.shard_bitfield[index] ]) assert bls_verify( pubkey=group_public_key, From f57387cc83f16e2ec6b19ee544964254786fa7fa Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 13:09:48 +0100 Subject: [PATCH 30/53] Justification bitvector length to constant --- specs/core/0_beacon-chain.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c1f1a7bcf..4d3fe9773 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,6 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -523,7 +524,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch + justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,21 +1292,21 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) + state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[1] = True + state.justification_bitfield[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[0] = True + state.justification_bitfield[0] = 0b1 # Process finalizations bitfield = state.justification_bitfield From a5154da1ff76e8b768e64987471820f958c391d1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 15:40:40 +0200 Subject: [PATCH 31/53] suggestion to implement bitfield like --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 7298fb3ca..f0ee944bd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,6 +128,8 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 + elif issubclass(typ, Bitfield): + return (typ.length + 7) // 8 // 32 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..53ab42743 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -281,10 +281,6 @@ class ElementsType(ParamsMeta): length: int -class BitElementsType(ElementsType): - elem_type = boolean - - class Elements(ParamsBase, metaclass=ElementsType): pass @@ -346,11 +342,16 @@ class BaseList(list, Elements): return self[len(self) - 1] -class BaseBitfield(BaseList, metaclass=BitElementsType): - elem_type = bool +class BitElementsType(ElementsType): + elem_type: SSZType = boolean + length: int -class Bitlist(BaseBitfield): +class Bitfield(BaseList, metaclass=BitElementsType): + pass + + +class Bitlist(Bitfield): @classmethod def is_fixed_size(cls): return False @@ -360,15 +361,29 @@ class Bitlist(BaseBitfield): return cls() -class Bitvector(BaseBitfield): +class Bitvector(Bitfield): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) + + @classmethod + def value_check(cls, value): + # check length limit strictly + return len(value) == cls.length and super().value_check(value) + @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From b574a581094b8a00b360de573bee6a949b44650a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 14:45:17 +0100 Subject: [PATCH 32/53] Remove not working py-ssz decoder tests --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 26ee6e913..b362d503b 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,9 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: + # TODO: Once has Bitfields and Bitvectors, add back + # spec.BeaconState and spec.BeaconBlock + for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, From 8ed638bb84ae3dd88e6658f7bfd5eeda874b97f9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 15:21:04 +0100 Subject: [PATCH 33/53] Linter fixes --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/phase_0/block_processing/test_process_attestation.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index b362d503b..c707c840a 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitfields and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index ee76ab23d..d17d93e6d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -311,7 +311,8 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..85f5e0bdb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,11 +364,12 @@ class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From afd86f71de09e556a7a4cc5d82a38b8d514e0e93 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:31:33 +0100 Subject: [PATCH 34/53] Fixes in ssz impl --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index f0ee944bd..53075845b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -129,7 +129,7 @@ def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 elif issubclass(typ, Bitfield): - return (typ.length + 7) // 8 // 32 + return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): From 93ce1688629f45f92f7a261a958e99351299606a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:47:48 +0100 Subject: [PATCH 35/53] More linting fixes --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 3 ++- specs/core/1_custody-game.md | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0a41fe5c0..1f5fe1ee6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, uint64, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d3fe9773..f596da520 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,7 +1292,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( + *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 17f599fcd..9ad00516e 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -574,7 +574,6 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) - assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) From 7adf07ea5f1fca3696cc52cfa6e7127aa7e80883 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 22:58:44 +0100 Subject: [PATCH 36/53] A few more tests for Bitvector/Bitlist --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 63f0c835d..88ccc838c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -85,6 +85,16 @@ test_data = [ "c5c2", chunk("c5c2")), ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("long bitvector", Bitvector[512](1 for i in range(512)), + "ff" * 64, h("ff" * 32, "ff" * 32)), + ("long bitlist", Bitlist[512](1), + "03", h(h(chunk("01"), chunk("")), chunk("01"))), + ("long bitlist", Bitlist[512](1 for i in range(512)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), chunk("0002"))), + ("odd bitvector", Bitvector[513](1 for i in range(513)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk("")))), + ("odd bitlist", Bitlist[513](1 for i in range(513)), + "ff" * 64 + "03", h(h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk(""))), chunk("0102"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 237b41df5b41a37972c66c51d5acd551134189b5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:18:54 +0100 Subject: [PATCH 37/53] Slice notation for justification_bitfield shift --- specs/core/0_beacon-chain.md | 5 +++-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f596da520..0ea24a255 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,8 +1292,9 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( - *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] + state.justification_bitfield[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 53ab42743..ba773b443 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -324,12 +324,18 @@ class BaseList(list, Elements): return super().__getitem__(k) def __setitem__(self, k, v): - if k < 0: - raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self): - raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self)})") - super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + if type(k) == slice: + if k.start < 0 or k.stop > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds slice {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) + else: + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) From 2677d233a830b07a7f6f60392a1a54b888d3ac60 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:31:37 +0100 Subject: [PATCH 38/53] Some more (shorter) Bitvector and Bitlist tests --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 88ccc838c..637d9c5c4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,6 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TTFTFTFF", Bitvector[8](1, 1, 0, 1, 0, 1, 0, 0), "2b", chunk("2b")), + ("bitlist TTFTFTFF", Bitlist[8](1, 1, 0, 1, 0, 1, 0, 0), "2b01", h(chunk("2b"), chunk("08"))), + ("bitvector FTFT", Bitvector[4](0, 1, 0, 1), "0a", chunk("0a")), + ("bitlist FTFT", Bitlist[4](0, 1, 0, 1), "1a", h(chunk("0a"), chunk("04"))), + ("bitvector FTF", Bitvector[3](0, 1, 0), "02", chunk("02")), + ("bitlist FTF", Bitlist[3](0, 1, 0), "0a", h(chunk("02"), chunk("03"))), ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), From 196ac4202595a716a7acba91e7a437d80cc1a4ad Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:23:22 +0100 Subject: [PATCH 39/53] Cleanup naming --- specs/core/0_beacon-chain.md | 49 +++++++++---------- specs/core/1_custody-game.md | 26 +++++----- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 20 ++++---- specs/validator/beacon_node_oapi.yaml | 8 +-- .../pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/fork_choice/test_on_attestation.py | 2 +- .../eth2spec/test/helpers/attestations.py | 12 ++--- .../test_process_attestation.py | 18 +++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +-- 12 files changed, 79 insertions(+), 80 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9ce7976b5..d41173424 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -190,7 +190,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | +| `JUSTIFICATION_BITS_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -360,7 +360,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -427,9 +427,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + custody_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -527,7 +527,7 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Finality - justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint @@ -867,12 +867,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, data: AttestationData, - bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: + bits: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bits``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) - return sorted([index for i, index in enumerate(committee) if bitfield[i]]) + return sorted([index for i, index in enumerate(committee) if bits[i]]) ``` ### `int_to_bytes` @@ -920,8 +920,8 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ Convert ``attestation`` to (almost) indexed-verifiable form. """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) assert set(custody_bit_1_indices).issubset(attesting_indices) custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] @@ -1146,7 +1146,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1254,7 +1254,7 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: - output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` @@ -1294,36 +1294,35 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ - state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] - state.justification_bitfield[0] = 0b0 + state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) - state.justification_bitfield[1] = 0b1 + state.justification_bits[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) - state.justification_bitfield[0] = 0b1 + state.justification_bits[0] = 0b1 # Process finalizations - bitfield = state.justification_bitfield + bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if all(bitfield[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if all(bitfield[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if all(bitfield[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if all(bitfield[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` @@ -1379,7 +1378,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations - if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) + if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward @@ -1661,7 +1660,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=data, - aggregation_bitfield=attestation.aggregation_bitfield, + aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), ) @@ -1680,7 +1679,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] - + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9ad00516e..883476c58 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,14 +272,14 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` -### `get_bitfield_bit` +### `get_bit` ```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: +def get_bit(serialization: bytes, i: int) -> int: """ - Extract the bit in ``bitfield`` at position ``i``. + Extract the bit in ``serialization`` at position ``i``. """ - return (bitfield[i // 8] >> (i % 8)) % 2 + return (serialization[i // 8] >> (i % 8)) % 2 ``` ### `get_custody_chunk_bit` @@ -287,17 +287,17 @@ def get_bitfield_bit(bitfield: bytes, i: int) -> int: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return bool(get_bitfield_bit(hash(key + chunk), 0)) + return bool(get_bit(hash(key + chunk), 0)) ``` ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: +def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: aggregated_bits = bytearray([0] * 32) - for i in range(0, len(chunk_bitfield), 32): + for i in range(0, len(chunk_bits), 32): for j in range(32): - aggregated_bits[j] ^= chunk_bitfield[i + j] + aggregated_bits[j] ^= chunk_bits[i + j] return hash(aggregated_bits) ``` @@ -489,7 +489,7 @@ def process_chunk_challenge(state: BeaconState, responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) assert challenge.responder_index in attesters # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: @@ -546,7 +546,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert challenge.responder_index in attesters # A validator can be the challenger for at most one challenge at a time @@ -575,8 +575,8 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] - assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) + custody_bit = attestation.custody_bits[attesters.index(challenge.responder_index)] + assert custody_bit != get_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, @@ -670,7 +670,7 @@ def process_bit_challenge_response(state: BeaconState, ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b2a5ea9ea..432820b7c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bitlist[PLACEHOLDER] + aggregation_bits: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -232,7 +232,7 @@ def verify_shard_attestation_signature(state: BeaconState, persistent_committee = get_persistent_committee(state, data.shard, data.slot) pubkeys = [] for i, index in enumerate(persistent_committee): - if attestation.aggregation_bitfield[i]: + if attestation.aggregation_bits[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a1b5777cf..3f5a37289 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': Bitlist[PLACEHOLDER], + 'shard_bits': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bits[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if proof.shard_bitfield[index] + if proof.shard_bits[index] ]) assert bls_verify( pubkey=group_public_key, @@ -196,4 +196,4 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bits) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66..813b8bcfe 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -44,8 +44,8 @@ - [Crosslink vote](#crosslink-vote) - [Construct attestation](#construct-attestation) - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) + - [Aggregation bits](#aggregation-bits) + - [Custody bits](#custody-bits) - [Aggregate signature](#aggregate-signature) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) @@ -322,19 +322,19 @@ Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-c Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). -##### Aggregation bitfield +##### Aggregation bits -* Let `aggregation_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. * Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bitfield[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bitfield = aggregation_bitfield`. +* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. +* Set `attestation.aggregation_bits = aggregation_bits`. -*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. -##### Custody bitfield +##### Custody bits -* Let `custody_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bitfield = custody_bitfield`. +* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Set `attestation.custody_bits = custody_bits`. *Note*: This is a stub for Phase 0. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 74be21fac..4da8f7933 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -415,16 +415,16 @@ components: type: object description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: - aggregation_bitfield: + aggregation_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Attester aggregation bitfield." - custody_bitfield: + description: "Attester aggregation bits." + custody_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield." + description: "Custody bits." signature: type: string format: byte diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index c707c840a..ea1f1d47f 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitlists and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 1a67e01d5..5adb022a6 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -118,5 +118,5 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid - attestation.custody_bitfield[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] + attestation.custody_bits[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index a184a5f5e..6fdf1538b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -67,12 +67,12 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) - custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=attestation_data, - custody_bitfield=custody_bitfield, + custody_bits=custody_bits, ) fill_aggregate_attestation(spec, state, attestation) if signed: @@ -105,7 +105,7 @@ def sign_attestation(spec, state, attestation): participants = spec.get_attesting_indices( state, attestation.data, - attestation.aggregation_bitfield, + attestation.aggregation_bits, ) attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) @@ -135,7 +135,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield[i] = True + attestation.aggregation_bits[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 1ea7c92cf..916724001 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -275,14 +275,14 @@ def test_bad_crosslink_end_epoch(spec, state): @with_all_phases @spec_state_test -def test_inconsistent_bitfields(spec, state): +def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bitfield = deepcopy(attestation.aggregation_bitfield) - custody_bitfield.append(False) + custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits.append(False) - attestation.custody_bitfield = custody_bitfield + attestation.custody_bits = custody_bits sign_attestation(spec, state, attestation) @@ -291,11 +291,11 @@ def test_inconsistent_bitfields(spec, state): @with_phases(['phase0']) @spec_state_test -def test_non_empty_custody_bitfield(spec, state): +def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + attestation.custody_bits = deepcopy(attestation.aggregation_bits) sign_attestation(spec, state, attestation) @@ -304,12 +304,12 @@ def test_non_empty_custody_bitfield(spec, state): @with_all_phases @spec_state_test -def test_empty_aggregation_bitfield(spec, state): +def test_empty_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( - *([0b0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bits))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 53075845b..d5855a755 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,7 +128,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 - elif issubclass(typ, Bitfield): + elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ba773b443..91d3ff4f2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -353,11 +353,11 @@ class BitElementsType(ElementsType): length: int -class Bitfield(BaseList, metaclass=BitElementsType): +class Bits(BaseList, metaclass=BitElementsType): pass -class Bitlist(Bitfield): +class Bitlist(Bits): @classmethod def is_fixed_size(cls): return False @@ -367,7 +367,7 @@ class Bitlist(Bitfield): return cls() -class Bitvector(Bitfield): +class Bitvector(Bits): @classmethod def extract_args(cls, *args): From 6f9d37485d624cb96188714db495cdb07ace9aa6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:34:01 +0100 Subject: [PATCH 40/53] Cleanups --- specs/core/0_beacon-chain.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d41173424..9de024909 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,20 +1294,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] - state.justification_bits[0] = 0b0 - previous_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, previous_epoch) - ) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, - root=get_block_root(state, previous_epoch)) + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - current_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, current_epoch) - ) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From e36593b155d3fdffdb8ebe451236d5dfb54e22c9 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:35:50 +0100 Subject: [PATCH 41/53] Add comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9de024909..96b8981fe 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) From c764202a5768003bfaf354e90e591f66c48c9679 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 28 Jun 2019 21:35:26 +0800 Subject: [PATCH 42/53] Slashing penalty calculation change (#1217) If the exit queue is very long, then a validator may take many months to exit. With the code as currently written, however, self-slashing is a potentially lucrative route to get one's money out faster, because one can exit in 36 days. This PR changes it so that slashing can only extend your withdrawal time, not contract it. Also, instead of the slashed balances used to calculate one's slashing penalty being those in `[withdrawal - 54 days ... withdrawal - 18 days]`, we now run the penalization algorithm once every 36 days that a validator is slashed but not withdrawn, so that it covers the 36-day period where the validator was actually slashed. It also moves the minimum slashing penalty to the `slash_validator` function so that it is only applied once. We also simplify the `slashed_balances` logic to be per-epoch. --- configs/constant_presets/mainnet.yaml | 4 +- configs/constant_presets/minimal.yaml | 4 +- specs/core/0_beacon-chain.md | 46 ++++------- specs/core/1_custody-game.md | 2 +- .../test_process_attester_slashing.py | 82 ++++++++++++++++--- 5 files changed, 92 insertions(+), 46 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 9f7ca950f..38f10c8fc 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -76,7 +76,7 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # 2**16 (= 65,536) epochs ~0.8 years EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +EPOCHS_PER_SLASHINGS_VECTOR: 8192 # 2**24 (= 16,777,216) historical roots, ~26,131 years HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -88,7 +88,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 3e3f7ccb4..3aa4a6b71 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -77,7 +77,7 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # [customized] smaller state EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state -EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +EPOCHS_PER_SLASHINGS_VECTOR: 64 # 2**24 (= 16,777,216) historical roots HISTORICAL_ROOTS_LIMIT: 16777216 # 2**40 (= 1,099,511,627,776) validator spots @@ -89,7 +89,7 @@ VALIDATOR_REGISTRY_LIMIT: 1099511627776 # 2**5 (= 32) BASE_REWARD_FACTOR: 32 # 2**9 (= 512) -WHISTLEBLOWING_REWARD_QUOTIENT: 512 +WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 # 2**25 (= 33,554,432) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1b7cda0a4..61ef8b742 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -234,7 +234,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | -| `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | | `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | @@ -243,7 +243,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `BASE_REWARD_FACTOR` | `2**6` (= 64) | -| `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | +| `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | @@ -520,7 +520,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1097,21 +1097,22 @@ def slash_validator(state: BeaconState, """ Slash the validator with index ``slashed_index``. """ - current_epoch = get_current_epoch(state) + epoch = get_current_epoch(state) initiate_validator_exit(state, slashed_index) - state.validators[slashed_index].slashed = True - state.validators[slashed_index].withdrawable_epoch = Epoch(current_epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR) - slashed_balance = state.validators[slashed_index].effective_balance - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] += slashed_balance + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + # Apply proposer and whistleblower rewards proposer_index = get_beacon_proposer_index(state) if whistleblower_index is None: whistleblower_index = proposer_index - whistleblowing_reward = Gwei(slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT) - proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) - decrease_balance(state, slashed_index, whistleblowing_reward) + increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) ``` ## Genesis @@ -1174,7 +1175,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1493,18 +1494,9 @@ def process_registry_updates(state: BeaconState) -> None: def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) - - # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_at_end = state.slashed_balances[epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_penalties = total_at_end - total_at_start - for index, validator in enumerate(state.validators): - if validator.slashed and epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 == validator.withdrawable_epoch: - penalty = max( - validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, - validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT - ) + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + penalty = validator.effective_balance * min(sum(state.slashings) * 3, total_balance) // total_balance decrease_balance(state, ValidatorIndex(index), penalty) ``` @@ -1532,10 +1524,8 @@ def process_final_updates(state: BeaconState) -> None: get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) - # Set total slashed balances - state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( - state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - ) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) # Set historical root accumulator diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 47d1578c5..c29033fe3 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -453,7 +453,7 @@ def process_early_derived_secret_reveal(state: BeaconState, # Apply penalty proposer_index = get_beacon_proposer_index(state) whistleblower_index = reveal.masker_index - whistleblowing_reward = Gwei(penalty // WHISTLEBLOWING_REWARD_QUOTIENT) + whistleblowing_reward = Gwei(penalty // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblowing_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index e2b50ea0b..e78e1a866 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -25,31 +25,56 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', None return - slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - pre_slashed_balance = get_balance(state, slashed_index) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) + pre_slashings = {slashed_index: get_balance(state, slashed_index) for slashed_index in slashed_indices} + pre_withdrawalable_epochs = { + slashed_index: state.validators[slashed_index].withdrawable_epoch + for slashed_index in slashed_indices + } + + total_proposer_rewards = sum( + balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT + for balance in pre_slashings.values() + ) # Process slashing spec.process_attester_slashing(state, attester_slashing) - slashed_validator = state.validators[slashed_index] + for slashed_index in slashed_indices: + pre_withdrawalable_epoch = pre_withdrawalable_epochs[slashed_index] + slashed_validator = state.validators[slashed_index] - # Check slashing - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # Check slashing + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + if pre_withdrawalable_epoch < spec.FAR_FUTURE_EPOCH: + expected_withdrawable_epoch = max( + pre_withdrawalable_epoch, + spec.get_current_epoch(state) + spec.EPOCHS_PER_SLASHINGS_VECTOR + ) + assert slashed_validator.withdrawable_epoch == expected_withdrawable_epoch + else: + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + assert get_balance(state, slashed_index) < pre_slashings[slashed_index] - if slashed_index != proposer_index: - # lost whistleblower reward - assert get_balance(state, slashed_index) < pre_slashed_balance + if proposer_index not in slashed_indices: # gained whistleblower reward - assert get_balance(state, proposer_index) > pre_proposer_balance + assert get_balance(state, proposer_index) == pre_proposer_balance + total_proposer_rewards else: # gained rewards for all slashings, which may include others. And only lost that of themselves. - # Netto at least 0, if more people where slashed, a balance increase. - assert get_balance(state, slashed_index) >= pre_slashed_balance + expected_balance = ( + pre_proposer_balance + + total_proposer_rewards + - pre_slashings[proposer_index] // spec.MIN_SLASHING_PENALTY_QUOTIENT + ) + + assert get_balance(state, proposer_index) == expected_balance yield 'post', state @@ -82,6 +107,37 @@ def test_success_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_recent(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + +@with_all_phases +@always_bls +@spec_state_test +def test_success_already_exited_long_ago(spec, state): + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + slashed_indices = ( + attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices + ) + for index in slashed_indices: + spec.initiate_validator_exit(state, index) + state.validators[index].withdrawable_epoch = spec.get_current_epoch(state) + 2 + + yield from run_attester_slashing_processing(spec, state, attester_slashing) + + @with_all_phases @always_bls @spec_state_test From 2739767a7147f6b2efd6537189f3f3437b885058 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 14:43:44 +0100 Subject: [PATCH 43/53] Hardened Eth 1.0 voting strategy (#1218) --- specs/validator/0_beacon-chain-validator.md | 31 +++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66..44b5e2a3c 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -221,19 +221,26 @@ epoch_signature = bls_sign( ##### Eth1 Data -`block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. +The `block.eth1_data` field is for block proposers to vote on recent Eth 1.0 data. This recent data contains an Eth 1.0 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth 1.0 block. If over half of the block proposers in the current Eth 1.0 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. -* Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.eth1_data.block_hash`. - * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. - * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. -* If `D` is empty: - * Let `block_hash` be the block hash of the `ETH1_FOLLOW_DISTANCE`'th ancestor of the head of the canonical Eth 1.0 chain. - * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` - * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. -* If `D` is nonempty: - * Let `best_vote_data` be the `eth1_data` member of `D` that has the highest vote count (`D.count(eth1_data)`), breaking ties by favoring block hashes with higher associated block height. -* Set `block.eth1_data = best_vote_data`. +Let `get_eth1_data(distance: int) -> Eth1Data` be the (subjective) function that returns the Eth 1.0 data at distance `distance` relative to the Eth 1.0 head at the start of the current Eth 1.0 voting period. Let `previous_eth1_distance` be the distance relative to the Eth 1.0 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth 1.0 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: + +```python +def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: + new_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, 2 * ETH1_FOLLOW_DISTANCE)] + all_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, previous_eth1_distance)] + + valid_votes = [] + for slot, vote in enumerate(state.eth1_data_votes): + period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_square_root(SLOTS_PER_ETH1_VOTING_PERIOD) + if vote in new_eth1_data or (period_tail and vote in all_eth1_data): + valid_votes.append(vote) + + return max(valid_votes, + key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)), # Tiebreak by smallest distance + default=get_eth1_data(ETH1_FOLLOW_DISTANCE), + ) +``` ##### Signature From 05842f83718644184ce997277e7817bada07d568 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 15:26:02 +0100 Subject: [PATCH 44/53] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 96b8981fe..b9c3f5f0c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,14 +1294,15 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift + state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5ff13dd81a95d7852c2afa65dd3851d5642c17dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:07:36 +0200 Subject: [PATCH 45/53] be explicit about limiting for HTR and chunk padding --- specs/simple-serialize.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 97b1d560c..3fba59fba 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -49,13 +49,13 @@ class ContainerExample(Container): foo: uint64 bar: boolean ``` -* **vector**: ordered fixed-length homogeneous collection of values +* **vector**: ordered fixed-length homogeneous collection, with `N` values * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` -* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` +* **list**: ordered variable-length homogeneous collection, limited to `N` values * notation `List[type, N]`, e.g. `List[uint64, N]` -* **bitvector**: ordered fixed-length collection of `boolean` values +* **bitvector**: ordered fixed-length collection of `boolean` values, with `N` bits * notation `Bitvector[N]` -* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` +* **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` @@ -161,22 +161,31 @@ return serialized_type_index + serialized_bytes Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). +Note that deserialization requires hardening against invalid inputs. A non-exhaustive list: +- Offsets: out of order, out of range, mismatching minimum element size +- Scope: Extra unused bytes, not aligned with element size. +- More elements than a list limit allows. Part of enforcing consensus. + ## Merkleization We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. -* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) +* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + There merkleization depends on the input length: + - `0` chunks: A chunk filled with zeroes. + - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - `1+` chunks: pad to the next power of 2, merkleize as binary tree. + - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From dcb0244a4f99f50dfa82884a81bd14e1eabb316e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 10:19:59 -0500 Subject: [PATCH 46/53] get_attesting_indices set instead of sorted (#1225) --- specs/core/0_beacon-chain.md | 14 +++--- specs/core/1_custody-game.md | 94 ++++++++++++------------------------ 2 files changed, 39 insertions(+), 69 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 61ef8b742..abca72ff8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -866,13 +866,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ### `get_attesting_indices` ```python -def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Sequence[ValidatorIndex]: +def get_attesting_indices(state: BeaconState, data: AttestationData, bitfield: bytes) -> Set[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the set of attesting indices corresponding to ``data`` and ``bitfield``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return set(index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1) ``` ### `int_to_bytes` @@ -950,12 +950,12 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) - assert set(custody_bit_1_indices).issubset(attesting_indices) - custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] + assert custody_bit_1_indices.issubset(attesting_indices) + custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) return IndexedAttestation( - custody_bit_0_indices=custody_bit_0_indices, - custody_bit_1_indices=custody_bit_1_indices, + custody_bit_0_indices=sorted(custody_bit_0_indices), + custody_bit_1_indices=sorted(custody_bit_1_indices), data=attestation.data, signature=attestation.signature, ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index c29033fe3..9b17b39ed 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -36,7 +36,7 @@ - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) + - [`get_reveal_period`](#get_reveal_period) - [`replace_empty_or_append`](#replace_empty_or_append) - [Per-block processing](#per-block-processing) - [Operations](#operations) @@ -224,7 +224,7 @@ Add the following fields to the end of the specified container objects. Fields w class Validator(Container): # next_custody_reveal_period is initialised to the custody period # (of the particular validator) in which the validator is activated - # = get_validators_custody_reveal_period(...) + # = get_reveal_period(...) next_custody_reveal_period: uint64 max_reveal_lateness: uint64 ``` @@ -299,17 +299,12 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI return Epoch(next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING) ``` -### `get_validators_custody_reveal_period` +### `get_reveal_period` ```python -def get_validators_custody_reveal_period(state: BeaconState, - validator_index: ValidatorIndex, - epoch: Epoch=None) -> int: +def get_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: ''' - This function returns the reveal period for a given validator. - If no epoch is supplied, the current epoch is assumed. - Note: This function implicitly requires that validators are not removed from the - validator set in fewer than EPOCHS_PER_CUSTODY_PERIOD epochs + Return the reveal period for a given validator. ''' epoch = get_current_epoch(state) if epoch is None else epoch return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD @@ -340,17 +335,15 @@ Verify that `len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS`. For each `reveal` in `block.body.custody_key_reveals`, run the following function: ```python -def process_custody_key_reveal(state: BeaconState, - reveal: CustodyKeyReveal) -> None: +def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> None: """ Process ``CustodyKeyReveal`` operation. Note that this function mutates ``state``. """ - revealer = state.validators[reveal.revealer_index] epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_reveal_period, reveal.revealed_index) - assert revealer.next_custody_reveal_period < get_validators_custody_reveal_period(state, reveal.revealed_index) + assert revealer.next_custody_reveal_period < get_reveal_period(state, reveal.revealed_index) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -368,11 +361,11 @@ def process_custody_key_reveal(state: BeaconState, ) # Decrement max reveal lateness if response is timely - if revealer.next_custody_reveal_period == get_validators_custody_reveal_period(state, reveal.revealer_index) - 2: + if revealer.next_custody_reveal_period == get_reveal_period(state, reveal.revealer_index) - 2: revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT revealer.max_reveal_lateness = max( revealer.max_reveal_lateness, - get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period + get_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period ) # Process reveal @@ -394,13 +387,11 @@ Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_S For each `reveal` in `block.body.early_derived_secret_reveals`, run the following function: ```python -def process_early_derived_secret_reveal(state: BeaconState, - reveal: EarlyDerivedSecretReveal) -> None: +def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerivedSecretReveal) -> None: """ Process ``EarlyDerivedSecretReveal`` operation. Note that this function mutates ``state``. """ - revealed_validator = state.validators[reveal.revealed_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS @@ -470,8 +461,7 @@ Verify that `len(block.body.custody_chunk_challenges) <= MAX_CUSTODY_CHUNK_CHALL For each `challenge` in `block.body.custody_chunk_challenges`, run the following function: ```python -def process_chunk_challenge(state: BeaconState, - challenge: CustodyChunkChallenge) -> None: +def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge @@ -514,59 +504,41 @@ Verify that `len(block.body.custody_bit_challenges) <= MAX_CUSTODY_BIT_CHALLENGE For each `challenge` in `block.body.custody_bit_challenges`, run the following function: ```python -def process_bit_challenge(state: BeaconState, - challenge: CustodyBitChallenge) -> None: +def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: + attestation = challenge.attestation + epoch = slot_to_epoch(attestation.data.slot) + shard = attestation.data.crosslink.shard # Verify challenge signature challenger = state.validators[challenge.challenger_index] - assert bls_verify( - pubkey=challenger.pubkey, - message_hash=signing_root(challenge), - signature=challenge.signature, - domain=get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)), - ) + domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) + assert bls_verify(challenger.pubkey, signing_root(challenge), challenge.signature, domain) + # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) - - # Verify the attestation - attestation = challenge.attestation + # Verify attestation validate_indexed_attestation(state, convert_to_indexed(state, attestation)) - # Verify the attestation is eligible for challenging + # Verify attestation is eligible for challenging responder = state.validators[challenge.responder_index] - assert (slot_to_epoch(attestation.data.slot) + responder.max_reveal_lateness <= - get_validators_custody_reveal_period(state, challenge.responder_index)) - - # Verify the responder participated in the attestation + assert epoch + responder.max_reveal_lateness <= get_reveal_period(state, challenge.responder_index) + # Verify responder participated in the attestation attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - - # A validator can be the challenger for at most one challenge at a time + # Verifier challenger is not already challenging for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - - # Verify the responder is a valid custody key + # Verify the responder custody key epoch_to_sign = get_randao_epoch_for_custody_period( - get_validators_custody_reveal_period( - state, - challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot)), - challenge.responder_index + get_reveal_period(state, challenge.responder_index, epoch), + challenge.responder_index, ) - assert bls_verify( - pubkey=responder.pubkey, - message_hash=hash_tree_root(epoch_to_sign), - signature=challenge.responder_key, - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=epoch_to_sign, - ), - ) - + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + committee = get_crosslink_committee(state, epoch, shard) + custody_bit = get_bitfield_bit(attestation.custody_bitfield, committee.index(challenge.responder_index)) assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( @@ -581,7 +553,6 @@ def process_bit_challenge(state: BeaconState, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) state.custody_challenge_index += 1 - # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` @@ -593,8 +564,7 @@ Verify that `len(block.body.custody_responses) <= MAX_CUSTODY_RESPONSES`. For each `response` in `block.body.custody_responses`, run the following function: ```python -def process_custody_response(state: BeaconState, - response: CustodyResponse) -> None: +def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: @@ -682,7 +652,7 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validators): deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - if get_validators_custody_reveal_period(state, ValidatorIndex(index)) > deadline: + if get_reveal_period(state, ValidatorIndex(index)) > deadline: slash_validator(state, ValidatorIndex(index)) ``` From 128bbbc665c8eabe10640c019ccb0765c5be8f11 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:27:59 +0200 Subject: [PATCH 47/53] fix slicing, and support partial slice bounds --- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b9c3f5f0c..887ae6802 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 91d3ff4f2..d87a22399 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -325,7 +325,7 @@ class BaseList(list, Elements): def __setitem__(self, k, v): if type(k) == slice: - if k.start < 0 or k.stop > len(self): + if (k.start is not None and k.start < 0) or (k.stop is not None and k.stop > len(self)): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds slice {k} (to {v}, bound: {len(self)})") super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) From 25db39755047ce6ae28332d90080c502ce899423 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:34:31 +0200 Subject: [PATCH 48/53] fix line length lint problem in checkpoint --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 887ae6802..ba5b591d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1298,11 +1298,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From fa84c496592d722b544c775065b9258cd22dbd73 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:23:34 +0100 Subject: [PATCH 49/53] Update specs/core/0_beacon-chain.md Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cb50e2f86..6f28c0e3d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1295,7 +1295,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:-1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: From 6a2d2c84a8f3fa3ea846d17dff5d10ebfb3508e6 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 20:49:57 +0100 Subject: [PATCH 50/53] Bitlist for attestation doc --- specs/validator/0_beacon-chain-validator.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ad01476e9..7d5630c7b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -331,17 +331,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bits = aggregation_bits`. +* Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bits = custody_bits`. +* Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0. From 4dcb47e393ca68ab9e32af8ba913eaf829d6cb69 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:06 +0100 Subject: [PATCH 51/53] Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 916724001..b114c85e9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -279,7 +279,7 @@ def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits = attestation.aggregation_bits[:] custody_bits.append(False) attestation.custody_bits = custody_bits From be04eb2673563b6250cc5637777badc76913ec5f Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:16 +0100 Subject: [PATCH 52/53] Change copy style, and remove deepcopy import Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index b114c85e9..41207fdf4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, @@ -295,7 +293,7 @@ def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bits = deepcopy(attestation.aggregation_bits) + attestation.custody_bits = attestation.aggregation_bits[:] sign_attestation(spec, state, attestation) From 4f31207b7f2cd4ba2bf21976b6e8bc6644e24202 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 22:45:20 +0200 Subject: [PATCH 53/53] reword merkleize with limit / length --- specs/simple-serialize.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3fba59fba..7076b6410 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -171,21 +171,21 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. - There merkleization depends on the input length: - - `0` chunks: A chunk filled with zeroes. - - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. - - `1+` chunks: pad to the next power of 2, merkleize as binary tree. - - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). +* `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` +* `merkleize(data, pad_for)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + The merkleization depends on the effective input, which can be padded: if `pad_for=L`, then pad the `data` with zeroed chunks to `next_pow_of_two(L)` (virtually for memory efficiency). + Then, merkleize the chunks (empty input is padded to 1 zero chunk): + - If `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - If `> 1` chunks: pad to `next_pow_of_two(len(chunks))`, merkleize as binary tree. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). +* `mix_in_length(merkleize(pack(value), pad_for=(N * elem_size / BYTES_PER_CHUNK)), len(value))` if `value` is a list of basic objects. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_for=N), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]`