From c10e59bdf7b2fad73d967061dcdc0519ef1fa06f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 8 Apr 2020 10:32:16 +0800 Subject: [PATCH 001/127] Fix `INACTIVITY_PENALTY_QUOTIENT` The amount of inactivity penalty was adjusted to half since we were applying penalty for missing FFG target and source. But now we only apply it for missing target, so `INACTIVITY_PENALTY_QUOTIENT` should be `2**24`. --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/phase0/beacon-chain.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 6d71cfa47..3d2de75f0 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9daf428b4..b39a4fc01 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..c841d2dbf 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -242,10 +242,10 @@ The following values are (non-configurable) constants used throughout the specif | `BASE_REWARD_FACTOR` | `2**6` (= 64) | | `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | +| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12` epochs (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block From 890c27d091bb74947ba3fdd6b86464fddf9cc1ef Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:33:14 +0800 Subject: [PATCH 002/127] The input parameter `index` in `compute_shuffled_index` is the position of the given list, not `ValidatorIndex` --- specs/phase0/beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 23fa5ceee..8a51b5e8b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -722,9 +722,9 @@ def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint #### `compute_shuffled_index` ```python -def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Bytes32) -> ValidatorIndex: +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + Return the shuffled index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count @@ -732,14 +732,14 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Byt # See the 'generalized domain' algorithm on page 3 for current_round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index - return ValidatorIndex(index) + return index ``` #### `compute_proposer_index` @@ -753,11 +753,11 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] MAX_RANDOM_BYTE = 2**8 - 1 i = 0 while True: - candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] + candidate_index = indices[compute_shuffled_index(i % len(indices), len(indices), seed)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) + return candidate_index i += 1 ``` From b2f6325db339e630feb9a459099bc8f0c5f4ce49 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 Apr 2020 17:48:12 +0800 Subject: [PATCH 003/127] Fix `compute_committee` --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8a51b5e8b..82257947d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -773,7 +773,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[compute_shuffled_index(i, len(indices), seed)] for i in range(start, end)] ``` #### `compute_epoch_at_slot` From 6fdee7547502acc8f872031cc8d6f3afb502b83c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 17:27:57 +0800 Subject: [PATCH 004/127] Fix phase0 types --- setup.py | 10 +++++++--- specs/phase0/beacon-chain.md | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d1c62fb72..d78161b29 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ SSZObject = TypeVar('SSZObject', bound=View) PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional ) from dataclasses import ( @@ -146,8 +146,11 @@ _hash = hash hash_cache: Dict[bytes, Bytes32] = {} -def get_eth1_data(distance: uint64) -> Bytes32: - return hash(distance) +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data(block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore @@ -373,6 +376,7 @@ class PySpecCommand(Command): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md + specs/phase0/validator.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md specs/phase1/fraud-proofs.md diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 2ee1d46bc..5c84e41f7 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1125,7 +1125,7 @@ def slash_validator(state: BeaconState, whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` ## Genesis @@ -1229,7 +1229,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: # Process epoch on the start slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - state.slot += Slot(1) + state.slot = Slot(state.slot + 1) ``` ```python From cafd98b9e85550acc091c797f1deb1027b802b9a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:15:46 +0800 Subject: [PATCH 005/127] Fix utils.hash_function typing --- .../pyspec/eth2spec/utils/hash_function.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/hash_function.py b/tests/core/pyspec/eth2spec/utils/hash_function.py index 2c9b5a579..627f9b990 100644 --- a/tests/core/pyspec/eth2spec/utils/hash_function.py +++ b/tests/core/pyspec/eth2spec/utils/hash_function.py @@ -1,28 +1,17 @@ from hashlib import sha256 +from typing import Dict, Union ZERO_BYTES32 = b'\x00' * 32 -def _hash(x): +def _hash(x: Union[bytes, bytearray, memoryview]) -> bytes: return sha256(x).digest() -# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. -# Key = the hash input -# Value = the hash output -hash_cache = [] +hash_cache: Dict[bytes, bytes] = {} -def add_zero_hashes_to_cache(): - zerohashes = [(None, ZERO_BYTES32)] - for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) - hash_cache.extend(zerohashes[1:]) - - -def hash(x): - for (k, h) in hash_cache: - if x == k: - return h +def hash(x: bytes) -> bytes: + if x in hash_cache: + return hash_cache[x] return _hash(x) From 3575b18cd4b714427f876f646570a76df66f5f11 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 17 Apr 2020 18:18:22 +0800 Subject: [PATCH 006/127] Fix `config_util.py` typing --- tests/core/pyspec/eth2spec/config/config_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 4c5768a29..c43c1521b 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -24,12 +24,12 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = # Load presets from a file, and then prepares the global config setting. This does not apply the config. # To apply the config, reload the spec module (it will re-initialize with the config taken from here). -def prepare_config(configs_path, config_name): +def prepare_config(configs_path: str, config_name: str) -> None: global config config = load_config_file(configs_path, config_name) -def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: +def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: """ Loads the given preset :param presets_name: The name of the presets. (lowercase snake_case) @@ -38,7 +38,7 @@ def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: path = Path(join(configs_dir, presets_name + '.yaml')) yaml = YAML(typ='base') loaded = yaml.load(path) - out = dict() + out: Dict[str, Any] = dict() for k, v in loaded.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str From 5929aac799e8647b6905dd829948efcdfa5849c9 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 Apr 2020 17:56:27 +0100 Subject: [PATCH 007/127] Cosmetic changes from #1737 --- specs/phase0/beacon-chain.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 5c84e41f7..111655285 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -35,7 +35,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) - - [`SigningRoot`](#signingroot) + - [`SigningData`](#signingdata) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -191,7 +191,6 @@ The following values are (non-configurable) constants used throughout the specif | `HYSTERESIS_DOWNWARD_MULTIPLIER` | `1` | | `HYSTERESIS_UPWARD_MULTIPLIER` | `5` | - - For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -269,7 +268,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | | `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | - ## Containers The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. @@ -399,10 +397,10 @@ class BeaconBlockHeader(Container): body_root: Root ``` -#### `SigningRoot` +#### `SigningData` ```python -class SigningRoot(Container): +class SigningData(Container): object_root: Root domain: Domain ``` @@ -852,13 +850,12 @@ def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_ ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: """ - Return the signing root of an object by calculating the root of the object-domain tree. + Return the signing root for the corresponding signing data. """ - domain_wrapped_object = SigningRoot( + return hash_tree_root(SigningData( object_root=hash_tree_root(ssz_object), domain=domain, - ) - return hash_tree_root(domain_wrapped_object) + )) ``` ### Beacon state accessors From 9bbac0d2ccbde7e82afcee760b9fb0a47e16244b Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 21 Apr 2020 18:50:02 -0700 Subject: [PATCH 008/127] Added consistency check for FFG & LMD vote in validate_on_atttestation(), fixes #1636, fixes #1456, fixes #1408 --- specs/phase0/fork-choice.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index c42609be0..f844f735b 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -22,6 +22,7 @@ - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`filter_block_tree`](#filter_block_tree) - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [`is_descendant_block`](#is_descendant_block) - [`get_head`](#get_head) - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) - [`on_attestation` helpers](#on_attestation-helpers) @@ -162,7 +163,7 @@ def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages + if (i in store.latest_messages and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -220,6 +221,28 @@ def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: return blocks ``` +#### `is_descendant_block` + +```python +def is_descendant_block(store: Store, base_root: Root, descendant_root: Root) -> bool: + """ + Checks if the block with root ``descendant_root`` is a descendant of the block with root ``base_root`` + """ + descendants = [base_root] + + # Traverse the descendants block tree and check if ``descendant_root`` is encountered + while(descendants): + if descendants[0] == descendant_root: + return True + descendants.extend([ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == descendants[0] + ]) + descendants = descendants[1:] + + return False +``` + #### `get_head` ```python @@ -286,6 +309,9 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + # FFG and LMD vote must be consistent with each other + assert is_descendant_block(store, target.root, attestation.data.beacon_block_root) + # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. assert get_current_slot(store) >= attestation.data.slot + 1 From fe13bab33803eea248385016825ae1a8c4ae27e6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 23 Apr 2020 10:26:34 -0600 Subject: [PATCH 009/127] rework rewards/penalties to be more granular --- specs/phase0/beacon-chain.md | 101 +++++++++++++++++++++++++++-------- 1 file changed, 80 insertions(+), 21 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index cdf38dc1e..00e1d3a73 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1347,32 +1347,57 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: previous_epoch = get_previous_epoch(state) - total_balance = get_total_active_balance(state) - rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] - eligible_validator_indices = [ + return [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) ] +``` - # Micro-incentives for matching FFG source, FFG target, and head - matching_source_attestations = get_matching_source_attestations(state, previous_epoch) - matching_target_attestations = get_matching_target_attestations(state, previous_epoch) - matching_head_attestations = get_matching_head_attestations(state, previous_epoch) - for attestations in (matching_source_attestations, matching_target_attestations, matching_head_attestations): - unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) - attesting_balance = get_total_balance(state, unslashed_attesting_indices) - for index in eligible_validator_indices: - if index in unslashed_attesting_indices: - increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) - rewards[index] += reward_numerator // (total_balance // increment) - else: - penalties[index] += get_base_reward(state, index) +```python +def compute_attestation_component_deltas(state: BeaconState, + attestations: Sequence[PendingAttestation] + ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] + total_balance = get_total_active_balance(state) + unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) + attesting_balance = get_total_balance(state, unslashed_attesting_indices) + for index in get_eligible_validator_indices(state): + if index in unslashed_attesting_indices: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) + else: + penalties[index] += get_base_reward(state, index) + return rewards, penalties +``` +```python +def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return compute_attestation_component_deltas(state, matching_source_attestations) +``` + +```python +def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + matching_target_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return compute_attestation_component_deltas(state, matching_target_attestations) +``` + +```python +def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + matching_head_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return compute_attestation_component_deltas(state, matching_head_attestations) +``` + +```python +def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: # Proposer and inclusion delay micro-rewards + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) for index in get_unslashed_attesting_indices(state, matching_source_attestations): attestation = min([ a for a in matching_source_attestations @@ -1382,16 +1407,50 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence rewards[attestation.proposer_index] += proposer_reward max_attester_reward = get_base_reward(state, index) - proposer_reward rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) + return rewards, penalties +``` +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: # Inactivity penalty - finality_delay = previous_epoch - state.finalized_checkpoint.epoch + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] + finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch + if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + matching_target_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) - for index in eligible_validator_indices: + for index in get_eligible_validator_indices(state): penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) + return rewards, penalties +``` + +```python +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + source_rewards, source_penalties = get_source_deltas(state) + target_rewards, target_penalties = get_target_deltas(state) + head_rewards, head_penalties = get_head_deltas(state) + inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) + _, inactivity_penalties = get_inactivity_penalty_deltas(state) + + rewards = [ + source_rewards[i] + + target_rewards[i] + + head_rewards[i] + + inclusion_delay_rewards[i] + for i in range(len(state.validators)) + ] + + penalties = [ + source_penalties[i] + + target_penalties[i] + + head_penalties[i] + + inactivity_penalties[i] + for i in range(len(state.validators)) + ] return rewards, penalties ``` From 7612667bbef59c7df5255d8bebde1ba6618ce18b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 23 Apr 2020 11:13:09 -0600 Subject: [PATCH 010/127] minor feedback and fixes on rewards/penalites proposal Co-Authored-By: Hsiao-Wei Wang --- specs/phase0/beacon-chain.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 00e1d3a73..2d9429cf1 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1382,7 +1382,7 @@ def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei ```python def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - matching_target_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) return compute_attestation_component_deltas(state, matching_target_attestations) ``` @@ -1394,7 +1394,9 @@ def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]] ```python def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - # Proposer and inclusion delay micro-rewards + """ + Return proposer and inclusion delay micro-rewards. + """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) @@ -1412,13 +1414,15 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - # Inactivity penalty + """ + Return inactivity penalty. + """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: - matching_target_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) From 9acea519382c0fa8752328b106423890f8d1f01c Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Sat, 25 Apr 2020 14:17:28 -0700 Subject: [PATCH 011/127] Simplified by re-using get_ancestor() --- specs/phase0/fork-choice.md | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index f844f735b..42d0cea11 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -22,7 +22,6 @@ - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`filter_block_tree`](#filter_block_tree) - [`get_filtered_block_tree`](#get_filtered_block_tree) - - [`is_descendant_block`](#is_descendant_block) - [`get_head`](#get_head) - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) - [`on_attestation` helpers](#on_attestation-helpers) @@ -221,28 +220,6 @@ def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: return blocks ``` -#### `is_descendant_block` - -```python -def is_descendant_block(store: Store, base_root: Root, descendant_root: Root) -> bool: - """ - Checks if the block with root ``descendant_root`` is a descendant of the block with root ``base_root`` - """ - descendants = [base_root] - - # Traverse the descendants block tree and check if ``descendant_root`` is encountered - while(descendants): - if descendants[0] == descendant_root: - return True - descendants.extend([ - root for root in store.blocks.keys() - if store.blocks[root].parent_root == descendants[0] - ]) - descendants = descendants[1:] - - return False -``` - #### `get_head` ```python @@ -310,7 +287,8 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot # FFG and LMD vote must be consistent with each other - assert is_descendant_block(store, target.root, attestation.data.beacon_block_root) + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. From 4d980aec71f93f2f788fc8ea76bec14521d53abc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:23:31 +0800 Subject: [PATCH 012/127] Fix validator guide 1. Avoid negative computation in `is_candidate_block` 2. Fix `get_block_signature`: avoid extra casting; it's simpler to use BeaconBlock instead of BeaconHeader --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index bc7510403..4576c4b90 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -281,8 +281,8 @@ def voting_period_start_time(state: BeaconState) -> uint64: ```python def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: return ( - block.timestamp <= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE - and block.timestamp >= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start ) ``` @@ -350,9 +350,9 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: `signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: ```python -def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - signing_root = compute_signing_root(header, domain) +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) return bls.Sign(privkey, signing_root) ``` From 303d7d5adb35e51a49c570e15256763c136de8a1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 13:27:00 +0800 Subject: [PATCH 013/127] Add validator guide tests 1. "Becoming a validator" 2. "Validator assignments" 3. "Beacon chain responsibilities: Block proposal" --- .../test/validator/test_validator_unittest.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py new file mode 100644 index 000000000..019221eee --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -0,0 +1,182 @@ +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.state import next_epoch +from eth2spec.utils import bls + + +def run_is_candidate_block(spec, eth1_block, period_start, success): + result = spec.is_candidate_block(eth1_block, period_start) + if success: + assert result + else: + assert not result + + +def get_min_new_period_epochs(spec): + return int( + spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 # to seconds + / spec.SECONDS_PER_SLOT / spec.SLOTS_PER_EPOCH + ) + + +# +# Becoming a validator +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_check_if_validator_active(spec, state): + active_validator_index = len(state.validators) - 1 + assert spec.check_if_validator_active(state, active_validator_index) + new_validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, new_validator_index, amount, signed=True) + spec.process_deposit(state, deposit) + assert not spec.check_if_validator_active(state, new_validator_index) + + +# +# Validator assignments +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment(spec, state): + epoch = spec.get_current_epoch(state) + validator_index = len(state.validators) - 1 + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + + +@with_all_phases +@spec_state_test +@never_bls +def test_is_proposer(spec, state): + proposer_index = spec.get_beacon_proposer_index(state) + assert spec.is_proposer(state, proposer_index) + + proposer_index = proposer_index + 1 % len(state.validators) + assert not spec.is_proposer(state, proposer_index) + + +# +# Beacon chain responsibilities +# + + +# Block proposal + + +@with_all_phases +@spec_state_test +def test_get_epoch_signature(spec, state): + block = spec.BeaconBlock() + privkey = privkeys[0] + pubkey = pubkeys[0] + signature = spec.get_epoch_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_candidate_block(spec, state): + period_start = spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 + 1000 + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE + 1), + period_start, + success=False, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 - 1), + period_start, + success=False, + ) + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_default_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + state.eth1_data_votes = () + eth1_chain = [] + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data == state.eth1_data + + +@with_all_phases +@spec_state_test +def test_get_eth1_data_consensus_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + state.eth1_data_votes = () + eth1_chain = [] + eth1_data_votes = [] + block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + for i in range(votes_length): + eth1_chain.append(block) + eth1_data_votes.append(spec.get_eth1_data(block)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + print(state.eth1_data_votes) + assert eth1_data.block_hash == block.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_compute_new_state_root(spec, state): + pre_state = state.copy() + post_state = state.copy() + block = build_empty_block(spec, state, state.slot + 1) + state_root = spec.compute_new_state_root(state, block) + + assert state_root != pre_state.hash_tree_root() + + # dumb verification + spec.process_slots(post_state, block.slot) + spec.process_block(post_state, block) + assert state_root == post_state.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_block_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + block = build_empty_block(spec, state) + signature = spec.get_block_signature(state, block, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + signing_root = spec.compute_signing_root(block, domain) + assert bls.Verify(pubkey, signing_root, signature) From bdae27e317b44c1ab031a0cbdb5b4d61d0fe05a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:08:36 +0800 Subject: [PATCH 014/127] Add bls.AggregatePKs helper --- tests/core/pyspec/eth2spec/utils/bls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 83371ac62..3b648fac9 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -51,3 +51,8 @@ def Sign(SK, message): @only_with_bls(alt_return=STUB_COORDINATES) def signature_to_G2(signature): return _signature_to_G2(signature) + + +@only_with_bls(alt_return=STUB_PUBKEY) +def AggregatePKs(pubkeys): + return bls._AggregatePKs(pubkeys) From 8adc15e83de9ab6c0ac10b93aa739df2be989704 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 23 Apr 2020 15:09:23 +0800 Subject: [PATCH 015/127] Add validator guide tests 1. "Beacon chain responsibilities: Attesting" 2. "Beacon chain responsibilities: Attestation aggregation" --- .../test/validator/test_validator_unittest.py | 118 +++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 019221eee..4f6697f2f 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,9 +1,11 @@ from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.attestations import build_attestation_data from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.state import next_epoch from eth2spec.utils import bls +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_is_candidate_block(spec, eth1_block, period_start, success): @@ -21,6 +23,14 @@ def get_min_new_period_epochs(spec): ) +def get_mock_aggregate(spec): + return spec.Attestation( + data=spec.AttestationData( + slot=10, + ) + ) + + # # Becoming a validator # @@ -47,7 +57,7 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test @never_bls -def test_get_committee_assignment(spec, state): +def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 assignment = spec.get_committee_assignment(state, epoch, validator_index) @@ -150,7 +160,6 @@ def test_get_eth1_data_consensus_vote(spec, state): state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - print(state.eth1_data_votes) assert eth1_data.block_hash == block.hash_tree_root() @@ -180,3 +189,108 @@ def test_get_block_signature(spec, state): domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) signing_root = spec.compute_signing_root(block, domain) assert bls.Verify(pubkey, signing_root, signature) + + +# Attesting + + +@with_all_phases +@spec_state_test +def test_get_attestation_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + attestation_data = spec.AttestationData(slot=10) + signature = spec.get_attestation_signature(state, attestation_data, privkey) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +# Attestation aggregation + + +@with_all_phases +@spec_state_test +def test_get_slot_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + slot = 10 + signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) + signing_root = spec.compute_signing_root(spec.Slot(slot), domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_is_aggregator(spec, state): + # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` + # if we have more validators and larger committeee size + slot = state.slot + committee_index = 0 + has_aggregator = False + beacon_committee = spec.get_beacon_committee(state, slot, committee_index) + for validator_index in beacon_committee: + privkey = privkeys[validator_index] + slot_signature = spec.get_slot_signature(state, slot, privkey) + if spec.is_aggregator(state, slot, committee_index, slot_signature): + has_aggregator = True + break + assert has_aggregator + + +@with_all_phases +@spec_state_test +def test_get_aggregate_signature(spec, state): + attestations = [] + pubkeys = [] + slot = state.slot + committee_index = 0 + attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) + beacon_committee = spec.get_beacon_committee( + state, + attestation_data.slot, + attestation_data.index, + ) + committee_size = len(beacon_committee) + aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) + for i, validator_index in enumerate(beacon_committee): + bits = aggregation_bits + bits[i] = True + attestations.append( + spec.Attestation( + data=attestation_data, + aggregation_bits=bits, + ) + ) + pubkeys.append(state.validators[validator_index].pubkey) + pubkey = bls.AggregatePKs(pubkeys) + signature = spec.get_aggregate_signature(attestations) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof(spec, state): + privkey = privkeys[0] + aggregator_index = spec.ValidatorIndex(10) + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, aggregator_index, aggregate, privkey) + assert aggregate_and_proof.aggregator_index == aggregator_index + assert aggregate_and_proof.aggregate == aggregate + assert aggregate_and_proof.selection_proof == spec.get_slot_signature(state, aggregate.data.slot, privkey) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) + signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) + domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) + signing_root = spec.compute_signing_root(aggregate_and_proof, domain) + assert bls.Verify(pubkey, signing_root, signature) From 70bd73d2b5f4b8c3f19a9397fd983c23fd1eecc6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 20:47:36 +0800 Subject: [PATCH 016/127] Apply PR feedback from @djrtwo Fix get_eth1_vote test cases --- .../test/validator/test_validator_unittest.py | 164 ++++++++++++++---- 1 file changed, 131 insertions(+), 33 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 4f6697f2f..a655cb486 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -8,12 +8,29 @@ from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_is_candidate_block(spec, eth1_block, period_start, success): - result = spec.is_candidate_block(eth1_block, period_start) - if success: - assert result +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): + signature = get_signature_fn(state, obj, privkey) + signing_root = spec.compute_signing_root(obj, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +def run_get_committee_assignment(spec, state, epoch, validator_index, valid=True): + try: + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + assert validator_index in committee + assert valid + except AssertionError: + assert not valid else: - assert not result + assert valid + + +def run_is_candidate_block(spec, eth1_block, period_start, success=True): + assert success == spec.is_candidate_block(eth1_block, period_start) def get_min_new_period_epochs(spec): @@ -60,11 +77,25 @@ def test_check_if_validator_active(spec, state): def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 - assignment = spec.get_committee_assignment(state, epoch, validator_index) - committee, committee_index, slot = assignment - assert spec.compute_epoch_at_slot(slot) == epoch - assert committee == spec.get_beacon_committee(state, slot, committee_index) - assert committee_index < spec.get_committee_count_at_slot(state, slot) + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_next_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 1 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_out_bound_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 2 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=False) @with_all_phases @@ -92,10 +123,16 @@ def test_get_epoch_signature(spec, state): block = spec.BeaconBlock() privkey = privkeys[0] pubkey = pubkeys[0] - signature = spec.get_epoch_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_epoch_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -130,7 +167,7 @@ def test_is_candidate_block(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_default_vote(spec, state): +def test_get_eth1_vote_default_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) for _ in range(min_new_period_epochs): next_epoch(spec, state) @@ -143,24 +180,61 @@ def test_get_eth1_data_default_vote(spec, state): @with_all_phases @spec_state_test -def test_get_eth1_data_consensus_vote(spec, state): +def test_get_eth1_vote_consensus_vote(spec, state): min_new_period_epochs = get_min_new_period_epochs(spec) - for _ in range(min_new_period_epochs): + for _ in range(min_new_period_epochs + 2): next_epoch(spec, state) period_start = spec.voting_period_start_time(state) votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - eth1_chain = [] + + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] eth1_data_votes = [] - block = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + + # Only the first vote is for block_1 + eth1_data_votes.append(spec.get_eth1_data(block_1)) + # Other votes are for block_2 + for _ in range(votes_length - 1): + eth1_data_votes.append(spec.get_eth1_data(block_2)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data.block_hash == block_2.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_tie(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] + eth1_data_votes = [] + # Half votes are for block_1, another half votes are for block_2 for i in range(votes_length): - eth1_chain.append(block) + if i % 2 == 0: + block = block_1 + else: + block = block_2 eth1_data_votes.append(spec.get_eth1_data(block)) state.eth1_data_votes = eth1_data_votes eth1_data = spec.get_eth1_vote(state, eth1_chain) - assert eth1_data.block_hash == block.hash_tree_root() + + # Tiebreak by smallest distance -> eth1_chain[0] + assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() @with_all_phases @@ -185,10 +259,16 @@ def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] block = build_empty_block(spec, state) - signature = spec.get_block_signature(state, block, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - signing_root = spec.compute_signing_root(block, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_block_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attesting @@ -200,10 +280,16 @@ def test_get_attestation_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] attestation_data = spec.AttestationData(slot=10) - signature = spec.get_attestation_signature(state, attestation_data, privkey) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=attestation_data, + domain=domain, + get_signature_fn=spec.get_attestation_signature, + privkey=privkey, + pubkey=pubkey, + ) # Attestation aggregation @@ -214,11 +300,17 @@ def test_get_attestation_signature(spec, state): def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] - slot = 10 - signature = spec.get_slot_signature(state, spec.Slot(slot), privkey) + slot = spec.Slot(10) domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) - signing_root = spec.compute_signing_root(spec.Slot(slot), domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=slot, + domain=domain, + get_signature_fn=spec.get_slot_signature, + privkey=privkey, + pubkey=pubkey, + ) @with_all_phases @@ -290,7 +382,13 @@ def test_get_aggregate_and_proof_signature(spec, state): pubkey = pubkeys[0] aggregate = get_mock_aggregate(spec) aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) - signature = spec.get_aggregate_and_proof_signature(state, aggregate_and_proof, privkey) domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) - signing_root = spec.compute_signing_root(aggregate_and_proof, domain) - assert bls.Verify(pubkey, signing_root, signature) + run_get_signature_test( + spec=spec, + state=state, + obj=aggregate_and_proof, + domain=domain, + get_signature_fn=spec.get_aggregate_and_proof_signature, + privkey=privkey, + pubkey=pubkey, + ) From d311248d35684cd5aa6e47b286e075bb4feb46f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 21:45:01 +0800 Subject: [PATCH 017/127] Increase `EPOCHS_PER_ETH1_VOTING_PERIOD` from `2` to `4` for testing eth1 votes consensus --- configs/minimal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index b39a4fc01..c8b58146f 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -89,7 +89,7 @@ MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs MAX_SEED_LOOKAHEAD: 4 # [customized] higher frequency new deposits from eth1 for testing -EPOCHS_PER_ETH1_VOTING_PERIOD: 2 +EPOCHS_PER_ETH1_VOTING_PERIOD: 4 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs From 2dbc33327084d2814958f92eb0a838b9bc161903 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Apr 2020 22:05:47 +0800 Subject: [PATCH 018/127] Make `compute_new_state_root` a pure function --- specs/phase0/validator.md | 7 ++++--- .../eth2spec/test/validator/test_validator_unittest.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4576c4b90..cbe0c2d12 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -340,9 +340,10 @@ It is useful to be able to run a state transition function (working on a copy of ```python def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - process_slots(state, block.slot) - process_block(state, block) - return hash_tree_root(state) + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + temp_state = state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) ``` ##### Signature diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index a655cb486..5bb246ed5 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -246,6 +246,7 @@ def test_compute_new_state_root(spec, state): state_root = spec.compute_new_state_root(state, block) assert state_root != pre_state.hash_tree_root() + assert state == pre_state # dumb verification spec.process_slots(post_state, block.slot) From d128400da5cbcd16102fa760d4381ada07d4a577 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 16:16:33 -0600 Subject: [PATCH 019/127] remove interop from header and gossip sections in network spec --- specs/phase0/p2p-interface.md | 55 +++++------------------------------ 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f4f2f3d22..a3a9b48fe 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -4,7 +4,7 @@ This document contains the networking specification for Ethereum 2.0 clients. It consists of four main sections: -1. A specification of the network fundamentals detailing the two network configurations: interoperability test network and mainnet launch. +1. A specification of the network fundamentals. 2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. 3. The rationale and further explanation for the design choices made in the previous two sections. 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. @@ -120,42 +120,20 @@ It consists of four main sections: This section outlines the specification for the networking stack in Ethereum 2.0 clients. -Sections that have differing parameters for mainnet launch and interoperability testing are split into subsections. Sections that are not split have the same parameters for interoperability testing as mainnet launch. - ## Transport Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. -#### Interop - All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). -To facilitate connectivity and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. +Clients must support listening on at least one of IPv4 or IPv6. Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. (Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. -#### Mainnet - -All requirements from the interoperability testnet apply, except for the IPv4 addressing scheme requirement. - -At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. - -Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined closer to the time. - ## Encryption and identification -#### Interop - -[SecIO](https://github.com/libp2p/specs/tree/master/secio) with `secp256k1` identities will be used for initial interoperability testing. - -The following SecIO parameters MUST be supported by all stacks: - -- Key agreement: ECDH-P256. -- Cipher: AES-128. -- Digest: SHA-256. - #### Mainnet The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure @@ -167,13 +145,7 @@ As specified in the libp2p specification, clients MUST support the `XX` handshak Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. -#### Interop - -Connection-level and stream-level (see the [Rationale](#design-decision-rationale) section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. - -#### Mainnet - -Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95). Depending on the number of clients that have implementations for multiselect 2.0 by mainnet, [multistream-select 1.0](https://github.com/multiformats/multistream-select/) may be phased out. +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. ## Multiplexing @@ -181,7 +153,7 @@ During connection bootstrapping, libp2p dynamically negotiates a mutually suppor Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux MUST take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. # Eth2 network interaction domains @@ -265,7 +237,6 @@ The payload is carried in the `data` field of a gossipsub message, and varies de |------------------------------------------------|-------------------------| | beacon_block | SignedBeaconBlock | | beacon_aggregate_and_proof | SignedAggregateAndProof | -| beacon_attestation\* | Attestation | | committee_index{subnet_id}\_beacon_attestation | Attestation | | voluntary_exit | SignedVoluntaryExit | | proposer_slashing | ProposerSlashing | @@ -275,8 +246,6 @@ Clients MUST reject (fail validation) messages containing an incorrect type, or When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. -\* The `beacon_attestation` topic is only for interop and will be removed prior to mainnet. - #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: @@ -323,11 +292,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - The signature of `attestation` is valid. -#### Interop - -Unaggregated and aggregated attestations from all shards are sent as `Attestation`s to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD try to perform local attestation aggregation to prepare for block proposing. - -#### Mainnet +#### Attestations and Aggregation Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. @@ -339,17 +304,11 @@ Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `A Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. -#### Interop - -- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/eth2/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `SignedBeaconBlock`. - -#### Mainnet - - `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. -Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. +Implementations MUST use a single encoding for gossip. Changing an encoding will require coordination between participating implementations. ## The Req/Resp domain From 87586837c388a4005d09d7baf56213fc418211e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 Apr 2020 17:34:26 -0600 Subject: [PATCH 020/127] remove interop from phase 0 p2p specs --- specs/phase0/p2p-interface.md | 73 ++++++++++------------------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a3a9b48fe..f76502bb4 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -17,14 +17,8 @@ It consists of four main sections: - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - - [Interop](#interop) - - [Mainnet](#mainnet) - [Encryption and identification](#encryption-and-identification) - - [Interop](#interop-1) - - [Mainnet](#mainnet-1) - [Protocol Negotiation](#protocol-negotiation) - - [Interop](#interop-2) - - [Mainnet](#mainnet-2) - [Multiplexing](#multiplexing) - [Eth2 network interaction domains](#eth2-network-interaction-domains) - [Configuration](#configuration) @@ -33,11 +27,8 @@ It consists of four main sections: - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [Attestation subnets](#attestation-subnets) - - [Interop](#interop-3) - - [Mainnet](#mainnet-3) + - [Attestations and Aggregation](#attestations-and-aggregation) - [Encodings](#encodings) - - [Interop](#interop-4) - - [Mainnet](#mainnet-4) - [The Req/Resp domain](#the-reqresp-domain) - [Protocol identification](#protocol-identification) - [Req/Resp interaction](#reqresp-interaction) @@ -56,29 +47,25 @@ It consists of four main sections: - [Integration into libp2p stacks](#integration-into-libp2p-stacks) - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - - [Interop](#interop-5) - - [Mainnet](#mainnet-5) - - [`eth2` field](#eth2-field) - - [General capabilities](#general-capabilities) + - [`eth2` field](#eth2-field) + - [General capabilities](#general-capabilities) - [Topic advertisement](#topic-advertisement) - - [Mainnet](#mainnet-6) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) - - [Why are we not using QUIC for mainnet from the start?](#why-are-we-not-using-quic-for-mainnet-from-the-start) + - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) - [Multiplexing](#multiplexing-1) - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) - [Protocol Negotiation](#protocol-negotiation-1) - - [When is multiselect 2.0 due and why are we using it for mainnet?](#when-is-multiselect-20-due-and-why-are-we-using-it-for-mainnet) + - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) - [Encryption](#encryption) - - [Why are we using SecIO for interop? Why not for mainnet?](#why-are-we-using-secio-for-interop-why-not-for-mainnet) - - [Why are we using Noise/TLS 1.3 for mainnet?](#why-are-we-using-noisetls-13-for-mainnet) + - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) + - [Why are we using Noise/TLS 1.3?](#why-are-we-using-noisetls-13) - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) - - [Will mainnnet networking be untested when it launches?](#will-mainnnet-networking-be-untested-when-it-launches) - [Gossipsub](#gossipsub) - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) @@ -134,10 +121,8 @@ Nodes operating behind a NAT, or otherwise undialable by default (e.g. container ## Encryption and identification -#### Mainnet - The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure -channel handshake with `secp256k1` identities will be used for mainnet. +channel handshake with `secp256k1` identities will be used for encryption. As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. @@ -409,7 +394,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: - `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. -- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. This encoding type MUST be supported by all clients. #### SSZ-encoding strategy (with or without Snappy) @@ -642,7 +627,7 @@ The response MUST consist of a single `response_chunk`. ## The discovery domain: discv5 -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery. `discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. @@ -682,15 +667,7 @@ If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `a If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. -#### Interop - -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth2 capability information, as it would be superfluous. - -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an Eth2 peer, in order to eschew connecting to Eth 1.0 peers. - -#### Mainnet - -##### `eth2` field +#### `eth2` field ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, and next fork epoch to ensure connections are made with peers on the intended eth2 network. @@ -722,14 +699,12 @@ Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `ne Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -##### General capabilities +#### General capabilities -On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. +ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. ### Topic advertisement -#### Mainnet - discv5's topic advertisement feature is not expected to be ready for mainnet launch of Phase 0. Once this feature is built out and stable, we expect to use topic advertisement as a rendezvous facility for peers on shards. Until then, the ENR [attestation subnet bitfield](#attestation-subnet-bitfield) will be used for discovery of peers on particular subnets. @@ -775,7 +750,7 @@ Modeling for upgradeability and dynamic transport selection from the get-go lays Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. -### Why are we not using QUIC for mainnet from the start? +### Why are we not using QUIC from the start? The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). @@ -791,13 +766,13 @@ Overlay multiplexers are not necessary with QUIC since the protocol provides nat ## Protocol Negotiation -### When is multiselect 2.0 due and why are we using it for mainnet? +### When is multiselect 2.0 due and why do we plan to migrate to it? multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. -In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. +At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. -We plan to use multiselect 2.0 for mainnet because it will: +We plan to eventually migrate to multiselect 2.0 because it will: 1. Reduce round trips during connection bootstrapping and stream protocol negotiation. 2. Enable efficient one-stream-per-request interaction patterns. @@ -819,17 +794,15 @@ At present, multistream-select 1.0 is used for both types of negotiation, but mu ## Encryption -### Why are we using SecIO for interop? Why not for mainnet? +### Why are we not supporting SecIO? SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. -SecIO is the common denominator across the various language libraries at this stage. It is widely implemented. That’s why we have chosen to use it for initial interop to minimize overhead in getting to a basic interoperability testnet. - -We won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. +Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. SecIO is not considered secure for the purposes of this spec. -### Why are we using Noise/TLS 1.3 for mainnet? +### Why are we using Noise/TLS 1.3? Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): @@ -855,10 +828,6 @@ Transport level encryption secures message exchange and provides properties that Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). -### Will mainnnet networking be untested when it launches? - -Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. - ## Gossipsub ### Why are we using a pub/sub algorithm for block and attestation propagation? @@ -967,7 +936,7 @@ Requests are segregated by protocol ID to: 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). 7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. -**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to significantly hinder this domain. ### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? From 5f18dd778c1d44203140a27f088b9596767189d5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 28 Apr 2020 19:26:14 -0600 Subject: [PATCH 021/127] add baseline get_target_deltas tests --- specs/phase0/beacon-chain.md | 24 ++++- .../eth2spec/test/helpers/attestations.py | 33 ++++++- .../rewards/test_get_target_deltas.py | 92 +++++++++++++++++++ .../test_process_rewards_and_penalties.py | 28 +----- 4 files changed, 147 insertions(+), 30 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 33689ece9..f40f0096b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1341,6 +1341,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: #### Rewards and penalties +##### Helpers + ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) @@ -1376,20 +1378,31 @@ def compute_attestation_component_deltas(state: BeaconState, return rewards, penalties ``` +##### Components of attestation deltas + ```python def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for source-vote for each validator. + """ matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) return compute_attestation_component_deltas(state, matching_source_attestations) ``` ```python def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for target-vote for each validator. + """ matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) return compute_attestation_component_deltas(state, matching_target_attestations) ``` ```python def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for head-vote for each validator. + """ matching_head_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) return compute_attestation_component_deltas(state, matching_head_attestations) ``` @@ -1397,7 +1410,7 @@ def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]] ```python def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Return proposer and inclusion delay micro-rewards. + Return proposer and inclusion delay micro-rewards/penalties for each validator. """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] @@ -1417,7 +1430,7 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Return inactivity penalty. + Return inactivity reward/penalty deltas for each validator. """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] @@ -1434,8 +1447,13 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S return rewards, penalties ``` +##### `get_attestation_deltas` + ```python def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attestation reward/penalty deltas for each validator. + """ source_rewards, source_penalties = get_source_deltas(state) target_rewards, target_penalties = get_target_deltas(state) head_rewards, head_penalties = get_head_deltas(state) @@ -1461,6 +1479,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence return rewards, penalties ``` +##### `process_rewards_and_penalties` + ```python def process_rewards_and_penalties(state: BeaconState) -> None: if get_current_epoch(state) == GENESIS_EPOCH: diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..692e874de 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,7 +1,7 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0 -from eth2spec.test.helpers.state import state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls @@ -278,3 +278,34 @@ def next_epoch_with_attestations(spec, signed_blocks.append(signed_block) return state, signed_blocks, post_state + + +def prepare_state_with_full_attestations(spec, state, empty=False): + """ + Fill ``state`` with maximally full attestations. + Move to the start of the next epoch to ensure full epoch worth. + """ + # Go to start of next epoch to ensure can have full participation + next_epoch(spec, state) + + start_slot = state.slot + start_epoch = spec.get_current_epoch(state) + next_epoch_start_slot = spec.compute_start_slot_at_epoch(start_epoch + 1) + attestations = [] + for _ in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): + # create an attestation for each index in each slot in epoch + if state.slot < next_epoch_start_slot: + for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): + attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) + attestations.append(attestation) + # fill each created slot in state after inclusion delay + if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: + inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + include_attestations = [att for att in attestations if att.data.slot == inclusion_slot] + add_attestations_to_state(spec, state, include_attestations, state.slot) + next_slot(spec, state) + + assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + assert len(state.previous_epoch_attestations) == len(attestations) + + return attestations diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py new file mode 100644 index 000000000..c30865cf0 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py @@ -0,0 +1,92 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations + + +def run_get_target_deltas(spec, state): + """ + Run ``process_block_header``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + yield 'pre', state + + rewards, penalties = spec.get_target_deltas(state) + + yield 'rewards', rewards + yield 'penalties', penalties + + matching_target_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_target_indices = spec.get_unslashed_attesting_indices(state, matching_target_attestations) + for index in spec.get_eligible_validator_indices(state): + if index in matching_target_indices and not state.validators[index].slashed: + assert rewards[index] > 0 + assert penalties[index] == 0 + else: + assert rewards[index] == 0 + assert penalties[index] > 0 + + +@with_all_phases +@spec_state_test +def test_empty(spec, state): + # Do not add any attestations to state + + yield from run_get_target_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_all_correct(spec, state): + prepare_state_with_full_attestations(spec, state) + + yield from run_get_target_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_half_correct(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Make half of pending attestations have bad target + for pending_attestation in state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2]: + pending_attestation.data.target.root = b'\x66'*32 + + yield from run_get_target_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_half_full(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Remove half of attestations + state.previous_epoch_attestations = state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2] + + yield from run_get_target_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_one_correct(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Remove half of attestations + state.previous_epoch_attestations = state.previous_epoch_attestations[:1] + + yield from run_get_target_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Slash half of validators + for validator in state.validators: + validator.slashed = True + + yield from run_get_target_deltas(spec, state) + +def test_some_zero_balances(spec, state): + diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index af695fe69..f1d86c373 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -12,6 +12,7 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.attestations import ( add_attestations_to_state, get_valid_attestation, + prepare_state_with_full_attestations, ) from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -21,33 +22,6 @@ def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') -def prepare_state_with_full_attestations(spec, state, empty=False): - # Go to start of next epoch to ensure can have full participation - next_epoch(spec, state) - - start_slot = state.slot - start_epoch = spec.get_current_epoch(state) - next_epoch_start_slot = spec.compute_start_slot_at_epoch(start_epoch + 1) - attestations = [] - for _ in range(spec.SLOTS_PER_EPOCH + spec.MIN_ATTESTATION_INCLUSION_DELAY): - # create an attestation for each index in each slot in epoch - if state.slot < next_epoch_start_slot: - for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): - attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) - attestations.append(attestation) - # fill each created slot in state after inclusion delay - if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: - inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY - include_attestations = [att for att in attestations if att.data.slot == inclusion_slot] - add_attestations_to_state(spec, state, include_attestations, state.slot) - next_slot(spec, state) - - assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY - assert len(state.previous_epoch_attestations) == len(attestations) - - return attestations - - @with_phases(['phase0']) @spec_state_test def test_genesis_epoch_no_attestations_no_penalties(spec, state): From cd27e5e045f8a2f5d3a073e496596126fcedfc3a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 29 Apr 2020 09:56:48 -0600 Subject: [PATCH 022/127] add tests for source, target, head --- specs/phase0/beacon-chain.md | 6 +- .../pyspec/eth2spec/test/helpers/rewards.py | 117 ++++++++++++++++ .../rewards/test_get_head_deltas.py | 109 +++++++++++++++ .../rewards/test_get_source_deltas.py | 116 ++++++++++++++++ .../rewards/test_get_target_deltas.py | 125 ++++++++++-------- 5 files changed, 418 insertions(+), 55 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/helpers/rewards.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index f40f0096b..95f06f125 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -110,6 +110,10 @@ - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) - [Rewards and penalties](#rewards-and-penalties-1) + - [Helpers](#helpers) + - [Components of attestation deltas](#components-of-attestation-deltas) + - [`get_attestation_deltas`](#get_attestation_deltas) + - [`process_rewards_and_penalties`](#process_rewards_and_penalties) - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) @@ -1403,7 +1407,7 @@ def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]] """ Return attester micro-rewards/penalties for head-vote for each validator. """ - matching_head_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) return compute_attestation_component_deltas(state, matching_head_attestations) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py new file mode 100644 index 000000000..41497fe60 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -0,0 +1,117 @@ +from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations + + +def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): + """ + Run ``component_delta_fn``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + yield 'pre', state + + rewards, penalties = component_delta_fn(state) + + yield 'rewards', rewards + yield 'penalties', penalties + + matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) + matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + for index in spec.get_eligible_validator_indices(state): + validator = state.validators[index] + enough_for_reward = ( + validator.effective_balance * spec.BASE_REWARD_FACTOR + > spec.integer_squareroot(spec.get_total_active_balance(state)) // spec.BASE_REWARDS_PER_EPOCH + ) + + if index in matching_indices and not validator.slashed: + if enough_for_reward: + assert rewards[index] > 0 + else: + assert rewards[index] == 0 + assert penalties[index] == 0 + else: + assert rewards[index] == 0 + if enough_for_reward: + assert penalties[index] > 0 + else: + assert penalties[index] == 0 + + +def test_empty(spec, state, runner): + # Do not add any attestations to state + + yield from runner(spec, state) + + +def test_full_all_correct(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + yield from runner(spec, state) + + +def test_half_full(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + # Remove half of attestations + state.previous_epoch_attestations = state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2] + + yield from runner(spec, state) + + +def test_one_attestation_one_correct(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + # Remove half of attestations + state.previous_epoch_attestations = state.previous_epoch_attestations[:1] + + yield from runner(spec, state) + + +def test_with_slashed_validators(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + # Slash half of validators + for validator in state.validators[:len(state.validators) // 2]: + validator.slashed = True + + yield from runner(spec, state) + + +def test_some_zero_effective_balances_that_attested(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + # Set some balances to zero + state.validators[0].effective_balance = 0 + state.validators[1].effective_balance = 0 + + yield from runner(spec, state) + + +def test_some_zero_effective_balances_that_did_not_attest(spec, state, runner): + prepare_state_with_full_attestations(spec, state) + + # Set some balances to zero + attestation = state.previous_epoch_attestations[0] + # Remove attestation + state.previous_epoch_attestations = state.previous_epoch_attestations[1:] + # Set removed indices effective balance to zero + indices = spec.get_unslashed_attesting_indices(state, [attestation]) + for index in indices: + state.validators[index].effective_balance = 0 + + yield from runner(spec, state) + + +def test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): + prepare_state_with_full_attestations(spec, state) + + # Make fraction_incorrect of pending attestations have bad target/head as specified + num_incorrect = int(fraction_incorrect * len(state.previous_epoch_attestations)) + for pending_attestation in state.previous_epoch_attestations[:num_incorrect]: + if not correct_target: + pending_attestation.data.target.root = b'\x55' * 32 + if not correct_head: + pending_attestation.data.beacon_block_root = b'\x66' * 32 + + yield from runner(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py new file mode 100644 index 000000000..24e3aaac5 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py @@ -0,0 +1,109 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.rewards import run_attestation_component_deltas +import eth2spec.test.helpers.rewards as rewards_helpers + + +def run_get_head_deltas(spec, state): + """ + Run ``get_head_deltas``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + + yield from run_attestation_component_deltas( + spec, + state, + spec.get_head_deltas, + spec.get_matching_head_attestations, + ) + + +@with_all_phases +@spec_state_test +def test_empty(spec, state): + yield from rewards_helpers.test_empty(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_full_all_correct(spec, state): + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_half_full(spec, state): + yield from rewards_helpers.test_half_full(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_one_attestation_one_correct(spec, state): + yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators(spec, state): + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_head_deltas) + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_head_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + runner=run_get_head_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_head_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + runner=run_get_head_deltas + ) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py new file mode 100644 index 000000000..da47bd204 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py @@ -0,0 +1,116 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.rewards import run_attestation_component_deltas +import eth2spec.test.helpers.rewards as rewards_helpers + + +def run_get_source_deltas(spec, state): + """ + Run ``get_source_deltas``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + + yield from run_attestation_component_deltas( + spec, + state, + spec.get_source_deltas, + spec.get_matching_source_attestations, + ) + + +@with_all_phases +@spec_state_test +def test_empty(spec, state): + yield from rewards_helpers.test_empty(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_full_all_correct(spec, state): + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_half_full(spec, state): + yield from rewards_helpers.test_half_full(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_one_attestation_one_correct(spec, state): + yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators(spec, state): + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_source_deltas) + + +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_source_deltas) + + +# +# NOTE: No source incorrect tests +# All PendingAttestations in state have source validated +# We choose to keep this invariant in these tests to not force clients to test with degenerate states +# + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_source_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + runner=run_get_source_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_source_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + runner=run_get_source_deltas + ) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py index c30865cf0..406471c67 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py @@ -1,92 +1,109 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations +from eth2spec.test.helpers.rewards import run_attestation_component_deltas +import eth2spec.test.helpers.rewards as rewards_helpers def run_get_target_deltas(spec, state): """ - Run ``process_block_header``, yielding: + Run ``get_target_deltas``, yielding: - pre-state ('pre') - rewards ('rewards') - penalties ('penalties') """ - yield 'pre', state - rewards, penalties = spec.get_target_deltas(state) - - yield 'rewards', rewards - yield 'penalties', penalties - - matching_target_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_target_indices = spec.get_unslashed_attesting_indices(state, matching_target_attestations) - for index in spec.get_eligible_validator_indices(state): - if index in matching_target_indices and not state.validators[index].slashed: - assert rewards[index] > 0 - assert penalties[index] == 0 - else: - assert rewards[index] == 0 - assert penalties[index] > 0 + yield from run_attestation_component_deltas( + spec, + state, + spec.get_target_deltas, + spec.get_matching_target_attestations, + ) @with_all_phases @spec_state_test def test_empty(spec, state): - # Do not add any attestations to state - - yield from run_get_target_deltas(spec, state) + yield from rewards_helpers.test_empty(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - prepare_state_with_full_attestations(spec, state) - - yield from run_get_target_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_half_correct(spec, state): - prepare_state_with_full_attestations(spec, state) - - # Make half of pending attestations have bad target - for pending_attestation in state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2]: - pending_attestation.data.target.root = b'\x66'*32 - - yield from run_get_target_deltas(spec, state) + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_half_full(spec, state): - prepare_state_with_full_attestations(spec, state) - - # Remove half of attestations - state.previous_epoch_attestations = state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2] - - yield from run_get_target_deltas(spec, state) + yield from rewards_helpers.test_half_full(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test -def test_one_correct(spec, state): - prepare_state_with_full_attestations(spec, state) - - # Remove half of attestations - state.previous_epoch_attestations = state.previous_epoch_attestations[:1] - - yield from run_get_target_deltas(spec, state) +def test_one_attestation_one_correct(spec, state): + yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - prepare_state_with_full_attestations(spec, state) + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_target_deltas) - # Slash half of validators - for validator in state.validators: - validator.slashed = True - yield from run_get_target_deltas(spec, state) +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_target_deltas) -def test_some_zero_balances(spec, state): +@with_all_phases +@spec_state_test +def test_some_zero_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_target_deltas) + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_target_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + runner=run_get_target_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + runner=run_get_target_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head(spec, state): + yield from rewards_helpers.test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + runner=run_get_target_deltas + ) From 09cae4b3ccab47fcb7f9eba198af46d3bcdfa3d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 1 May 2020 15:17:41 +0200 Subject: [PATCH 023/127] Handle empty-aggregation-bits case, and add tests. See #1713 --- specs/phase0/beacon-chain.md | 4 ++-- specs/phase0/p2p-interface.md | 2 +- specs/phase1/beacon-chain.md | 3 ++- .../eth2spec/test/helpers/attestations.py | 3 +++ .../test_process_attestation.py | 23 +++++++++++++++++++ .../test_process_rewards_and_penalties.py | 3 +++ 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index cdf38dc1e..5ab499bfc 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -684,11 +684,11 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Check if ``indexed_attestation`` has sorted and unique indices and a valid aggregate signature. + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. """ # Verify indices are sorted and unique indices = indexed_attestation.attesting_indices - if not indices == sorted(set(indices)): + if len(indices) == 0 or not indices == sorted(set(indices)): return False # Verify aggregate signature pubkeys = [state.validators[i].pubkey for i in indices] diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 7197581dc..a44de8c8e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -293,7 +293,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. + - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - The signature of `aggregate` is valid. diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 596b3818f..1770b7a98 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -571,7 +571,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe attestation = indexed_attestation.attestation domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits - assert len(aggregation_bits) == len(indexed_attestation.committee) + if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): + return False if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..377426006 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -114,6 +114,9 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): + # If empty is true, the attestation has 0 participants, and will not be signed. + # Thus strictly speaking invalid when no participant is added later. + if slot is None: slot = state.slot if index is None: diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 8663391aa..df3279faa 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -64,6 +64,29 @@ def test_invalid_attestation_signature(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_zeroes_sig(spec, state): + attestation = get_valid_attestation(spec, state, empty=True) + attestation.signature = spec.BLSSignature(b'\x00' * 96) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_seemingly_valid_sig(spec, state): + attestation = get_valid_attestation(spec, state, empty=True) + # Special BLS value, valid for zero pubkeys on some (but not all) BLS implementations. + attestation.signature = spec.BLSSignature(b'\xc0' + b'\x00' * 95) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_before_inclusion_delay(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index af695fe69..b862b5c48 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -22,6 +22,9 @@ def run_process_rewards_and_penalties(spec, state): def prepare_state_with_full_attestations(spec, state, empty=False): + # If empty is true, attestations have 0 participants, and are not signed. + # Thus strictly speaking invalid when no participant is added later. + # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) From 47ed5b6500e3028e745458295cae395b9323ef9e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 1 May 2020 16:10:28 +0200 Subject: [PATCH 024/127] Fix rewards testing for empty/weird participation cases, adding more as well --- .../eth2spec/test/helpers/attestations.py | 25 +++++--- .../test_process_attestation.py | 18 +----- .../test_process_rewards_and_penalties.py | 64 +++++++++++++++---- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 377426006..85b543104 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -113,8 +113,8 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False) -def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signed=False, on_time=True): - # If empty is true, the attestation has 0 participants, and will not be signed. +def get_valid_attestation(spec, state, slot=None, index=None, filter_participant_set=None, signed=False, on_time=True): + # If filter_participant_set is filters everything, the attestation has 0 participants, and cannot be signed. # Thus strictly speaking invalid when no participant is added later. if slot is None: @@ -136,10 +136,8 @@ def get_valid_attestation(spec, state, slot=None, index=None, empty=False, signe aggregation_bits=aggregation_bits, data=attestation_data, ) - if not empty: - fill_aggregate_attestation(spec, state, attestation) - if signed: - sign_attestation(spec, state, attestation) + # fill the attestation with (optionally filtered) participants, and optionally sign it + fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == 'phase1' and on_time: attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) @@ -232,16 +230,25 @@ def get_attestation_signature(spec, state, attestation_data, privkey): return bls.Sign(privkey, signing_root) -def fill_aggregate_attestation(spec, state, attestation, signed=False): +def fill_aggregate_attestation(spec, state, attestation, signed=False, filter_participant_set=None): + """ + `signed`: Signing is optional. + `filter_participant_set`: Optional, filters the full committee indices set (default) to a subset that participates + """ beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, attestation.data.index, ) + # By default, have everyone participate + participants = set(beacon_committee) + # But optionally filter the participants to a smaller amount + if filter_participant_set is not None: + participants = filter_participant_set(participants) for i in range(len(beacon_committee)): - attestation.aggregation_bits[i] = True + attestation.aggregation_bits[i] = beacon_committee[i] in participants - if signed: + if signed and len(participants) > 0: sign_attestation(spec, state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index df3279faa..bb25a384e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,7 +13,6 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - next_slot, next_slots, next_epoch, transition_to, @@ -68,7 +67,7 @@ def test_invalid_attestation_signature(spec, state): @spec_state_test @always_bls def test_empty_participants_zeroes_sig(spec, state): - attestation = get_valid_attestation(spec, state, empty=True) + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants attestation.signature = spec.BLSSignature(b'\x00' * 96) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -79,7 +78,7 @@ def test_empty_participants_zeroes_sig(spec, state): @spec_state_test @always_bls def test_empty_participants_seemingly_valid_sig(spec, state): - attestation = get_valid_attestation(spec, state, empty=True) + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants # Special BLS value, valid for zero pubkeys on some (but not all) BLS implementations. attestation.signature = spec.BLSSignature(b'\xc0' + b'\x00' * 95) next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -283,19 +282,6 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases -@spec_state_test -def test_empty_aggregation_bits(spec, state): - next_slot(spec, state) - attestation = get_valid_attestation(spec, state, empty=True) - next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) - - assert attestation.aggregation_bits == Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( - *([0b0] * len(attestation.aggregation_bits))) - - yield from run_attestation_processing(spec, state, attestation) - - @with_all_phases @spec_state_test def test_too_many_aggregation_bits(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index b862b5c48..63aafe521 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -15,15 +15,15 @@ from eth2spec.test.helpers.attestations import ( ) from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from random import Random def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') -def prepare_state_with_full_attestations(spec, state, empty=False): - # If empty is true, attestations have 0 participants, and are not signed. - # Thus strictly speaking invalid when no participant is added later. +def prepare_state_with_full_attestations(spec, state, participation_fn=None): + # participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) @@ -36,8 +36,15 @@ def prepare_state_with_full_attestations(spec, state, empty=False): # create an attestation for each index in each slot in epoch if state.slot < next_epoch_start_slot: for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): - attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) - attestations.append(attestation) + def temp_participants_filter(comm): + if participation_fn is None: + return comm + else: + return participation_fn(state.slot, committee_index, comm) + attestation = get_valid_attestation(spec, state, index=committee_index, + filter_participant_set=temp_participants_filter, signed=True) + if any(attestation.aggregation_bits): # Only if there is at least 1 participant. + attestations.append(attestation) # fill each created slot in state after inclusion delay if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -192,20 +199,55 @@ def test_no_attestations_all_penalties(spec, state): assert state.balances[index] < pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_empty_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state, empty=True) +def run_with_participation(spec, state, participation_fn): + participated = set() + + def participation_tracker(slot, comm_index, comm): + att_participants = participation_fn(slot, comm_index, comm) + participated.update(att_participants) + return att_participants + + attestations = prepare_state_with_full_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == 0 + assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_almost_empty_attestations(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + +@with_all_phases +@spec_state_test +def test_random_fill_attestations(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + +@with_all_phases +@spec_state_test +def test_almost_full_attestations(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + +@with_all_phases +@spec_state_test +def test_full_attestation_participation(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) @with_all_phases From 45ad270f762c3949507d65bc08d92b353b9b6eb9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 1 May 2020 19:14:01 +0200 Subject: [PATCH 025/127] test double proposer slashings and exits --- .../test/helpers/proposer_slashings.py | 10 +- .../eth2spec/test/sanity/test_blocks.py | 132 +++++++++++++----- 2 files changed, 106 insertions(+), 36 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index ac2ebcf9c..d753d55dd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -2,9 +2,11 @@ from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey -def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] +def get_valid_proposer_slashing(spec, state, random_root=b'\x99' * 32, + validator_index=None, signed_1=False, signed_2=False): + if validator_index is None: + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot @@ -16,7 +18,7 @@ def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): body_root=b'\x55' * 32, ) header_2 = header_1.copy() - header_2.parent_root = b'\x99' * 32 + header_2.parent_root = random_root if signed_1: signed_header_1 = sign_block_header(spec, state, header_1, privkey) diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index b6b671872..3347d4f8b 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.utils import bls from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot, next_epoch @@ -228,7 +226,7 @@ def test_empty_epoch_transition_not_finalizing(spec, state): @spec_state_test def test_proposer_slashing(spec, state): # copy for later balance lookups. - pre_state = deepcopy(state) + pre_state = state.copy() proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) validator_index = proposer_slashing.signed_header_1.message.proposer_index @@ -256,11 +254,65 @@ def test_proposer_slashing(spec, state): assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) +@with_all_phases +@spec_state_test +def test_double_same_proposer_slashings_same_block(spec, state): + proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) + validator_index = proposer_slashing.signed_header_1.message.proposer_index + assert not state.validators[validator_index].slashed + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.proposer_slashings = [proposer_slashing, proposer_slashing] + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_all_phases +@spec_state_test +def test_double_similar_proposer_slashings_same_block(spec, state): + validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + + # Same validator, but different slashable offences in the same block + proposer_slashing_1 = get_valid_proposer_slashing(spec, state, random_root=b'\xaa' * 32, + validator_index=validator_index, + signed_1=True, signed_2=True) + proposer_slashing_2 = get_valid_proposer_slashing(spec, state, random_root=b'\xbb' * 32, + validator_index=validator_index, + signed_1=True, signed_2=True) + assert not state.validators[validator_index].slashed + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.proposer_slashings = [proposer_slashing_1, proposer_slashing_2] + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +def check_attester_slashing_effect(spec, pre_state, state, validator_index): + slashed_validator = state.validators[validator_index] + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) + + proposer_index = spec.get_beacon_proposer_index(state) + # gained whistleblower reward + assert get_balance(state, proposer_index) > get_balance(pre_state, proposer_index) + + @with_all_phases @spec_state_test def test_attester_slashing(spec, state): # copy for later balance lookups. - pre_state = deepcopy(state) + pre_state = state.copy() attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) validator_index = get_indexed_attestation_participants(spec, attester_slashing.attestation_1)[0] @@ -280,19 +332,11 @@ def test_attester_slashing(spec, state): yield 'blocks', [signed_block] yield 'post', state - slashed_validator = state.validators[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) + check_attester_slashing_effect(spec, pre_state, state, validator_index) - proposer_index = spec.get_beacon_proposer_index(state) - # gained whistleblower reward - assert ( - get_balance(state, proposer_index) > - get_balance(pre_state, proposer_index) - ) +# TODO: currently mainnet limits attester-slashings per block to 1. +# When this is increased, it should be tested to cover varrious combinations +# of duplicate slashings and overlaps of slashed attestations within the same block @with_all_phases @@ -443,35 +487,38 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root +def prepare_signed_exits(spec, state, indices): + domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) + + def create_signed_exit(index): + exit = spec.VoluntaryExit( + epoch=spec.get_current_epoch(state), + validator_index=index, + ) + signing_root = spec.compute_signing_root(exit, domain) + return spec.SignedVoluntaryExit(message=exit, signature=bls.Sign(privkeys[index], signing_root)) + + return [create_signed_exit(index) for index in indices] + + # In phase1 a committee is computed for PERSISTENT_COMMITTEE_PERIOD slots ago, # exceeding the minimal-config randao mixes memory size. +# Applies to all voluntary-exit sanity block tests. + @with_phases(['phase0']) @spec_state_test def test_voluntary_exit(spec, state): - validator_index = spec.get_active_validator_indices( - state, - spec.get_current_epoch(state) - )[-1] + validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + signed_exits = prepare_signed_exits(spec, state, [validator_index]) yield 'pre', state - voluntary_exit = spec.VoluntaryExit( - epoch=spec.get_current_epoch(state), - validator_index=validator_index, - ) - domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) - signing_root = spec.compute_signing_root(voluntary_exit, domain) - signed_voluntary_exit = spec.SignedVoluntaryExit( - message=voluntary_exit, - signature=bls.Sign(privkeys[validator_index], signing_root) - ) - # Add to state via block transition initiate_exit_block = build_empty_block_for_next_slot(spec, state) - initiate_exit_block.body.voluntary_exits.append(signed_voluntary_exit) + initiate_exit_block.body.voluntary_exits = signed_exits signed_initiate_exit_block = state_transition_and_sign_block(spec, state, initiate_exit_block) assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -486,6 +533,27 @@ def test_voluntary_exit(spec, state): assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH +@with_phases(['phase0']) +@spec_state_test +def test_double_validator_exit_same_block(spec, state): + validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + # Same index tries to exit twice, but should only be able to do so once. + signed_exits = prepare_signed_exits(spec, state, [validator_index, validator_index]) + yield 'pre', state + + # Add to state via block transition + initiate_exit_block = build_empty_block_for_next_slot(spec, state) + initiate_exit_block.body.voluntary_exits = signed_exits + signed_initiate_exit_block = state_transition_and_sign_block(spec, state, initiate_exit_block, expect_fail=True) + + yield 'blocks', [signed_initiate_exit_block] + yield 'post', None + + @with_all_phases @spec_state_test def test_balance_driven_status_transitions(spec, state): From eda249957e1891a1211738e67133ee2524a6efb6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 4 May 2020 13:20:32 -0600 Subject: [PATCH 026/127] basic generators work --- .../pyspec/eth2spec/test/helpers/rewards.py | 10 +++- .../eth2spec/test/phase_0/rewards/__init__.py | 0 .../rewards/test_get_head_deltas.py | 0 .../rewards/test_get_source_deltas.py | 0 .../rewards/test_get_target_deltas.py | 0 tests/formats/rewards/README.md | 47 +++++++++++++++++++ tests/generators/rewards/README.md | 8 ++++ tests/generators/rewards/main.py | 44 +++++++++++++++++ tests/generators/rewards/requirements.txt | 2 + 9 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/__init__.py rename tests/core/pyspec/eth2spec/test/phase_0/{epoch_processing => }/rewards/test_get_head_deltas.py (100%) rename tests/core/pyspec/eth2spec/test/phase_0/{epoch_processing => }/rewards/test_get_source_deltas.py (100%) rename tests/core/pyspec/eth2spec/test/phase_0/{epoch_processing => }/rewards/test_get_target_deltas.py (100%) create mode 100644 tests/formats/rewards/README.md create mode 100644 tests/generators/rewards/README.md create mode 100644 tests/generators/rewards/main.py create mode 100644 tests/generators/rewards/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 41497fe60..a98aa3415 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,4 +1,10 @@ from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List + + +# HACK to get the generators outputting correctly +class Deltas(Container): + delta_list: List[uint64, 2**30] def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): @@ -12,8 +18,8 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a rewards, penalties = component_delta_fn(state) - yield 'rewards', rewards - yield 'penalties', penalties + yield 'rewards', Deltas(delta_list=rewards) + yield 'penalties', Deltas(delta_list=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/__init__.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_head_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_source_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/rewards/test_get_target_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md new file mode 100644 index 000000000..dce2b5ac8 --- /dev/null +++ b/tests/formats/rewards/README.md @@ -0,0 +1,47 @@ +# Rewards tests + +The different rewards deltas sub-functions are testing individually with the test handlers, each returning the related `rewards`/`penalties`. +There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. +Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) + + +## Test case format + +### `meta.yaml` + +```yaml +description: string -- Optional description of test case, purely for debugging purposes. + Tests should use the directory name of the test case as identifier, not the description. +bls_setting: int -- see general test-format spec. +``` + +### `pre.yaml` + +A YAML-encoded `BeaconState`, the state before running the rewards sub-function. + +Also available as `pre.ssz`. + + +### `rewards.yaml` + +A YAML-encoded list of integers representing the 0th item in the return value (i.e. the rewards deltas) + +### `penalties.yaml` + +A YAML-encoded list of integers representing the 1st item in the return value (i.e. the penalties deltas) + +## Condition + +A handler of the `rewards` test-runner should process these cases, + calling the corresponding rewards deltas function (same name in spec). +This excludes all other parts of `process_rewards_and_penalties` + +The provided pre-state is ready to be input into the designated handler. + +The resulting `rewards`/`penalties` should match the return values of the +handler. Specifically the following must hold true: + +```python + rewards == handler(state)[0] + penalties == handler(state)[1] +``` diff --git a/tests/generators/rewards/README.md b/tests/generators/rewards/README.md new file mode 100644 index 000000000..60f106836 --- /dev/null +++ b/tests/generators/rewards/README.md @@ -0,0 +1,8 @@ +# Rewards + +Rewards covers the sub-functions of `process_rewards_and_penalties` for granular testing of components of the rewards function. + +A rewards test-runner can consume these sub-transition test-suites, + and handle different kinds of epoch sub-transitions by processing the cases using the specified test handler. + +Information on the format of the tests can be found in the [rewards test formats documentation](../../formats/rewards/README.md). diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py new file mode 100644 index 000000000..caffc3bd1 --- /dev/null +++ b/tests/generators/rewards/main.py @@ -0,0 +1,44 @@ +from typing import Iterable + +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.phase_0.rewards import ( + test_get_source_deltas, + test_get_target_deltas, + test_get_head_deltas, +) +from gen_base import gen_runner, gen_typing +from gen_from_tests.gen import generate_from_tests +from importlib import reload +from eth2spec.config import config_util +from eth2spec.test.context import PHASE0 + + +def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: + + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + reload(spec_phase0) + reload(spec_phase1) + return config_name + + def cases_fn() -> Iterable[gen_typing.TestCase]: + return generate_from_tests( + runner_name='rewards', + handler_name=handler_name, + src=tests_src, + fork_name=PHASE0, + ) + + return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) + + +if __name__ == "__main__": + gen_runner.run_generator("epoch_processing", [ + create_provider('get_source_deltas', test_get_source_deltas, 'minimal'), + create_provider('get_source_deltas', test_get_source_deltas, 'mainnet'), + create_provider('get_target_deltas', test_get_target_deltas, 'minimal'), + create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), + create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), + create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), + ]) diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt new file mode 100644 index 000000000..b82314298 --- /dev/null +++ b/tests/generators/rewards/requirements.txt @@ -0,0 +1,2 @@ +../../core/gen_helpers +../../../ \ No newline at end of file From 5194c1f2d2dba0c1083b6d1cd075c9f41f0aba62 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 4 May 2020 16:53:29 -0600 Subject: [PATCH 027/127] add test_get_inclusion_delay_deltas --- .../pyspec/eth2spec/test/helpers/rewards.py | 66 ++++-- .../phase_0/rewards/test_get_head_deltas.py | 10 +- .../test_get_inclusion_delay_deltas.py | 208 ++++++++++++++++++ .../phase_0/rewards/test_get_source_deltas.py | 16 +- .../phase_0/rewards/test_get_target_deltas.py | 16 +- 5 files changed, 297 insertions(+), 19 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index a98aa3415..e8cd950ce 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,3 +1,5 @@ +from random import Random + from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -7,6 +9,13 @@ class Deltas(Container): delta_list: List[uint64, 2**30] +def has_enough_for_reward(spec, state, index): + return ( + state.validators[index].effective_balance * spec.BASE_REWARD_FACTOR + > spec.integer_squareroot(spec.get_total_active_balance(state)) // spec.BASE_REWARDS_PER_EPOCH + ) + + def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): """ Run ``component_delta_fn``, yielding: @@ -25,11 +34,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) for index in spec.get_eligible_validator_indices(state): validator = state.validators[index] - enough_for_reward = ( - validator.effective_balance * spec.BASE_REWARD_FACTOR - > spec.integer_squareroot(spec.get_total_active_balance(state)) // spec.BASE_REWARDS_PER_EPOCH - ) - + enough_for_reward = has_enough_for_reward(spec, state, index) if index in matching_indices and not validator.slashed: if enough_for_reward: assert rewards[index] > 0 @@ -56,15 +61,29 @@ def test_full_all_correct(spec, state, runner): yield from runner(spec, state) -def test_half_full(spec, state, runner): +def test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): prepare_state_with_full_attestations(spec, state) - # Remove half of attestations - state.previous_epoch_attestations = state.previous_epoch_attestations[:len(state.previous_epoch_attestations) // 2] + for a in state.previous_epoch_attestations: + a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] yield from runner(spec, state) +def test_partial(spec, state, fraction_filled, runner): + prepare_state_with_full_attestations(spec, state) + + # Remove portion of attestations + num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + yield from runner(spec, state) + + +def test_half_full(spec, state, runner): + yield from test_partial(spec, state, 0.5, runner) + + def test_one_attestation_one_correct(spec, state, runner): prepare_state_with_full_attestations(spec, state) @@ -84,12 +103,16 @@ def test_with_slashed_validators(spec, state, runner): yield from runner(spec, state) -def test_some_zero_effective_balances_that_attested(spec, state, runner): +def test_some_very_low_effective_balances_that_attested(spec, state, runner): + state.balances prepare_state_with_full_attestations(spec, state) - # Set some balances to zero + # Set some balances to be very low (including 0) state.validators[0].effective_balance = 0 - state.validators[1].effective_balance = 0 + state.validators[1].effective_balance = 2 + state.validators[2].effective_balance = 10 + state.validators[3].effective_balance = 100 + state.validators[4].effective_balance = 1000 yield from runner(spec, state) @@ -97,9 +120,8 @@ def test_some_zero_effective_balances_that_attested(spec, state, runner): def test_some_zero_effective_balances_that_did_not_attest(spec, state, runner): prepare_state_with_full_attestations(spec, state) - # Set some balances to zero - attestation = state.previous_epoch_attestations[0] # Remove attestation + attestation = state.previous_epoch_attestations[0] state.previous_epoch_attestations = state.previous_epoch_attestations[1:] # Set removed indices effective balance to zero indices = spec.get_unslashed_attesting_indices(state, [attestation]) @@ -121,3 +143,21 @@ def test_full_fraction_incorrect(spec, state, correct_target, correct_head, frac pending_attestation.data.beacon_block_root = b'\x66' * 32 yield from runner(spec, state) + + +def test_full_random(spec, state, runner, rng=Random(8020)): + prepare_state_with_full_attestations(spec, state) + + for pending_attestation in state.previous_epoch_attestations: + # 1/3 have bad target + if rng.randint(0, 2) == 0: + pending_attestation.data.target.root = b'\x55' * 32 + # 1/3 have bad head + if rng.randint(0, 2) == 0: + pending_attestation.data.beacon_block_root = b'\x66' * 32 + # ~50% participation + pending_attestation.aggregation_bits = [rng.choice([True, False]) for _ in pending_attestation.aggregation_bits] + # Random inclusion delay + pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from runner(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index 24e3aaac5..b9aab92cb 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -51,8 +51,8 @@ def test_with_slashed_validators(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_head_deltas) +def test_some_very_low_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) @with_all_phases @@ -107,3 +107,9 @@ def test_full_half_incorrect_target_correct_head(spec, state): fraction_incorrect=0.5, runner=run_get_head_deltas ) + + +@with_all_phases +@spec_state_test +def test_full_random(spec, state): + yield from rewards_helpers.test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py new file mode 100644 index 000000000..b9ae9882c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -0,0 +1,208 @@ +from random import Random + +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations +from eth2spec.test.helpers.rewards import has_enough_for_reward +import eth2spec.test.helpers.rewards as rewards_helpers +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List + + +# HACK to get the generators outputting correctly +class Deltas(Container): + delta_list: List[uint64, 2**30] + + +def run_get_inclusion_delay_deltas(spec, state): + """ + Run ``get_inclusion_delay_deltas``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + + yield 'pre', state + + rewards, penalties = spec.get_inclusion_delay_deltas(state) + + yield 'rewards', Deltas(delta_list=rewards) + yield 'penalties', Deltas(delta_list=penalties) + + eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) + attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) + + rewarded_indices = set() + rewarded_proposer_indices = set() + # Ensure attesters with enough balance are rewarded for attestations + # Track those that are rewarded and track proposers that should be rewarded + for index in range(len(state.validators)): + if index in attesting_indices and has_enough_for_reward(spec, state, index): + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Track proposer of earliest included attestation for the validator defined by index + earliest_attestation = min([ + a for a in eligible_attestations + if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewarded_proposer_indices.add(earliest_attestation.proposer_index) + + # Ensure all expected proposers have been rewarded + # Track rewarde indices + proposing_indices = [a.proposer_index for a in eligible_attestations] + for index in proposing_indices: + if index in rewarded_proposer_indices: + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Ensure all expected non-rewarded indices received no reward + for index in range(len(state.validators)): + assert penalties[index] == 0 + if index not in rewarded_indices: + assert rewards[index] == 0 + + +@with_all_phases +@spec_state_test +def test_empty(spec, state): + yield from rewards_helpers.test_empty(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_full(spec, state): + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_half_full(spec, state): + yield from rewards_helpers.test_half_full(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_quarter_full(spec, state): + yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation(spec, state): + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators(spec, state): + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + spec, + state, + run_get_inclusion_delay_deltas + ) + + +@with_all_phases +@spec_state_test +def test_full_random(spec, state): + yield from rewards_helpers.test_full_random(spec, state, run_get_inclusion_delay_deltas) + + +@with_all_phases +@spec_state_test +def test_full_delay_one_slot(spec, state): + prepare_state_with_full_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += 1 + + yield from run_get_inclusion_delay_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_delay_max_slots(spec, state): + prepare_state_with_full_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += spec.SLOTS_PER_EPOCH + + yield from run_get_inclusion_delay_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_mixed_delay(spec, state): + rng = Random(1234) + + prepare_state_with_full_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from run_get_inclusion_delay_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_proposer_not_in_attestations(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Get an attestation where the proposer is not in the committee + non_proposer_attestations = [] + for a in state.previous_epoch_attestations: + if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): + non_proposer_attestations.append(a) + + assert any(non_proposer_attestations) + state.previous_epoch_attestations = non_proposer_attestations + + yield from run_get_inclusion_delay_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_duplicate_attestations_at_later_slots(spec, state): + prepare_state_with_full_attestations(spec, state) + + # Remove 2/3 of attestations to make it more interesting + num_attestations = int(len(state.previous_epoch_attestations) * 0.33) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + # Get map of the proposer at each slot to make valid-looking duplicate attestations + per_slot_proposers = { + (a.data.slot + a.inclusion_delay): a.proposer_index + for a in state.previous_epoch_attestations + } + max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) + later_attestations = [] + for a in state.previous_epoch_attestations: + # Do not create later duplicate if goes into next epoch + if a.data.slot + a.inclusion_delay >= max_slot: + continue + later_a = a.copy() + later_a.inclusion_delay += 1 + later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] + later_attestations.append(later_a) + + assert any(later_attestations) + + state.previous_epoch_attestations = sorted( + state.previous_epoch_attestations + later_attestations, + key=lambda a: a.data.slot + a.inclusion_delay + ) + + yield from run_get_inclusion_delay_deltas(spec, state) + + +@with_all_phases +@spec_state_test +def test_all_balances_too_low_for_reward(spec, state): + prepare_state_with_full_attestations(spec, state) + + for index in range(len(state.validators)): + state.validators[index].effective_balance = 10 + + yield from run_get_inclusion_delay_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index da47bd204..9063abfc6 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -37,6 +37,12 @@ def test_half_full(spec, state): yield from rewards_helpers.test_half_full(spec, state, run_get_source_deltas) +@with_all_phases +@spec_state_test +def test_full_but_partial_participation(spec, state): + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_source_deltas) + + @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): @@ -51,8 +57,8 @@ def test_with_slashed_validators(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_source_deltas) +def test_some_very_low_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_source_deltas) @with_all_phases @@ -114,3 +120,9 @@ def test_full_half_incorrect_target_correct_head(spec, state): fraction_incorrect=0.5, runner=run_get_source_deltas ) + + +@with_all_phases +@spec_state_test +def test_full_random(spec, state): + yield from rewards_helpers.test_full_random(spec, state, run_get_source_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index 406471c67..ff20014c8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -37,6 +37,12 @@ def test_half_full(spec, state): yield from rewards_helpers.test_half_full(spec, state, run_get_target_deltas) +@with_all_phases +@spec_state_test +def test_full_but_partial_participation(spec, state): + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_target_deltas) + + @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): @@ -51,8 +57,8 @@ def test_with_slashed_validators(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_attested(spec, state, run_get_target_deltas) +def test_some_very_low_effective_balances_that_attested(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_target_deltas) @with_all_phases @@ -107,3 +113,9 @@ def test_full_half_incorrect_target_correct_head(spec, state): fraction_incorrect=0.5, runner=run_get_target_deltas ) + + +@with_all_phases +@spec_state_test +def test_full_random(spec, state): + yield from rewards_helpers.test_full_random(spec, state, run_get_target_deltas) From 8f569a8ddc05bc5d13c94a4dc7e2465c3eb1448a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 4 May 2020 20:58:42 -0600 Subject: [PATCH 028/127] add inactivity penalty deltas tests --- .../pyspec/eth2spec/test/helpers/rewards.py | 14 +- .../phase_0/rewards/test_get_head_deltas.py | 14 +- .../test_get_inactivity_penalty_deltas.py | 204 ++++++++++++++++++ .../phase_0/rewards/test_get_source_deltas.py | 8 +- .../phase_0/rewards/test_get_target_deltas.py | 8 +- tests/generators/rewards/main.py | 6 + 6 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index e8cd950ce..09c99c3ac 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -32,7 +32,13 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - for index in spec.get_eligible_validator_indices(state): + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + if index not in eligible_indices: + assert rewards[index] == 0 + assert penalties[index] == 0 + continue + validator = state.validators[index] enough_for_reward = has_enough_for_reward(spec, state, index) if index in matching_indices and not validator.slashed: @@ -117,7 +123,7 @@ def test_some_very_low_effective_balances_that_attested(spec, state, runner): yield from runner(spec, state) -def test_some_zero_effective_balances_that_did_not_attest(spec, state, runner): +def test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): prepare_state_with_full_attestations(spec, state) # Remove attestation @@ -125,8 +131,8 @@ def test_some_zero_effective_balances_that_did_not_attest(spec, state, runner): state.previous_epoch_attestations = state.previous_epoch_attestations[1:] # Set removed indices effective balance to zero indices = spec.get_unslashed_attesting_indices(state, [attestation]) - for index in indices: - state.validators[index].effective_balance = 0 + for i, index in enumerate(indices): + state.validators[index].effective_balance = i yield from runner(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index b9aab92cb..237920732 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -37,6 +37,12 @@ def test_half_full(spec, state): yield from rewards_helpers.test_half_full(spec, state, run_get_head_deltas) +@with_all_phases +@spec_state_test +def test_full_but_partial_participation(spec, state): + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_head_deltas) + + @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): @@ -57,8 +63,12 @@ def test_some_very_low_effective_balances_that_attested(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_head_deltas) +def test_some_very_low_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + spec, + state, + run_get_head_deltas, + ) @with_all_phases diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py new file mode 100644 index 000000000..a81451e38 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -0,0 +1,204 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.rewards import has_enough_for_reward +from eth2spec.test.helpers.state import next_epoch +import eth2spec.test.helpers.rewards as rewards_helpers +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List + + +# HACK to get the generators outputting correctly +class Deltas(Container): + delta_list: List[uint64, 2**30] + + +def run_get_inactivity_penalty_deltas(spec, state): + """ + Run ``get_inactivity_penalty_deltas``, yielding: + - pre-state ('pre') + - rewards ('rewards') + - penalties ('penalties') + """ + + yield 'pre', state + + rewards, penalties = spec.get_inactivity_penalty_deltas(state) + + yield 'rewards', Deltas(delta_list=rewards) + yield 'penalties', Deltas(delta_list=penalties) + + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + + finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + assert rewards[index] == 0 + if index not in eligible_indices: + assert penalties[index] == 0 + continue + + if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if not has_enough_for_reward(spec, state, index): + assert penalties[index] == 0 + elif index in matching_attesting_indices: + assert penalties[index] == base_penalty + else: + assert penalties[index] > base_penalty + else: + assert penalties[index] == 0 + + +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +@with_all_phases +@spec_state_test +def test_empty_no_leak(spec, state): + yield from rewards_helpers.test_empty(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_empty_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_empty(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_no_leak(spec, state): + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_half_full_no_leak(spec, state): + yield from rewards_helpers.test_half_full(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_half_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_half_full(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_quarter_full_no_leak(spec, state): + yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_quarter_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation_no_leak(spec, state): + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators_no_leak(spec, state): + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_full_random_no_leak(spec, state): + yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_random_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_random_five_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=5) + yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_full_random_ten_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=10) + yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index 9063abfc6..842728db8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -63,8 +63,12 @@ def test_some_very_low_effective_balances_that_attested(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_source_deltas) +def test_some_very_low_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + spec, + state, + run_get_source_deltas, + ) # diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index ff20014c8..ad2a65d2a 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -63,8 +63,12 @@ def test_some_very_low_effective_balances_that_attested(spec, state): @with_all_phases @spec_state_test -def test_some_zero_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_zero_effective_balances_that_did_not_attest(spec, state, run_get_target_deltas) +def test_some_very_low_effective_balances_that_did_not_attest(spec, state): + yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + spec, + state, + run_get_target_deltas, + ) @with_all_phases diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index caffc3bd1..fd95dcfaa 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -6,6 +6,8 @@ from eth2spec.test.phase_0.rewards import ( test_get_source_deltas, test_get_target_deltas, test_get_head_deltas, + test_get_inclusion_delay_deltas, + test_get_inactivity_penalty_deltas, ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests @@ -41,4 +43,8 @@ if __name__ == "__main__": create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), + create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'minimal'), + create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'mainnet'), + create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'minimal'), + create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'mainnet'), ]) From 3f250f7dd32296ae50a417424665b4279f9f588e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 4 May 2020 21:05:10 -0600 Subject: [PATCH 029/127] PR feedback --- specs/phase0/beacon-chain.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 95f06f125..82fbb2856 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1364,9 +1364,12 @@ def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorInde ``` ```python -def compute_attestation_component_deltas(state: BeaconState, - attestations: Sequence[PendingAttestation] - ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_attestation_component_deltas(state: BeaconState, + attestations: Sequence[PendingAttestation] + ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Helper with shared logic for use by get source, target, and head deltas functions + """ rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] total_balance = get_total_active_balance(state) @@ -1390,7 +1393,7 @@ def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei Return attester micro-rewards/penalties for source-vote for each validator. """ matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) - return compute_attestation_component_deltas(state, matching_source_attestations) + return get_attestation_component_deltas(state, matching_source_attestations) ``` ```python @@ -1399,7 +1402,7 @@ def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei Return attester micro-rewards/penalties for target-vote for each validator. """ matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - return compute_attestation_component_deltas(state, matching_target_attestations) + return get_attestation_component_deltas(state, matching_target_attestations) ``` ```python @@ -1408,7 +1411,7 @@ def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]] Return attester micro-rewards/penalties for head-vote for each validator. """ matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) - return compute_attestation_component_deltas(state, matching_head_attestations) + return get_attestation_component_deltas(state, matching_head_attestations) ``` ```python @@ -1465,18 +1468,12 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence _, inactivity_penalties = get_inactivity_penalty_deltas(state) rewards = [ - source_rewards[i] - + target_rewards[i] - + head_rewards[i] - + inclusion_delay_rewards[i] + source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] for i in range(len(state.validators)) ] penalties = [ - source_penalties[i] - + target_penalties[i] - + head_penalties[i] - + inactivity_penalties[i] + source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] for i in range(len(state.validators)) ] From f35106d9eec822dd02ca768dc5bc5918a6ad05ba Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 09:43:25 -0600 Subject: [PATCH 030/127] add comment for helper -- has_enouh_for_reward --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 09c99c3ac..1b428891b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -10,6 +10,12 @@ class Deltas(Container): def has_enough_for_reward(spec, state, index): + """ + Check if base_reward will be non-zero. + + At very low balances, it is possible for a validator have a positive effective_balance + but a zero base reward. + """ return ( state.validators[index].effective_balance * spec.BASE_REWARD_FACTOR > spec.integer_squareroot(spec.get_total_active_balance(state)) // spec.BASE_REWARDS_PER_EPOCH From 5b3ed8a3e73b0a3825a1ac7ffa469e3759a03922 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 09:45:24 -0600 Subject: [PATCH 031/127] bump VERSION.txt to 0.12.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index a8839f70d..d33c3a212 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file From 38f29ba0a851718bd31c47783f0e9b21360aa813 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 11:44:53 -0600 Subject: [PATCH 032/127] remove PERSISTENT_COMMITTEE_PERIOD in favor of SHARD_COMMITTEE_PERIOD --- configs/mainnet.yaml | 6 ++-- configs/minimal.yaml | 4 +-- specs/phase0/beacon-chain.md | 5 ++-- specs/phase1/beacon-chain.md | 1 - .../test_process_voluntary_exit.py | 30 +++++++++---------- .../test/phase_0/sanity/test_blocks.py | 6 ++-- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 7691b7481..60bd1c087 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,8 +94,8 @@ EPOCHS_PER_ETH1_VOTING_PERIOD: 32 SLOTS_PER_HISTORICAL_ROOT: 8192 # 2**8 (= 256) epochs ~27 hours MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**11 (= 2,048) epochs 9 days -PERSISTENT_COMMITTEE_PERIOD: 2048 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 # 2**6 (= 64) epochs ~7 hours MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes @@ -175,8 +175,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs | ~27 hours LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs | ~27 hours -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 38f12d297..5c1511e6d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -95,7 +95,7 @@ SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit -PERSISTENT_COMMITTEE_PERIOD: 128 +SHARD_COMMITTEE_PERIOD: 64 # [customized] fast catchup crosslinks MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs @@ -178,8 +178,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 62615b0e7..0b66cff3c 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -223,7 +223,8 @@ The following values are (non-configurable) constants used throughout the specif | `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**5` (= 32) | epochs | ~3.4 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | +| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | + ### State list lengths @@ -1688,7 +1689,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Exits must specify an epoch when they become valid; they are not valid before then assert get_current_epoch(state) >= voluntary_exit.epoch # Verify the validator has been active long enough - assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) signing_root = compute_signing_root(voluntary_exit, domain) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 35f6a6425..fc5fe9e14 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -100,7 +100,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | | `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 19915750f..9464f80aa 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -34,8 +34,8 @@ def run_voluntary_exit_processing(spec, state, signed_voluntary_exit, valid=True @with_all_phases @spec_state_test def test_success(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -53,8 +53,8 @@ def test_success(spec, state): @spec_state_test @always_bls def test_invalid_signature(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -71,8 +71,8 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_success_exit_queue(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) @@ -115,8 +115,8 @@ def test_success_exit_queue(spec, state): @with_all_phases @spec_state_test def test_default_exit_epoch_subsequent_exit(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -137,8 +137,8 @@ def test_default_exit_epoch_subsequent_exit(spec, state): @with_all_phases @spec_state_test def test_validator_exit_in_future(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -156,8 +156,8 @@ def test_validator_exit_in_future(spec, state): @with_all_phases @spec_state_test def test_validator_invalid_validator_index(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -190,8 +190,8 @@ def test_validator_not_active(spec, state): @with_all_phases @spec_state_test def test_validator_already_exited(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow validator able to exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -218,7 +218,7 @@ def test_validator_not_active_long_enough(spec, state): assert ( current_epoch - state.validators[validator_index].activation_epoch < - spec.PERSISTENT_COMMITTEE_PERIOD + spec.SHARD_COMMITTEE_PERIOD ) yield from run_voluntary_exit_processing(spec, state, signed_voluntary_exit, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 29a9dcca2..6ae71c16e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -448,7 +448,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root -# In phase1 a committee is computed for PERSISTENT_COMMITTEE_PERIOD slots ago, +# In phase1 a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, # exceeding the minimal-config randao mixes memory size. @with_phases(['phase0']) @spec_state_test @@ -458,8 +458,8 @@ def test_voluntary_exit(spec, state): spec.get_current_epoch(state) )[-1] - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH yield 'pre', state From f60f8ca332a55b5512179d96267f23ae86849b24 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 May 2020 02:10:43 +0800 Subject: [PATCH 033/127] Fix README spec links --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fed65eedb..16f0dc97a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi ### Phase 0 * [The Beacon Chain](specs/phase0/beacon-chain.md) -* [Fork Choice](specs/phase0/fork-choice.md) +* [Beacon Chain Fork Choice](specs/phase0/fork-choice.md) * [Deposit Contract](specs/phase0/deposit-contract.md) * [Honest Validator](specs/phase0/validator.md) * [P2P Networking](specs/phase0/p2p-interface.md) @@ -22,8 +22,9 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi * [From Phase 0 to Phase 1](specs/phase1/phase1-fork.md) * [The Beacon Chain for Shards](specs/phase1/beacon-chain.md) * [Custody Game](specs/phase1/custody-game.md) -* [Shard Transition and Fraud Proofs](specs/phase1/fraud-proofs.md) +* [Shard Transition and Fraud Proofs](specs/phase1/shard-transition.md) * [Light client syncing protocol](specs/phase1/light-client-sync.md) +* [Beacon Chain Fork Choice for Shards](specs/phase1/fork-choice.md) ### Phase 2 From d26cfd2e599abc0f8f2691d21406fdaddf9de300 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 13:08:41 -0600 Subject: [PATCH 034/127] Apply suggestions from code review from @hwwhww Co-authored-by: Hsiao-Wei Wang --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 1b428891b..17849e7d1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -99,7 +99,7 @@ def test_half_full(spec, state, runner): def test_one_attestation_one_correct(spec, state, runner): prepare_state_with_full_attestations(spec, state) - # Remove half of attestations + # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] yield from runner(spec, state) @@ -120,11 +120,9 @@ def test_some_very_low_effective_balances_that_attested(spec, state, runner): prepare_state_with_full_attestations(spec, state) # Set some balances to be very low (including 0) - state.validators[0].effective_balance = 0 - state.validators[1].effective_balance = 2 - state.validators[2].effective_balance = 10 - state.validators[3].effective_balance = 100 - state.validators[4].effective_balance = 1000 + assert len(state.validators) >= 5: + for i, index in enumerate(5): + state.validators[index].effective_balance = i yield from runner(spec, state) @@ -135,7 +133,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state, runne # Remove attestation attestation = state.previous_epoch_attestations[0] state.previous_epoch_attestations = state.previous_epoch_attestations[1:] - # Set removed indices effective balance to zero + # Set removed indices effective balance to very low amount indices = spec.get_unslashed_attesting_indices(state, [attestation]) for i, index in enumerate(indices): state.validators[index].effective_balance = i From b2dfb6cebe9e40e2f2bef2cb4e03397893d76e1a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 13:33:44 -0600 Subject: [PATCH 035/127] PR feedback from @hwwhww --- .../pyspec/eth2spec/test/helpers/rewards.py | 32 +++++++-------- .../phase_0/rewards/test_get_head_deltas.py | 26 ++++++------ .../test_get_inactivity_penalty_deltas.py | 40 +++++++++---------- .../test_get_inclusion_delay_deltas.py | 19 ++++----- .../phase_0/rewards/test_get_source_deltas.py | 30 ++++++++------ .../phase_0/rewards/test_get_target_deltas.py | 30 ++++++++------ 6 files changed, 93 insertions(+), 84 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 17849e7d1..deee1b120 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -61,19 +61,19 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 -def test_empty(spec, state, runner): +def run_test_empty(spec, state, runner): # Do not add any attestations to state yield from runner(spec, state) -def test_full_all_correct(spec, state, runner): +def run_test_full_all_correct(spec, state, runner): prepare_state_with_full_attestations(spec, state) yield from runner(spec, state) -def test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): +def run_test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): prepare_state_with_full_attestations(spec, state) for a in state.previous_epoch_attestations: @@ -82,7 +82,7 @@ def test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): yield from runner(spec, state) -def test_partial(spec, state, fraction_filled, runner): +def run_test_partial(spec, state, fraction_filled, runner): prepare_state_with_full_attestations(spec, state) # Remove portion of attestations @@ -92,11 +92,11 @@ def test_partial(spec, state, fraction_filled, runner): yield from runner(spec, state) -def test_half_full(spec, state, runner): - yield from test_partial(spec, state, 0.5, runner) +def run_test_half_full(spec, state, runner): + yield from run_test_partial(spec, state, 0.5, runner) -def test_one_attestation_one_correct(spec, state, runner): +def run_test_one_attestation_one_correct(spec, state, runner): prepare_state_with_full_attestations(spec, state) # Remove all attestations except for the first one @@ -105,7 +105,7 @@ def test_one_attestation_one_correct(spec, state, runner): yield from runner(spec, state) -def test_with_slashed_validators(spec, state, runner): +def run_test_with_slashed_validators(spec, state, runner): prepare_state_with_full_attestations(spec, state) # Slash half of validators @@ -115,19 +115,19 @@ def test_with_slashed_validators(spec, state, runner): yield from runner(spec, state) -def test_some_very_low_effective_balances_that_attested(spec, state, runner): +def run_test_some_very_low_effective_balances_that_attested(spec, state, runner): state.balances prepare_state_with_full_attestations(spec, state) # Set some balances to be very low (including 0) - assert len(state.validators) >= 5: - for i, index in enumerate(5): + assert len(state.validators) >= 5 + for i, index in enumerate(range(5)): state.validators[index].effective_balance = i yield from runner(spec, state) -def test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): +def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): prepare_state_with_full_attestations(spec, state) # Remove attestation @@ -141,7 +141,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state, runne yield from runner(spec, state) -def test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): +def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): prepare_state_with_full_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified @@ -155,14 +155,14 @@ def test_full_fraction_incorrect(spec, state, correct_target, correct_head, frac yield from runner(spec, state) -def test_full_random(spec, state, runner, rng=Random(8020)): +def run_test_full_random(spec, state, runner, rng=Random(8020)): prepare_state_with_full_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: - # 1/3 have bad target + # ~1/3 have bad target if rng.randint(0, 2) == 0: pending_attestation.data.target.root = b'\x55' * 32 - # 1/3 have bad head + # ~1/3 have bad head if rng.randint(0, 2) == 0: pending_attestation.data.beacon_block_root = b'\x66' * 32 # ~50% participation diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index 237920732..19a3e7191 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -22,49 +22,49 @@ def run_get_head_deltas(spec, state): @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.test_empty(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.test_half_full(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( spec, state, run_get_head_deltas, @@ -74,7 +74,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state): @with_all_phases @spec_state_test def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -86,7 +86,7 @@ def test_full_half_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -98,7 +98,7 @@ def test_full_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=False, @@ -110,7 +110,7 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=True, @@ -122,4 +122,4 @@ def test_full_half_incorrect_target_correct_head(spec, state): @with_all_phases @spec_state_test def test_full_random(spec, state): - yield from rewards_helpers.test_full_random(spec, state, run_get_head_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index a81451e38..fb24e09d8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -60,85 +60,85 @@ def transition_state_to_leak(spec, state, epochs=None): @with_all_phases @spec_state_test def test_empty_no_leak(spec, state): - yield from rewards_helpers.test_empty(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_empty_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_empty(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_no_leak(spec, state): - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_half_full_no_leak(spec, state): - yield from rewards_helpers.test_half_full(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_half_full_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_half_full(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_quarter_full_no_leak(spec, state): - yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_quarter_full_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation_no_leak(spec, state): - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators_no_leak(spec, state): - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( spec, state, run_get_inactivity_penalty_deltas, @@ -149,7 +149,7 @@ def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): @spec_state_test def test_some_very_low_effective_balances_that_attested_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( spec, state, run_get_inactivity_penalty_deltas, @@ -159,7 +159,7 @@ def test_some_very_low_effective_balances_that_attested_leak(spec, state): @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( spec, state, run_get_inactivity_penalty_deltas, @@ -170,7 +170,7 @@ def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, stat @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( spec, state, run_get_inactivity_penalty_deltas, @@ -180,25 +180,25 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_all_phases @spec_state_test def test_full_random_no_leak(spec, state): - yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_random_leak(spec, state): transition_state_to_leak(spec, state) - yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_random_five_epoch_leak(spec, state): transition_state_to_leak(spec, state, epochs=5) - yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) @with_all_phases @spec_state_test def test_full_random_ten_epoch_leak(spec, state): transition_state_to_leak(spec, state, epochs=10) - yield from rewards_helpers.test_full_random(spec, state, run_get_inactivity_penalty_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index b9ae9882c..9eda75290 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -64,43 +64,43 @@ def run_get_inclusion_delay_deltas(spec, state): @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.test_empty(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_full(spec, state): - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.test_half_full(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_quarter_full(spec, state): - yield from rewards_helpers.test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( spec, state, run_get_inclusion_delay_deltas @@ -110,7 +110,7 @@ def test_some_very_low_effective_balances_that_attested(spec, state): @with_all_phases @spec_state_test def test_full_random(spec, state): - yield from rewards_helpers.test_full_random(spec, state, run_get_inclusion_delay_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_inclusion_delay_deltas) @with_all_phases @@ -179,7 +179,8 @@ def test_duplicate_attestations_at_later_slots(spec, state): max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) later_attestations = [] for a in state.previous_epoch_attestations: - # Do not create later duplicate if goes into next epoch + # Only have proposers for previous epoch so do not create later + # duplicate if slot exceeds the max slot in previous_epoch_attestations if a.data.slot + a.inclusion_delay >= max_slot: continue later_a = a.copy() diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index 842728db8..efdc1fcb0 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -22,49 +22,53 @@ def run_get_source_deltas(spec, state): @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.test_empty(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.test_half_full(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_source_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( + spec, + state, + run_get_source_deltas + ) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( spec, state, run_get_source_deltas, @@ -81,7 +85,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state): @with_all_phases @spec_state_test def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -93,7 +97,7 @@ def test_full_half_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -105,7 +109,7 @@ def test_full_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=False, @@ -117,7 +121,7 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=True, @@ -129,4 +133,4 @@ def test_full_half_incorrect_target_correct_head(spec, state): @with_all_phases @spec_state_test def test_full_random(spec, state): - yield from rewards_helpers.test_full_random(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_source_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index ad2a65d2a..ad7198c84 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -22,49 +22,53 @@ def run_get_target_deltas(spec, state): @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.test_empty(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_empty(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.test_full_all_correct(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.test_half_full(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_half_full(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.test_full_but_partial_participation(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.test_one_attestation_one_correct(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.test_with_slashed_validators(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_target_deltas) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_attested(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( + spec, + state, + run_get_target_deltas + ) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.test_some_very_low_effective_balances_that_did_not_attest( + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( spec, state, run_get_target_deltas, @@ -74,7 +78,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state): @with_all_phases @spec_state_test def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -86,7 +90,7 @@ def test_full_half_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, correct_head=False, @@ -98,7 +102,7 @@ def test_full_correct_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=False, @@ -110,7 +114,7 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): @with_all_phases @spec_state_test def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.test_full_fraction_incorrect( + yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, correct_head=True, @@ -122,4 +126,4 @@ def test_full_half_incorrect_target_correct_head(spec, state): @with_all_phases @spec_state_test def test_full_random(spec, state): - yield from rewards_helpers.test_full_random(spec, state, run_get_target_deltas) + yield from rewards_helpers.run_test_full_random(spec, state, run_get_target_deltas) From b109e7da5a5c179e1081413c7bc3f8f8db95d95c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 14:46:02 -0600 Subject: [PATCH 036/127] add test for inconsistent head and target in attestation fork choice --- .../test/fork_choice/test_on_attestation.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..b2d33d0aa 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import PHASE0, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch +from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot def run_on_attestation(spec, state, store, attestation, valid=True): @@ -116,6 +116,44 @@ def test_on_attestation_mismatched_target_and_slot(spec, state): run_on_attestation(spec, state, store, attestation, False) +@with_all_phases +@spec_state_test +def test_on_attestation_inconsistent_target_and_head(spec, state): + store = spec.get_forkchoice_store(state) + spec.on_tick(store, store.time + 2 * spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) + + # Create chain 1 as empty chain between genesis and start of 1st epoch + target_state_1 = state.copy() + next_epoch(spec, target_state_1) + + # Create chain 2 with different block in chain from chain 1 from chain 1 from chain 1 from chain 1 + target_state_2 = state.copy() + diff_block = build_empty_block_for_next_slot(spec, target_state_2) + signed_diff_block = state_transition_and_sign_block(spec, target_state_2, diff_block) + spec.on_block(store, signed_diff_block) + next_epoch(spec, target_state_2) + next_slot(spec, target_state_2) + + # Create and store block new head block on target state 1 + head_block = build_empty_block_for_next_slot(spec, target_state_1) + signed_head_block = state_transition_and_sign_block(spec, target_state_1, head_block) + spec.on_block(store, signed_head_block) + + # Attest to head of chain 1 + attestation = get_valid_attestation(spec, target_state_1, slot=head_block.slot, signed=False) + epoch = spec.compute_epoch_at_slot(attestation.data.slot) + + # Set attestation target to be from chain 2 + attestation.data.target = spec.Checkpoint(epoch=epoch, root=spec.get_block_root(target_state_2, epoch)) + sign_attestation(spec, state, attestation) + + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + 1 + assert spec.compute_epoch_at_slot(attestation.data.slot) == spec.GENESIS_EPOCH + 1 + assert spec.get_block_root(target_state_1, epoch) != attestation.data.target.root + + run_on_attestation(spec, state, store, attestation, False) + + @with_all_phases @spec_state_test def test_on_attestation_target_not_in_store(spec, state): From 4f401133e14fac50ea9fce42264b3c09f6ac39b0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 15:37:14 -0600 Subject: [PATCH 037/127] address PR feedback from @protolambda --- specs/phase0/beacon-chain.md | 4 +-- .../pyspec/eth2spec/test/helpers/rewards.py | 8 +++--- .../test_get_inactivity_penalty_deltas.py | 10 ++----- .../test_get_inclusion_delay_deltas.py | 11 ++------ tests/formats/rewards/README.md | 27 +++++++++++-------- 5 files changed, 26 insertions(+), 34 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 82fbb2856..fe8fcc9bd 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1370,8 +1370,8 @@ def get_attestation_component_deltas(state: BeaconState, """ Helper with shared logic for use by get source, target, and head deltas functions """ - rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) total_balance = get_total_active_balance(state) unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) attesting_balance = get_total_balance(state, unslashed_attesting_indices) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index deee1b120..6e470d9e9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,12 +1,13 @@ from random import Random +from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations from eth2spec.utils.ssz.ssz_typing import Container, uint64, List -# HACK to get the generators outputting correctly class Deltas(Container): - delta_list: List[uint64, 2**30] + rewards: List[uint64, spec_phase0.VALIDATOR_REGISTRY_LIMIT] + penalties: List[uint64, spec_phase0.VALIDATOR_REGISTRY_LIMIT] def has_enough_for_reward(spec, state, index): @@ -33,8 +34,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a rewards, penalties = component_delta_fn(state) - yield 'rewards', Deltas(delta_list=rewards) - yield 'penalties', Deltas(delta_list=penalties) + yield 'deltas', Deltas(rewards=rewards, penalties=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index fb24e09d8..657acbdbb 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -1,13 +1,8 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.rewards import has_enough_for_reward from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.rewards import Deltas import eth2spec.test.helpers.rewards as rewards_helpers -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List - - -# HACK to get the generators outputting correctly -class Deltas(Container): - delta_list: List[uint64, 2**30] def run_get_inactivity_penalty_deltas(spec, state): @@ -22,8 +17,7 @@ def run_get_inactivity_penalty_deltas(spec, state): rewards, penalties = spec.get_inactivity_penalty_deltas(state) - yield 'rewards', Deltas(delta_list=rewards) - yield 'penalties', Deltas(delta_list=penalties) + yield 'deltas', Deltas(rewards=rewards, penalties=penalties) matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index 9eda75290..320f59a9d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -2,14 +2,8 @@ from random import Random from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations -from eth2spec.test.helpers.rewards import has_enough_for_reward +from eth2spec.test.helpers.rewards import Deltas, has_enough_for_reward import eth2spec.test.helpers.rewards as rewards_helpers -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List - - -# HACK to get the generators outputting correctly -class Deltas(Container): - delta_list: List[uint64, 2**30] def run_get_inclusion_delay_deltas(spec, state): @@ -24,8 +18,7 @@ def run_get_inclusion_delay_deltas(spec, state): rewards, penalties = spec.get_inclusion_delay_deltas(state) - yield 'rewards', Deltas(delta_list=rewards) - yield 'penalties', Deltas(delta_list=penalties) + yield 'deltas', Deltas(rewards=rewards, penalties=penalties) eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index dce2b5ac8..bffe2e7de 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -4,7 +4,6 @@ The different rewards deltas sub-functions are testing individually with the tes There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) - ## Test case format ### `meta.yaml` @@ -12,23 +11,29 @@ Hence, the format is shared between each test-handler. (See test condition docum ```yaml description: string -- Optional description of test case, purely for debugging purposes. Tests should use the directory name of the test case as identifier, not the description. -bls_setting: int -- see general test-format spec. ``` +_Note_: No signature verification happens within rewards sub-functions. These + tests can safely be run with or without BLS enabled. + ### `pre.yaml` A YAML-encoded `BeaconState`, the state before running the rewards sub-function. Also available as `pre.ssz`. +### `deltas.yaml` -### `rewards.yaml` +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards sub-function -A YAML-encoded list of integers representing the 0th item in the return value (i.e. the rewards deltas) +Where `Deltas` is defined as: +```python +class Deltas(Container): + rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] + penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] +``` -### `penalties.yaml` - -A YAML-encoded list of integers representing the 1st item in the return value (i.e. the penalties deltas) +Also available as `rewards.ssz`. ## Condition @@ -38,10 +43,10 @@ This excludes all other parts of `process_rewards_and_penalties` The provided pre-state is ready to be input into the designated handler. -The resulting `rewards`/`penalties` should match the return values of the -handler. Specifically the following must hold true: +The provided `deltas` should match the return values of the + handler. Specifically the following must hold true: ```python - rewards == handler(state)[0] - penalties == handler(state)[1] + deltas.rewards == handler(state)[0] + deltas.penalties == handler(state)[1] ``` From f0742b2f2def1de421469919384faea05db953c5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 16:28:29 -0600 Subject: [PATCH 038/127] add exited tests for rewards. make some valiators exited/withdrawable in slashed tests --- .../pyspec/eth2spec/test/helpers/rewards.py | 44 +++++++++++++++++-- .../phase_0/rewards/test_get_head_deltas.py | 6 +++ .../test_get_inactivity_penalty_deltas.py | 13 ++++++ .../test_get_inclusion_delay_deltas.py | 6 +++ .../phase_0/rewards/test_get_source_deltas.py | 6 +++ 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 6e470d9e9..fa041dae4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,6 +2,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations +from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -61,6 +62,32 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def exit_random_validators(spec, state, rng): + if spec.get_current_epoch(state) < 5: + # Move epochs forward to allow for some validators already exited/withdrawable + for _ in range(5): + next_epoch(spec, state) + + current_epoch = spec.get_current_epoch(state) + # Exit ~1/2 of validators + for validator in state.validators: + if rng.choice([True, False]): + continue + + validator.exit_epoch = rng.choice([current_epoch - 1, current_epoch - 2, current_epoch - 3]) + # ~1/2 are withdrawable + if rng.choice([True, False]): + validator.withdrawable_epoch = current_epoch + else: + validator.withdrawable_epoch = current_epoch + 1 + + +def slash_random_validators(spec, state, rng): + # Slash ~1/2 of validators + for validator in state.validators: + validator.slashed = rng.choice([True, False]) + + def run_test_empty(spec, state, runner): # Do not add any attestations to state @@ -105,12 +132,18 @@ def run_test_one_attestation_one_correct(spec, state, runner): yield from runner(spec, state) -def run_test_with_slashed_validators(spec, state, runner): +def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): + exit_random_validators(spec, state, rng) prepare_state_with_full_attestations(spec, state) - # Slash half of validators - for validator in state.validators[:len(state.validators) // 2]: - validator.slashed = True + yield from runner(spec, state) + + +def run_test_with_slashed_validators(spec, state, runner, rng=Random(3322)): + exit_random_validators(spec, state, rng) + slash_random_validators(spec, state, rng) + + prepare_state_with_full_attestations(spec, state) yield from runner(spec, state) @@ -156,6 +189,9 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_random(spec, state, runner, rng=Random(8020)): + exit_random_validators(spec, state, rng) + slash_random_validators(spec, state, rng) + prepare_state_with_full_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index 19a3e7191..ffaad15af 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -49,6 +49,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) +@with_all_phases +@spec_state_test +def test_with_exited_validators(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_head_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index 657acbdbb..52ad631ca 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -116,6 +116,19 @@ def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) +@with_all_phases +@spec_state_test +def test_with_exited_validators_no_leak(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators_no_leak(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index 320f59a9d..29ac10131 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -84,6 +84,12 @@ def test_full_but_partial_participation(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) +@with_all_phases +@spec_state_test +def test_with_exited_validators(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inclusion_delay_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index efdc1fcb0..120260d82 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -49,6 +49,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) +@with_all_phases +@spec_state_test +def test_with_exited_validators(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_source_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): From 4ffa0dba60a4d8670f1a50ddead4f7bce2a66351 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 May 2020 16:31:40 -0600 Subject: [PATCH 039/127] Apply suggestions from code review "rewards/penalties" -> "deltas" in throughout test comments/descriptions Co-authored-by: Diederik Loerakker --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 3 +-- .../eth2spec/test/phase_0/rewards/test_get_head_deltas.py | 3 +-- .../test/phase_0/rewards/test_get_inactivity_penalty_deltas.py | 3 +-- .../test/phase_0/rewards/test_get_inclusion_delay_deltas.py | 3 +-- .../eth2spec/test/phase_0/rewards/test_get_source_deltas.py | 3 +-- .../eth2spec/test/phase_0/rewards/test_get_target_deltas.py | 3 +-- tests/formats/rewards/README.md | 2 +- 7 files changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index fa041dae4..eaeb9252e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -28,8 +28,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a """ Run ``component_delta_fn``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index ffaad15af..2e4b9dbbc 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -7,8 +7,7 @@ def run_get_head_deltas(spec, state): """ Run ``get_head_deltas``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield from run_attestation_component_deltas( diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index 52ad631ca..4940cdc63 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -9,8 +9,7 @@ def run_get_inactivity_penalty_deltas(spec, state): """ Run ``get_inactivity_penalty_deltas``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index 29ac10131..526d135ed 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -10,8 +10,7 @@ def run_get_inclusion_delay_deltas(spec, state): """ Run ``get_inclusion_delay_deltas``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index 120260d82..54f8f3b5d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -7,8 +7,7 @@ def run_get_source_deltas(spec, state): """ Run ``get_source_deltas``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield from run_attestation_component_deltas( diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index ad7198c84..0ae985086 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -7,8 +7,7 @@ def run_get_target_deltas(spec, state): """ Run ``get_target_deltas``, yielding: - pre-state ('pre') - - rewards ('rewards') - - penalties ('penalties') + - deltas ('deltas') """ yield from run_attestation_component_deltas( diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index bffe2e7de..f70a20f9c 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -33,7 +33,7 @@ class Deltas(Container): penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] ``` -Also available as `rewards.ssz`. +Also available as `deltas.ssz`. ## Condition From c8b13c320cf1bacc1177b8c21e1fdd7255850620 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 May 2020 18:03:48 +0800 Subject: [PATCH 040/127] Add release & pypi badges --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 16f0dc97a..13c644a77 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ This repository hosts the current Eth2 specifications. Discussions about design ## Specs +[![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) + + Core specifications for Eth2 clients be found in [specs](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 From 12aa84fc8a7a0c93f5ad52fbed539e662a2e8e59 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 10:47:20 -0600 Subject: [PATCH 041/127] PR feedback --- .../eth2spec/test/helpers/attestations.py | 2 +- .../test_process_rewards_and_penalties.py | 20 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8c2e10edf..2c15a5136 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -152,7 +152,7 @@ def get_valid_attestation(spec, shard_transition=None, signed=False, on_time=True): - # If filter_participant_set is filters everything, the attestation has 0 participants, and cannot be signed. + # If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. # Thus strictly speaking invalid when no participant is added later. if slot is None: slot = state.slot diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 63aafe521..337f7f25c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -22,9 +22,13 @@ def run_process_rewards_and_penalties(spec, state): yield from run_epoch_processing_with(spec, state, 'process_rewards_and_penalties') -def prepare_state_with_full_attestations(spec, state, participation_fn=None): - # participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set +def prepare_state_with_attestations(spec, state, participation_fn=None): + """ + Prepare state with attestations according to the ``participation_fn``. + If no ``participation_fn``, default to "full" -- max committee participation at each slot. + participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set + """ # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) @@ -100,7 +104,7 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): @with_all_phases @spec_state_test def test_full_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -118,7 +122,7 @@ def test_full_attestations(spec, state): @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) for i, attestation in enumerate(state.previous_epoch_attestations): if i % 3 == 0: # Mess up some head votes @@ -143,7 +147,7 @@ def test_full_attestations_random_incorrect_fields(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.MAX_EFFECTIVE_BALANCE // 2) @single_phase def test_full_attestations_misc_balances(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -175,7 +179,7 @@ def test_full_attestations_misc_balances(spec, state): @with_custom_state(balances_fn=low_single_balance, threshold_fn=zero_activation_threshold) @single_phase def test_full_attestations_one_validaor_one_gwei(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) yield from run_process_rewards_and_penalties(spec, state) @@ -207,7 +211,7 @@ def run_with_participation(spec, state, participation_fn): participated.update(att_participants) return att_participants - attestations = prepare_state_with_full_attestations(spec, state, participation_fn=participation_tracker) + attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() @@ -292,7 +296,7 @@ def test_duplicate_attestation(spec, state): @spec_state_test # Case when some eligible attestations are slashed. Modifies attesting_balance and consequently rewards/penalties. def test_attestations_some_slashed(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) attesting_indices_before_slashings = list(spec.get_unslashed_attesting_indices(state, attestations)) # Slash maximum amount of validators allowed per epoch. From ee7d11d18f0fe1bcdab20d279441e782d71b2519 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 11:58:52 -0600 Subject: [PATCH 042/127] clean up proposer slashing tests and add a couple --- .../test/helpers/proposer_slashings.py | 41 ++++++++++-- .../test_process_proposer_slashing.py | 35 +++++++---- .../eth2spec/test/sanity/test_blocks.py | 62 ++++++++++++++----- 3 files changed, 103 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index d753d55dd..8b4b04879 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,18 +1,49 @@ from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey +from eth2spec.test.helpers.state import get_balance + + +def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): + slashed_validator = state.validators[slashed_index] + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + proposer_index = spec.get_beacon_proposer_index(state) + slash_penalty = state.validators[slashed_index].effective_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT + whistleblower_reward = state.validators[slashed_index].effective_balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT + if proposer_index != slashed_index: + # slashed validator lost initial slash penalty + assert ( + get_balance(state, slashed_index) + == get_balance(pre_state, slashed_index) - slash_penalty + ) + # block proposer gained whistleblower reward + # >= becase proposer could have reported multiple + assert ( + get_balance(state, proposer_index) + >= get_balance(pre_state, proposer_index) + whistleblower_reward + ) + else: + # proposer reported themself so get penalty and reward + # >= becase proposer could have reported multiple + assert ( + get_balance(state, slashed_index) + >= get_balance(pre_state, slashed_index) - slash_penalty + whistleblower_reward + ) def get_valid_proposer_slashing(spec, state, random_root=b'\x99' * 32, - validator_index=None, signed_1=False, signed_2=False): - if validator_index is None: + slashed_index=None, signed_1=False, signed_2=False): + if slashed_index is None: current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] + slashed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + privkey = pubkey_to_privkey[state.validators[slashed_index].pubkey] slot = state.slot header_1 = spec.BeaconBlockHeader( slot=slot, - proposer_index=validator_index, + proposer_index=slashed_index, parent_root=b'\x33' * 32, state_root=b'\x44' * 32, body_root=b'\x55' * 32, diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index 7657518fc..e2a6a3fe0 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -1,8 +1,9 @@ from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys -from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing -from eth2spec.test.helpers.state import get_balance, next_epoch +from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect +from eth2spec.test.helpers.state import next_epoch def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True): @@ -14,6 +15,8 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) If ``valid == False``, run expecting ``AssertionError`` """ + pre_state = state.copy() + yield 'pre', state yield 'proposer_slashing', proposer_slashing @@ -22,25 +25,31 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) yield 'post', None return - proposer_index = proposer_slashing.signed_header_1.message.proposer_index - pre_proposer_balance = get_balance(state, proposer_index) - spec.process_proposer_slashing(state, proposer_slashing) yield 'post', state - # check if slashed - slashed_validator = state.validators[proposer_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - - # lost whistleblower reward - assert get_balance(state, proposer_index) < pre_proposer_balance + slashed_proposer_index = proposer_slashing.signed_header_1.message.proposer_index + check_proposer_slashing_effect(spec, pre_state, state, slashed_proposer_index) @with_all_phases @spec_state_test def test_success(spec, state): + # Get proposer for next slot + block = build_empty_block_for_next_slot(spec, state) + proposer_index = block.proposer_index + + # Create slashing for same proposer + proposer_slashing = get_valid_proposer_slashing(spec, state, + slashed_index=proposer_index, + signed_1=True, signed_2=True) + + yield from run_proposer_slashing_processing(spec, state, proposer_slashing) + + +@with_all_phases +@spec_state_test +def test_success_slashed_and_proposer_index_the_same(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) yield from run_proposer_slashing_processing(spec, state, proposer_slashing) diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index 3347d4f8b..ddb884ae7 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_e transition_unsigned_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants -from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing +from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -228,9 +228,9 @@ def test_proposer_slashing(spec, state): # copy for later balance lookups. pre_state = state.copy() proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = proposer_slashing.signed_header_1.message.proposer_index + slashed_index = proposer_slashing.signed_header_1.message.proposer_index - assert not state.validators[validator_index].slashed + assert not state.validators[slashed_index].slashed yield 'pre', state @@ -245,21 +245,15 @@ def test_proposer_slashing(spec, state): yield 'blocks', [signed_block] yield 'post', state - # check if slashed - slashed_validator = state.validators[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) + check_proposer_slashing_effect(spec, pre_state, state, slashed_index) @with_all_phases @spec_state_test def test_double_same_proposer_slashings_same_block(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = proposer_slashing.signed_header_1.message.proposer_index - assert not state.validators[validator_index].slashed + slashed_index = proposer_slashing.signed_header_1.message.proposer_index + assert not state.validators[slashed_index].slashed yield 'pre', state @@ -274,16 +268,16 @@ def test_double_same_proposer_slashings_same_block(spec, state): @with_all_phases @spec_state_test def test_double_similar_proposer_slashings_same_block(spec, state): - validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + slashed_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # Same validator, but different slashable offences in the same block proposer_slashing_1 = get_valid_proposer_slashing(spec, state, random_root=b'\xaa' * 32, - validator_index=validator_index, + slashed_index=slashed_index, signed_1=True, signed_2=True) proposer_slashing_2 = get_valid_proposer_slashing(spec, state, random_root=b'\xbb' * 32, - validator_index=validator_index, + slashed_index=slashed_index, signed_1=True, signed_2=True) - assert not state.validators[validator_index].slashed + assert not state.validators[slashed_index].slashed yield 'pre', state @@ -295,6 +289,40 @@ def test_double_similar_proposer_slashings_same_block(spec, state): yield 'post', None +@with_all_phases +@spec_state_test +def test_multiple_different_proposer_slashings_same_block(spec, state): + pre_state = state.copy() + + num_slashings = 3 + proposer_slashings = [] + for i in range(num_slashings): + slashed_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] + assert not state.validators[slashed_index].slashed + + proposer_slashing = get_valid_proposer_slashing(spec, state, + slashed_index=slashed_index, + signed_1=True, signed_2=True) + proposer_slashings.append(proposer_slashing) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.proposer_slashings = proposer_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + for proposer_slashing in proposer_slashings: + slashed_index = proposer_slashing.signed_header_1.message.proposer_index + check_proposer_slashing_effect(spec, pre_state, state, slashed_index) + + def check_attester_slashing_effect(spec, pre_state, state, validator_index): slashed_validator = state.validators[validator_index] assert slashed_validator.slashed @@ -335,7 +363,7 @@ def test_attester_slashing(spec, state): check_attester_slashing_effect(spec, pre_state, state, validator_index) # TODO: currently mainnet limits attester-slashings per block to 1. -# When this is increased, it should be tested to cover varrious combinations +# When this is increased, it should be tested to cover various combinations # of duplicate slashings and overlaps of slashed attestations within the same block From 4ad3d65d100e7118250ea04a7ea0d066c2410ed5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 12:23:37 -0600 Subject: [PATCH 043/127] add multiple exits block sanity test --- .../test/helpers/proposer_slashings.py | 4 +-- .../eth2spec/test/sanity/test_blocks.py | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index 8b4b04879..87b4f5ca0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -19,14 +19,14 @@ def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): == get_balance(pre_state, slashed_index) - slash_penalty ) # block proposer gained whistleblower reward - # >= becase proposer could have reported multiple + # >= because proposer could have reported multiple assert ( get_balance(state, proposer_index) >= get_balance(pre_state, proposer_index) + whistleblower_reward ) else: # proposer reported themself so get penalty and reward - # >= becase proposer could have reported multiple + # >= because proposer could have reported multiple assert ( get_balance(state, slashed_index) >= get_balance(pre_state, slashed_index) - slash_penalty + whistleblower_reward diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index ddb884ae7..9faeb1f98 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -582,6 +582,38 @@ def test_double_validator_exit_same_block(spec, state): yield 'post', None +@with_phases(['phase0']) +@spec_state_test +def test_multiple_different_validator_exits_same_block(spec, state): + validator_indices = [ + spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] + for i in range(3) + ] + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + signed_exits = prepare_signed_exits(spec, state, validator_indices) + yield 'pre', state + + # Add to state via block transition + initiate_exit_block = build_empty_block_for_next_slot(spec, state) + initiate_exit_block.body.voluntary_exits = signed_exits + signed_initiate_exit_block = state_transition_and_sign_block(spec, state, initiate_exit_block) + + for index in validator_indices: + assert state.validators[index].exit_epoch < spec.FAR_FUTURE_EPOCH + + # Process within epoch transition + exit_block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + signed_exit_block = state_transition_and_sign_block(spec, state, exit_block) + + yield 'blocks', [signed_initiate_exit_block, signed_exit_block] + yield 'post', state + + for index in validator_indices: + assert state.validators[index].exit_epoch < spec.FAR_FUTURE_EPOCH + + @with_all_phases @spec_state_test def test_balance_driven_status_transitions(spec, state): From 38a5f41c30743ac97794c8a442ea28032b5d3319 Mon Sep 17 00:00:00 2001 From: Nathaniel Jensen Date: Fri, 8 May 2020 20:12:44 +1000 Subject: [PATCH 044/127] [Minor] Fix config example to not assign a return value. --- tests/core/pyspec/eth2spec/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/config/README.md b/tests/core/pyspec/eth2spec/config/README.md index ea2b2ccd8..314e68aae 100644 --- a/tests/core/pyspec/eth2spec/config/README.md +++ b/tests/core/pyspec/eth2spec/config/README.md @@ -12,7 +12,7 @@ configs_path = 'configs/' from eth2spec.config import config_util from eth2spec.phase0 import spec from importlib import reload -my_presets = config_util.prepare_config(configs_path, 'mainnet') +config_util.prepare_config(configs_path, 'mainnet') # reload spec to make loaded config effective reload(spec) ``` From 1137e0332d66a7f523640232d2a1856b5b2d3e95 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 May 2020 14:38:32 -0600 Subject: [PATCH 045/127] move MAX_ATTESTER_SLASHINGS to 2, add multiple slashings per block tests --- configs/mainnet.yaml | 4 +- configs/minimal.yaml | 4 +- specs/phase0/beacon-chain.md | 2 +- .../test/helpers/attester_slashings.py | 22 ++- .../test/phase_0/sanity/test_blocks.py | 140 ++++++++++++++++-- 5 files changed, 154 insertions(+), 18 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 60bd1c087..42845c235 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5c1511e6d..d8e346ffa 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 899778fd9..6d60d76e3 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -252,7 +252,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `MAX_PROPOSER_SLASHINGS` | `2**4` (= 16) | -| `MAX_ATTESTER_SLASHINGS` | `2**0` (= 1) | +| `MAX_ATTESTER_SLASHINGS` | `2**1` (= 2) | | `MAX_ATTESTATIONS` | `2**7` (= 128) | | `MAX_DEPOSITS` | `2**4` (= 16) | | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index 975f34c20..e743ca8ff 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,5 +1,5 @@ from eth2spec.test.context import PHASE1 -from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation +from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation, sign_indexed_attestation def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): @@ -17,6 +17,26 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): ) +def get_valid_attester_slashing_by_indices(spec, state, indices_1, indices_2=None, signed_1=False, signed_2=False): + if indices_2 is None: + indices_2 = indices_1 + + assert indices_1 == sorted(indices_1) + assert indices_2 == sorted(indices_2) + + attester_slashing = get_valid_attester_slashing(spec, state) + + attester_slashing.attestation_1.attesting_indices = indices_1 + attester_slashing.attestation_2.attesting_indices = indices_2 + + if signed_1: + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) + if signed_2: + sign_indexed_attestation(spec, state, attester_slashing.attestation_2) + + return attester_slashing + + def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 1864006bd..1c0fa5eb9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -4,7 +4,11 @@ from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_b from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ transition_unsigned_block from eth2spec.test.helpers.keys import privkeys, pubkeys -from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants +from eth2spec.test.helpers.attester_slashings import ( + get_valid_attester_slashing_by_indices, + get_valid_attester_slashing, + get_indexed_attestation_participants, +) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -326,13 +330,14 @@ def test_multiple_different_proposer_slashings_same_block(spec, state): check_proposer_slashing_effect(spec, pre_state, state, slashed_index) -def check_attester_slashing_effect(spec, pre_state, state, validator_index): - slashed_validator = state.validators[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) +def check_attester_slashing_effect(spec, pre_state, state, slashed_indices): + for slashed_index in slashed_indices: + slashed_validator = state.validators[slashed_index] + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert get_balance(state, slashed_index) < get_balance(pre_state, slashed_index) proposer_index = spec.get_beacon_proposer_index(state) # gained whistleblower reward @@ -346,9 +351,9 @@ def test_attester_slashing(spec, state): pre_state = state.copy() attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = get_indexed_attestation_participants(spec, attester_slashing.attestation_1)[0] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) - assert not state.validators[validator_index].slashed + assert not any(state.validators[i].slashed for i in slashed_indices) yield 'pre', state @@ -363,13 +368,124 @@ def test_attester_slashing(spec, state): yield 'blocks', [signed_block] yield 'post', state - check_attester_slashing_effect(spec, pre_state, state, validator_index) + check_attester_slashing_effect(spec, pre_state, state, slashed_indices) + + +@with_all_phases +@spec_state_test +def test_duplicate_attester_slashing(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + attester_slashings = [attester_slashing, attester_slashing.copy()] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) + + assert not any(state.validators[i].slashed for i in slashed_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +# All AttesterSlashing tests should be adopted for Phase 1 but helper support is not yet there + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_no_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + half_length = len(full_indices) // 2 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:half_length], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[half_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + check_attester_slashing_effect(spec, pre_state, state, full_indices) + + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_partial_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + one_third_length = len(full_indices) // 3 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:one_third_length * 2], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[one_third_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + check_attester_slashing_effect(spec, pre_state, state, full_indices) + # TODO: currently mainnet limits attester-slashings per block to 1. # When this is increased, it should be tested to cover various combinations # of duplicate slashings and overlaps of slashed attestations within the same block - @with_all_phases @spec_state_test def test_proposer_after_inactive_index(spec, state): From 7a130606ac73b72d033b6764285c3a617e322a67 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 May 2020 10:50:05 -0600 Subject: [PATCH 046/127] hww feedback --- specs/phase0/beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index fe8fcc9bd..5f04269e1 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1420,7 +1420,6 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ Return proposer and inclusion delay micro-rewards/penalties for each validator. """ rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) for index in get_unslashed_attesting_indices(state, matching_source_attestations): attestation = min([ @@ -1431,6 +1430,9 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ rewards[attestation.proposer_index] += proposer_reward max_attester_reward = get_base_reward(state, index) - proposer_reward rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) + + # No penalties associated with inclusion delay + penalties = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` @@ -1439,7 +1441,6 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S """ Return inactivity reward/penalty deltas for each validator. """ - rewards = [Gwei(0) for _ in range(len(state.validators))] penalties = [Gwei(0) for _ in range(len(state.validators))] finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch @@ -1451,6 +1452,9 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) + + # No rewards associated with inactivity penalties + rewards = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` From 01eaf6cc99091292a5778cc7d5828e544bab8561 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 May 2020 11:09:12 -0600 Subject: [PATCH 047/127] Clarify the response of BlocksByRange to address #1765 --- specs/phase0/p2p-interface.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..dae82f464 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -537,7 +537,9 @@ The response MUST contain no more than `count` blocks. Clients MUST order blocks by increasing slot number. -Clients MUST respond with blocks from their view of the current fork choice. In particular, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. +Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. + +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. #### BeaconBlocksByRoot From fdcc6d65bc1a14c7ddda3cdce40062e361370da6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 May 2020 12:00:05 -0600 Subject: [PATCH 048/127] gossipsub v1.1 and extended validators for gossip conditions --- specs/phase0/p2p-interface.md | 57 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..1b1348e96 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -176,7 +176,7 @@ Where ## The gossip domain: gossipsub -Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol. +Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. **Protocol ID:** `/meshsub/1.0.0` @@ -231,38 +231,41 @@ Clients MUST reject (fail validation) messages containing an incorrect type, or When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. +Gossipsub v1.1 introduces [Extended Validators](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#extended-validators) for the application to aid in the gossipsub peer-scoring scheme. +We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are application specific validations. If all validations pass, return `ACCEPT`. If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. + #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - - The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - - The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. - - The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. - - The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated. + - _[IGNORE]_ The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - _[IGNORE]_ The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). + - _[IGNORE]_ The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. + - _[REJECT]_ The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. + - _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - - The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. - - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - - The signature of `aggregate` is valid. + - _[IGNORE]_ `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). + - _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). + - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. + - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. + - _[REJECT]_ The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. + - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. + - _[REJECT]_ The signature of `aggregate` is valid. Additional global topics are used to propagate lower frequency validator messages. Their `Name`s are: - `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network - - The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - - All of the conditions within `process_voluntary_exit` pass validation. + - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. + - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. - - All of the conditions within `process_proposer_slashing` pass validation. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). - - All of the conditions within `process_attester_slashing` pass validation. + - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). + - _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. #### Attestation subnets @@ -270,12 +273,12 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - - The signature of `attestation` is valid. + - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. + - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). + - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). + - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. + - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. + - _[REJECT]_ The signature of `attestation` is valid. #### Attestations and Aggregation From cdd0ed0f7b7d0dcfee36f71df8a29d81863a52a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 9 May 2020 11:48:48 +0800 Subject: [PATCH 049/127] Update to IETF BLS draft-irtf-cfrg-bls-signature-02 --- specs/phase0/beacon-chain.md | 4 ++-- specs/phase1/beacon-chain.md | 4 ++-- specs/phase1/custody-game.md | 2 +- specs/phase1/shard-transition.md | 5 ++++- tests/core/pyspec/eth2spec/test/helpers/keys.py | 2 +- tests/core/pyspec/eth2spec/utils/bls.py | 8 ++++---- tests/generators/bls/main.py | 10 +++++----- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 899778fd9..83f9f7ec2 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,13 +603,13 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite which implements the following interfaces: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool` +- `def AggregateVerify(PKs: Sequence[BLSPubkey], message: Sequence[Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 331243de6..83d63ee2a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -596,7 +596,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) else: assert not cbit - return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) + return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) ``` #### `is_shard_attestation` @@ -764,7 +764,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for header in headers ] # Verify combined proposer signature - assert bls.AggregateVerify(zip(pubkeys, signing_roots), signature=transition.proposer_signature_aggregate) + assert bls.AggregateVerify(pubkeys, signing_roots, signature=transition.proposer_signature_aggregate) # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index eb243f8fb..5f5acd84f 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -300,7 +300,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) signing_roots = [compute_signing_root(root, domain) for root in [hash_tree_root(reveal.epoch), reveal.mask]] - assert bls.AggregateVerify(zip(pubkeys, signing_roots), reveal.reveal) + assert bls.AggregateVerify(pubkeys, signing_roots, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index a8de508fb..5b6a72f28 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,7 +280,10 @@ def get_shard_transition(beacon_state: BeaconState, if proposal.signature != BLSSignature(): proposer_signatures.append(proposal.signature) - proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + if len(proposer_signatures) > 0: + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + else: + proposer_signature_aggregate = BLSSignature() return ShardTransition( start_slot=start_slot, diff --git a/tests/core/pyspec/eth2spec/test/helpers/keys.py b/tests/core/pyspec/eth2spec/test/helpers/keys.py index 7f7820d3a..d813870e0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/keys.py +++ b/tests/core/pyspec/eth2spec/test/helpers/keys.py @@ -2,5 +2,5 @@ from py_ecc.bls import G2ProofOfPossession as bls from eth2spec.phase0 import spec privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 256)] -pubkeys = [bls.PrivToPub(privkey) for privkey in privkeys] +pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 3b648fac9..7f265b555 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -29,13 +29,13 @@ def Verify(PK, message, signature): @only_with_bls(alt_return=True) -def AggregateVerify(pairs, signature): - return bls.AggregateVerify(pairs, signature) +def AggregateVerify(pubkeys, messages, signature): + return bls.AggregateVerify(pubkeys, messages, signature) @only_with_bls(alt_return=True) -def FastAggregateVerify(PKs, message, signature): - return bls.FastAggregateVerify(PKs, message, signature) +def FastAggregateVerify(pubkeys, message, signature): + return bls.FastAggregateVerify(pubkeys, message, signature) @only_with_bls(alt_return=STUB_SIGNATURE) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 455292ae3..7bb093593 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -69,7 +69,7 @@ def case02_verify(): for message in MESSAGES: # Valid signature signature = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) + pubkey = bls.G2ProofOfPossession.SkToPk(privkey) identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -81,7 +81,7 @@ def case02_verify(): } # Invalid signatures -- wrong pubkey - wrong_pubkey = bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) + wrong_pubkey = bls.G2ProofOfPossession.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -119,7 +119,7 @@ def case04_fast_aggregate_verify(): privkeys = PRIVKEYS[:i + 1] sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in privkeys] aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) - pubkeys = [bls.G2ProofOfPossession.PrivToPub(privkey) for privkey in privkeys] + pubkeys = [bls.G2ProofOfPossession.SkToPk(privkey) for privkey in privkeys] pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] # Valid signature @@ -134,7 +134,7 @@ def case04_fast_aggregate_verify(): } # Invalid signature -- extra pubkey - pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[-1])] + pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.SkToPk(PRIVKEYS[-1])] pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { @@ -164,7 +164,7 @@ def case05_aggregate_verify(): sigs = [] for privkey, message in zip(PRIVKEYS, MESSAGES): sig = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) + pubkey = bls.G2ProofOfPossession.SkToPk(privkey) pairs.append({ 'pubkey': encode_hex(pubkey), 'message': encode_hex(message), From a8e3fe7551d063dd80e3578987c28e4306f593d0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 9 May 2020 21:22:38 +0200 Subject: [PATCH 050/127] Update p2p reqresp with explicit handling of reqresp size bounds --- specs/phase0/p2p-interface.md | 48 ++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d697e5da3..58f8653ca 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -98,6 +98,7 @@ It consists of four main sections: - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression) - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) + - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) - [libp2p implementations matrix](#libp2p-implementations-matrix) @@ -329,9 +330,12 @@ result ::= “0” | “1” | “2” | [“128” ... ”255”] The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. -A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. +A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. -Clients MUST ensure the each encoded payload of a `response_chunk` is less than or equal to `MAX_CHUNK_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +For both `request`s and `response`s, he `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +This includes type-specific bounds on payload size for some encoding strategies. Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. + +Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side @@ -339,13 +343,22 @@ Once a new stream with the protocol ID for the request type has been negotiated, The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. For responses consisting of potentially many `response_chunk`s (an SSZ-list) the requester SHOULD read from the stream until either; a) An error result is received in one of the chunks, b) The responder closes the stream, c) More than `MAX_CHUNK_SIZE` bytes have been read for a single `response_chunk` payload or d) More than the maximum number of requested chunks are read. For requests consisting of a single `response_chunk` and a length-prefix, the requester should read the exact number of bytes defined by the length-prefix before closing the stream. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. +A requester SHOULD read from the stream until either: +a) An error result is received in one of the chunks (the error payload MAY be read before stopping). +b) The responder closes the stream. +d) Any part of the `response_chunk` fails validation. +e) The maximum number of requested chunks are read. + +For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. + #### Responding side -Once a new stream with the protocol ID for the request type has been negotiated, the responder must process the incoming request message according to the encoding strategy, until EOF (denoting stream half-closure by the requester). +Once a new stream with the protocol ID for the request type has been negotiated, the responder SHOULD process the incoming request and MUST validate it before processing it. +Request processing and validation MUST be done according to the encoding strategy, until EOF (denoting stream half-closure by the requester). The responder MUST: @@ -415,18 +428,21 @@ If Snappy is applied, it can be passed through a buffered Snappy writer to compr *Reading*: After reading the expected SSZ byte length, the SSZ decoder can directly read the contents from the stream. If snappy is applied, it can be passed through a buffered Snappy reader to decompress frame by frame. -A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length prefix `n` from the header. +Before reading the payload, the header MUST be validated: +- The unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes, which is sufficient for any `uint64`. +- The length-prefix is within the expected [size bounds derived from the payload SSZ type](#what-are-ssz-type-size-bounds). + +After reading a valid header, the payload MAY be read, while maintaining the size constraints from the header. + +A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length-prefix `n` from the header. - For `ssz` this is: `n` - For `ssz_snappy` this is: `32 + n + n // 6`. This is considered the [worst-case compression result](https://github.com/google/snappy/blob/537f4ad6240e586970fe554614542e9717df7902/snappy.cc#L98) by Snappy. A reader SHOULD consider the following cases as invalid input: -- A SSZ length prefix that, compared against the SSZ type information (vector lengths, list limits, integer sizes, etc.), is: - - Smaller than the expected minimum serialized length. - - Bigger than the expected maximum serialized length. -- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected. -- An early EOF, before fully reading the declared length prefix worth of SSZ bytes. +- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected if more bytes are read than required. +- An early EOF, before fully reading the declared length-prefix worth of SSZ bytes. -In case of an invalid input, a reader MUST: +In case of an invalid input (header or payload), a reader MUST: - From requests: send back an error message, response code `InvalidRequest`. The request itself is ignored. - From responses: ignore the response, the response MUST be considered bad server behavior. @@ -1055,7 +1071,7 @@ For all these reasons, generically negotiating compression algorithms may be tre At this stage, the wisest choice is to consider libp2p a messenger of bytes, and to make application layer participate in compressing those bytes. This looks different depending on the interaction layer: -- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialized form. +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length-prefixing is necessary because protobuf takes care of bounding the field in the serialized form. - Req/Resp domain: since we define custom protocols that operate on byte streams, implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. ### Why are using Snappy for compression? @@ -1070,6 +1086,14 @@ If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. +### What are SSZ type size bounds? + +The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. +Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. +Other types are static, they have a fixed size: no dynamic-length content is involved, and the minimum and maximum bounds are the same. + +For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. + # libp2p implementations matrix This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. From 16363604266967f91bb60d3b00e23be7a4339452 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:02:09 -0600 Subject: [PATCH 051/127] remove todo comment Co-authored-by: Hsiao-Wei Wang --- tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index 1c0fa5eb9..4fee297ad 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -482,10 +482,6 @@ def test_multiple_attester_slashings_partial_overlap(spec, state): check_attester_slashing_effect(spec, pre_state, state, full_indices) -# TODO: currently mainnet limits attester-slashings per block to 1. -# When this is increased, it should be tested to cover various combinations -# of duplicate slashings and overlaps of slashed attestations within the same block - @with_all_phases @spec_state_test def test_proposer_after_inactive_index(spec, state): From b3dd99f4f4d624eed075a2510099a782d309920e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 19 Apr 2020 15:37:08 +1000 Subject: [PATCH 052/127] Loosen restrictions for aggregate propogation --- specs/phase0/p2p-interface.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1b1348e96..4ffc45bf7 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -249,8 +249,9 @@ There are two primary global topics used to propagate beacon blocks and aggregat - _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - _[REJECT]_ The attestation has participants -- that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - _[REJECT]_ The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. This also means that it must never have an empty set of participants. - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - _[REJECT]_ The signature of `aggregate` is valid. @@ -275,7 +276,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). + - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The signature of `attestation` is valid. From 7d4d3e43ef4a6b2002faf52263ae8d3e5bd63194 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:38:22 -0600 Subject: [PATCH 053/127] remove aggregate clarifiyng text that is not longer valid Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 4ffc45bf7..bfdc34ef9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -251,7 +251,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation has participants -- that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. This also means that it must never have an empty set of participants. + - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - _[REJECT]_ The signature of `aggregate` is valid. From b4bc2038e1a7727a18d05c8543504b7d7344cb8a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 08:39:30 -0600 Subject: [PATCH 054/127] clarify that clients MAY stop block requests if fork choice changes --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index dae82f464..399e1b466 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -539,7 +539,7 @@ Clients MUST order blocks by increasing slot number. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block clients MAY stop in the process of responding, if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot From fcf003859d576dec19c1e7c01093f484bfbf2314 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 11 May 2020 17:01:18 +0200 Subject: [PATCH 055/127] remove duplicate response diagram --- specs/phase0/p2p-interface.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 58f8653ca..1130dbe35 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -394,14 +394,6 @@ The `ErrorMessage` schema is: *Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. -A response therefore has the form of one or more `response_chunk`s, each structured as follows: -``` - +--------+--------+--------+--------+--------+--------+ - | result | header (opt) | encoded_response | - +--------+--------+--------+--------+--------+--------+ -``` -Here, `result` represents the 1-byte response code. - ### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: From 65108aeee0936ba54661b3ddd3bdb48b10ed8315 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 11:17:48 -0600 Subject: [PATCH 056/127] start on_block just slots test at time 0 --- tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index 4438dff92..016326b30 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -184,7 +184,7 @@ def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state): def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) @@ -215,7 +215,7 @@ def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) From b1c2c6e3a220c6074a9189433fa04507fed3e00b Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 11 May 2020 19:18:49 +0200 Subject: [PATCH 057/127] Default BLS to ON, keep CI BLS off for now, add milagro option --- Makefile | 6 ++-- setup.py | 1 + tests/core/pyspec/README.md | 5 +++ tests/core/pyspec/eth2spec/test/conftest.py | 35 ++++++++++++++++++--- tests/core/pyspec/eth2spec/test/context.py | 5 +-- tests/core/pyspec/eth2spec/utils/bls.py | 17 +++++++--- 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index e53aaf8a2..f52ef050d 100644 --- a/Makefile +++ b/Makefile @@ -75,15 +75,15 @@ install_test: test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --junitxml=eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --disable-bls --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & diff --git a/setup.py b/setup.py index 37f3c16ef..16ae6ac5c 100644 --- a/setup.py +++ b/setup.py @@ -502,6 +502,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==2.0.0", + "milagro_bls_binding==1.0.2", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index a9ee80105..f7092dbce 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -55,6 +55,11 @@ Run the test command from the `tests/core/pyspec` directory: pytest --config=minimal eth2spec ``` +Options: +- `--config`, to change the config. Defaults to `minimal`, can be set to `mainnet`, or other configs from the configs directory. +- `--disable-bls`, to disable BLS (only for tests that can run without) +- `--bls-type`, `milagro` or `py_ecc` (default) + ### How to view code coverage report Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report. diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 01187b05f..01c974ae0 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,6 +1,6 @@ from eth2spec.config import config_util -from eth2spec.test.context import reload_specs - +from eth2spec.test import context +from eth2spec.utils import bls as bls_utils # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. @@ -27,7 +27,16 @@ def fixture(*args, **kwargs): def pytest_addoption(parser): parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + "--config", action="store", type=str, default="minimal", + help="config: make the pyspec use the specified configuration" + ) + parser.addoption( + "--disable-bls", action="store_true", + help="bls-default: make tests that are not dependent on BLS run without BLS" + ) + parser.addoption( + "--bls-type", action="store", type=str, default="py_ecc", choices=["py_ecc", "milagro"], + help="bls-type: use 'pyecc' or 'milagro' implementation for BLS" ) @@ -36,4 +45,22 @@ def config(request): config_name = request.config.getoption("--config") config_util.prepare_config('../../../configs/', config_name) # now that the presets are loaded, reload the specs to apply them - reload_specs() + context.reload_specs() + + +@fixture(autouse=True) +def bls_default(request): + disable_bls = request.config.getoption("--disable-bls") + if disable_bls: + context.DEFAULT_BLS_ACTIVE = False + + +@fixture(autouse=True) +def bls_type(request): + bls_type = request.config.getoption("--bls-type") + if bls_type == "py_ecc": + bls_utils.bls = bls_utils.py_ecc_bls + elif bls_type == "milagro": + bls_utils.bls = bls_utils.milagro_bls + else: + raise Exception(f"unrecognized bls type: {bls_type}") diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd31..1108edcf7 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -147,14 +147,15 @@ def single_phase(fn): return entry -# BLS is turned off by default *for performance purposes during TESTING*. +# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`. +# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*. # The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). # - Some tests are marked as BLS-requiring, and ignore this setting. # (tests that express differences caused by BLS, e.g. invalid signatures being rejected) # - Some other tests are marked as BLS-ignoring, and ignore this setting. # (tests that are heavily performance impacted / require unsigned state transitions) # - Most tests respect the BLS setting. -DEFAULT_BLS_ACTIVE = False +DEFAULT_BLS_ACTIVE = True def spec_test(fn): diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 3b648fac9..44d0a8012 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -1,9 +1,13 @@ -from py_ecc.bls import G2ProofOfPossession as bls +from py_ecc.bls import G2ProofOfPossession as py_ecc_bls from py_ecc.bls.g2_primatives import signature_to_G2 as _signature_to_G2 +import milagro_bls_binding as milagro_bls # noqa: F401 for BLS switching option # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True +# To change bls implementation, default to PyECC for correctness. Milagro is a good faster alternative. +bls = py_ecc_bls + STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) @@ -30,12 +34,12 @@ def Verify(PK, message, signature): @only_with_bls(alt_return=True) def AggregateVerify(pairs, signature): - return bls.AggregateVerify(pairs, signature) + return bls.AggregateVerify(list(pairs), signature) @only_with_bls(alt_return=True) def FastAggregateVerify(PKs, message, signature): - return bls.FastAggregateVerify(PKs, message, signature) + return bls.FastAggregateVerify(list(PKs), message, signature) @only_with_bls(alt_return=STUB_SIGNATURE) @@ -45,7 +49,10 @@ def Aggregate(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def Sign(SK, message): - return bls.Sign(SK, message) + if bls == py_ecc_bls: + return bls.Sign(SK, message) + else: + return bls.Sign(SK.to_bytes(48, 'big'), message) @only_with_bls(alt_return=STUB_COORDINATES) @@ -55,4 +62,4 @@ def signature_to_G2(signature): @only_with_bls(alt_return=STUB_PUBKEY) def AggregatePKs(pubkeys): - return bls._AggregatePKs(pubkeys) + return bls._AggregatePKs(list(pubkeys)) From 30ecd9b602a62daa61d6e737f4bb0cc06eb38067 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 11:22:34 -0600 Subject: [PATCH 058/127] p2p PR feedback --- specs/phase0/p2p-interface.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1130dbe35..7c1a30e8a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -332,7 +332,7 @@ The encoding-dependent header may carry metadata or assertions such as the encod A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. -For both `request`s and `response`s, he `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +For both `request`s and `response`s, the `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. This includes type-specific bounds on payload size for some encoding strategies. Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. @@ -348,10 +348,10 @@ The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. A requester SHOULD read from the stream until either: -a) An error result is received in one of the chunks (the error payload MAY be read before stopping). -b) The responder closes the stream. -d) Any part of the `response_chunk` fails validation. -e) The maximum number of requested chunks are read. +1. An error result is received in one of the chunks (the error payload MAY be read before stopping). +2. The responder closes the stream. +3. Any part of the `response_chunk` fails validation. +4. The maximum number of requested chunks are read. For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. From 511f8034969bc24e12aaa870fbbf6cadeaea7d79 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 May 2020 12:44:46 -0600 Subject: [PATCH 059/127] use all attnets across the epoch even when not max committees per slot --- specs/phase0/p2p-interface.md | 21 ++++++++++----------- specs/phase0/validator.md | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index e06630ae5..98ee6edd2 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -151,7 +151,6 @@ This section outlines constants that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | @@ -221,12 +220,12 @@ The payload is carried in the `data` field of a gossipsub message, and varies de | Name | Message Type | |------------------------------------------------|-------------------------| -| beacon_block | SignedBeaconBlock | -| beacon_aggregate_and_proof | SignedAggregateAndProof | -| committee_index{subnet_id}\_beacon_attestation | Attestation | -| voluntary_exit | SignedVoluntaryExit | -| proposer_slashing | ProposerSlashing | -| attester_slashing | AttesterSlashing | +| beacon\_block | SignedBeaconBlock | +| beacon\_aggregate\_and\_proof | SignedAggregateAndProof | +| beacon_attestation\_{subnet\_id} | Attestation | +| voluntary\_exit | SignedVoluntaryExit | +| proposer\_slashing | ProposerSlashing | +| attester\_slashing | AttesterSlashing | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. @@ -273,8 +272,8 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: -- `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation's committee index (`attestation.data.index`) is for the correct subnet. +- `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -283,9 +282,9 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio #### Attestations and Aggregation -Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. +Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 62fdc0a93..398b882ee 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -90,6 +90,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | +| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator @@ -418,7 +419,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. + +```python +def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: + slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH + committees_since_epoch_start = sum([ + get_committee_count_at_slot(state, Slot(slot)) + for slot in range(slots_since_epoch_start) + ]) + + return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT +``` ### Attestation aggregation @@ -519,7 +531,7 @@ class SignedAggregateAndProof(Container): ## Phase 0 attestation subnet stability -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must: +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: * Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets * Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets From 78d83b6c7d7b45b519cfd1d006925d6f964ea136 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 12 May 2020 04:24:56 +0800 Subject: [PATCH 060/127] Bump py_ecc to v3.0.0 (IETF BLS v2 + H2C v6) --- setup.py | 2 +- tests/generators/bls/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 37f3c16ef..a2c6b8f88 100644 --- a/setup.py +++ b/setup.py @@ -501,7 +501,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==2.0.0", + "py_ecc==3.0.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 24ea127c4..21920089f 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==2.0.0 +py_ecc==3.0.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From ed194de26b9d89cca2478cbc5c928e1c68d655e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 02:27:42 +0800 Subject: [PATCH 061/127] Bump py_ecc to v4.0.0 (IETF BLS v2 + H2C v7) --- setup.py | 2 +- tests/generators/bls/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a2c6b8f88..5f0dce763 100644 --- a/setup.py +++ b/setup.py @@ -501,7 +501,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==3.0.0", + "py_ecc==4.0.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 21920089f..254705282 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==3.0.0 +py_ecc==4.0.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From 2718dcc4abe8d37af67426a0eaca86bc20e3161d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 02:34:10 +0800 Subject: [PATCH 062/127] Update IETF standard description --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 83f9f7ec2..3d24ff043 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,7 +603,7 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: + Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From 3c11a4dc020a21c2cdbf8b667bcec2be717e08ba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 12:56:50 +0800 Subject: [PATCH 063/127] Fix `AggregateVerify` param name: `message` -> `messages` Co-authored-by: Danny Ryan --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 3d24ff043..1c88c2a4c 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -609,7 +609,7 @@ def bytes_to_int(data: bytes) -> uint64: - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(PKs: Sequence[BLSPubkey], message: Sequence[Bytes], signature: BLSSignature) -> bool` +- `def AggregateVerify(PKs: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. From 4a246ba5ac95f657fabea05506c86c8c48f453c0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 13:01:08 +0800 Subject: [PATCH 064/127] Apply feedback from Danny, add a note of the hash to curve configuration --- specs/phase0/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1c88c2a4c..f96842b20 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -613,6 +613,8 @@ def bytes_to_int(data: bytes) -> uint64: Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. +*Note*: The non-standard configuration of the BLS and hash to curve specs is temporary and will be resolved once IETF releases BLS spec draft 3. + ### Predicates #### `is_active_validator` From b9e4bccb21a9c415c6454026fa90df3779a7d16f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 May 2020 16:26:20 +0800 Subject: [PATCH 065/127] Fix type error --- ssz/simple-serialize.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 1c4f588eb..8c76ad916 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -211,8 +211,8 @@ We first define helper functions: * `List[B, N]` and `Vector[B, N]`, where `B` is a basic type: `(N * size_of(B) + 31) // 32` (dividing by chunk size, rounding up) * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` -* `bitfield_bytes(bits)`: return the bits of the bitlist or bitvector, packed in bytes, aligned to the start. Length-delimiting bit for bitlists is excluded. -* `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack(value)`: given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. Length-delimiting bit for bitlists is excluded. And them pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` * `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: * The merkleization depends on the effective input, which can be padded/limited: @@ -228,9 +228,9 @@ We first define helper functions: We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects. -* `merkleize(bitfield_bytes(value), limit=chunk_count(type))` if `value` is a bitvector. +* `merkleize(pack_bits(value), limit=chunk_count(type))` if `value` is a bitvector. * `mix_in_length(merkleize(pack(value), limit=chunk_count(type)), len(value))` if `value` is a list of basic objects. -* `mix_in_length(merkleize(bitfield_bytes(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. +* `mix_in_length(merkleize(pack_bits(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container. * `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type. From 4431e399b1c42f6dbe17aa35dddc87e600988a52 Mon Sep 17 00:00:00 2001 From: ericsson Date: Wed, 13 May 2020 16:13:33 +0300 Subject: [PATCH 066/127] Fix bugs: - `range(len(col))`` instead of `range(col)` - `beacon_parent_block.body.shard_transitions` instead of `beacon_parent_block.shard_transitions` - `shard_states[len(shard_states)-1]` instead of `shard_states[-1]` --- specs/phase1/shard-transition.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index a8de508fb..227ea9343 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -127,7 +127,7 @@ def is_valid_fraud_proof(beacon_state: BeaconState, beacon_parent_block: BeaconBlock) -> bool: # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`. custody_bits = attestation.custody_bits_blocks - for j in range(custody_bits[offset_index]): + for j in range(len(custody_bits[offset_index])): if custody_bits[offset_index][j] != generate_custody_bit(subkey, block): return True @@ -135,7 +135,8 @@ def is_valid_fraud_proof(beacon_state: BeaconState, # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`. if offset_index == 0: shard = get_shard(beacon_state, attestation) - shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1] + shard_states = beacon_parent_block.body.shard_transitions[shard].shard_states + shard_state = shard_states[len(shard_states)-1] else: shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. From b41410eade5c4ddb7333949ee928ee93a5240493 Mon Sep 17 00:00:00 2001 From: ericsson Date: Wed, 13 May 2020 16:25:16 +0300 Subject: [PATCH 067/127] lint problem fixed --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 227ea9343..68bfb3507 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -136,7 +136,7 @@ def is_valid_fraud_proof(beacon_state: BeaconState, if offset_index == 0: shard = get_shard(beacon_state, attestation) shard_states = beacon_parent_block.body.shard_transitions[shard].shard_states - shard_state = shard_states[len(shard_states)-1] + shard_state = shard_states[len(shard_states) - 1] else: shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. From 68442c2eefbd245a717b1710e8bb155c1c22678f Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 13 May 2020 16:46:28 +0200 Subject: [PATCH 068/127] Update testing and restrict to incremental block transitions --- specs/phase0/beacon-chain.md | 2 +- .../pyspec/eth2spec/test/helpers/attestations.py | 3 ++- tests/core/pyspec/eth2spec/test/helpers/block.py | 15 ++++++++++----- tests/core/pyspec/eth2spec/test/helpers/state.py | 6 ++++-- .../epoch_processing/run_epoch_process_base.py | 3 ++- .../pyspec/eth2spec/test/sanity/test_slots.py | 3 ++- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index cdf38dc1e..1fd18336d 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1219,7 +1219,7 @@ def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) ```python def process_slots(state: BeaconState, slot: Slot) -> None: - assert state.slot <= slot + assert state.slot < slot while state.slot < slot: process_slot(state) # Process epoch on the start slot of the next epoch diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8215f5c5b..76e90e89f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -243,7 +243,8 @@ def fill_aggregate_attestation(spec, state, attestation, signed=False): def add_attestations_to_state(spec, state, attestations, slot): - spec.process_slots(state, slot) + if state.slot < slot: + spec.process_slots(state, slot) for attestation in attestations: spec.process_attestation(state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index 96cc30e35..b6167c569 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -15,7 +15,8 @@ def get_proposer_index_maybe(spec, state, slot, proposer_index=None): " Signing block is slow due to transition for proposer index calculation.") # use stub state to get proposer index of future slot stub_state = state.copy() - spec.process_slots(stub_state, slot) + if stub_state.slot < slot: + spec.process_slots(stub_state, slot) proposer_index = spec.get_beacon_proposer_index(stub_state) return proposer_index @@ -52,7 +53,10 @@ def sign_block(spec, state, block, proposer_index=None): def transition_unsigned_block(spec, state, block): - spec.process_slots(state, block.slot) + if state.slot < block.slot: + spec.process_slots(state, block.slot) + assert state.latest_block_header.slot < block.slot # There may not already be a block in this slot or past it. + assert state.slot == block.slot # The block must be for this slot spec.process_block(state, block) @@ -74,9 +78,10 @@ def build_empty_block(spec, state, slot=None): if slot < state.slot: raise Exception("build_empty_block cannot build blocks for past slots") if slot > state.slot: - # transition forward in copied state to grab relevant data from state - state = state.copy() - spec.process_slots(state, slot) + if state.slot < slot: + # transition forward in copied state to grab relevant data from state + state = state.copy() + spec.process_slots(state, slot) empty_block = spec.BeaconBlock() empty_block.slot = slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 46a7ce2b5..3860a11ce 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -17,7 +17,8 @@ def next_slots(spec, state, slots): """ Transition given slots forward. """ - spec.process_slots(state, state.slot + slots) + if slots > 0: + spec.process_slots(state, state.slot + slots) def transition_to(spec, state, slot): @@ -35,7 +36,8 @@ def next_epoch(spec, state): Transition to the start slot of the next epoch """ slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) - spec.process_slots(state, slot) + if slot > state.slot: + spec.process_slots(state, slot) def get_state_root(spec, state, slot) -> bytes: diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py index af4587a2a..b8692227f 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py @@ -18,7 +18,8 @@ def run_epoch_processing_to(spec, state, process_name: str): slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) # transition state to slot before epoch state transition - spec.process_slots(state, slot - 1) + if state.slot < slot - 1: + spec.process_slots(state, slot - 1) # start transitioning, do one slot update before the epoch itself. spec.process_slot(state) diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_slots.py b/tests/core/pyspec/eth2spec/test/sanity/test_slots.py index 6ef6be4d3..dd4f302ae 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_slots.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_slots.py @@ -51,7 +51,8 @@ def test_double_empty_epoch(spec, state): @with_all_phases @spec_state_test def test_over_epoch_boundary(spec, state): - spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH // 2)) + if spec.SLOTS_PER_EPOCH > 1: + spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH // 2)) yield 'pre', state slots = spec.SLOTS_PER_EPOCH yield 'slots', slots From 1d147037eb6b7dd86b7b785a88c860b4cd875030 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 13 May 2020 11:48:30 -0600 Subject: [PATCH 069/127] write block tests for double slot proposal and genesis proposal --- .../pyspec/eth2spec/test/helpers/block.py | 9 ++- .../eth2spec/test/sanity/test_blocks.py | 58 ++++++++++++++++++- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index b6167c569..cc8b6f5e0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -77,11 +77,10 @@ def build_empty_block(spec, state, slot=None): slot = state.slot if slot < state.slot: raise Exception("build_empty_block cannot build blocks for past slots") - if slot > state.slot: - if state.slot < slot: - # transition forward in copied state to grab relevant data from state - state = state.copy() - spec.process_slots(state, slot) + if state.slot < slot: + # transition forward in copied state to grab relevant data from state + state = state.copy() + spec.process_slots(state, slot) empty_block = spec.BeaconBlock() empty_block.slot = slot diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index b6b671872..60bcaddfc 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -3,8 +3,11 @@ from copy import deepcopy from eth2spec.utils import bls from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot, next_epoch -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ - transition_unsigned_block +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, build_empty_block, + sign_block, + transition_unsigned_block, +) from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -71,6 +74,57 @@ def test_empty_block_transition(spec, state): assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.Bytes32() +@with_phases(['phase0']) +@spec_state_test +def test_proposal_for_genesis_slot(spec, state): + assert state.slot == spec.GENESIS_SLOT + + yield 'pre', state + + block = build_empty_block(spec, state, spec.GENESIS_SLOT) + block.parent_root = state.latest_block_header.hash_tree_root() + + # Show that normal path through transition fails + failed_state = state.copy() + expect_assertion_error( + lambda: spec.state_transition(failed_state, spec.SignedBeaconBlock(message=block), validate_result=False) + ) + + # Artifically bypass the restriction in the state transition to transition and sign block for test vectors + spec.process_block(state, block) + block.state_root = state.hash_tree_root() + signed_block = sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_all_phases +@spec_state_test +def test_parent_from_same_slot(spec, state): + yield 'pre', state + + parent_block = build_empty_block_for_next_slot(spec, state) + signed_parent_block = state_transition_and_sign_block(spec, state, parent_block) + + child_block = parent_block.copy() + child_block.parent_root = state.latest_block_header.hash_tree_root() + + # Show that normal path through transition fails + failed_state = state.copy() + expect_assertion_error( + lambda: spec.state_transition(failed_state, spec.SignedBeaconBlock(message=child_block), validate_result=False) + ) + + # Artifically bypass the restriction in the state transition to transition and sign block for test vectors + spec.process_block(state, child_block) + child_block.state_root = state.hash_tree_root() + signed_child_block = sign_block(spec, state, child_block) + + yield 'blocks', [signed_parent_block, signed_child_block] + yield 'post', None + + @with_all_phases @spec_state_test def test_invalid_state_root(spec, state): From aef564733af3644801ed0b6449ac67f5f56ac326 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 13 May 2020 10:56:52 -0700 Subject: [PATCH 070/127] Update beacon-chain.md --- specs/phase1/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 35f6a6425..fccf157aa 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -476,7 +476,7 @@ def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD - if source_epoch > 0: + if source_epoch > SHARD_COMMITTEE_PERIOD: source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) @@ -494,7 +494,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch > 0: + if source_epoch > LIGHT_CLIENT_COMMITTEE_PERIOD: source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) From e7ae4be8a97284d9aec62eb6db0b99ddd56ed0d5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 13 May 2020 17:25:39 -0600 Subject: [PATCH 071/127] ensure that pre-state slot is less than block for all block tests --- .../pyspec/eth2spec/test/helpers/block.py | 5 ++-- .../pyspec/eth2spec/test/helpers/state.py | 18 ++++++++++++- .../test_process_attestation.py | 16 +++++------ .../test_process_attester_slashing.py | 6 ++--- ...est_process_early_derived_secret_reveal.py | 6 ++--- .../eth2spec/test/sanity/test_blocks.py | 26 +++++++++--------- .../pyspec/eth2spec/test/test_finality.py | 27 +++++++------------ 7 files changed, 53 insertions(+), 51 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index cc8b6f5e0..4ed84b307 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -53,6 +53,7 @@ def sign_block(spec, state, block, proposer_index=None): def transition_unsigned_block(spec, state, block): + assert state.slot < block.slot # Preserve assertion from state transition to avoid strange pre-states from testing if state.slot < block.slot: spec.process_slots(state, block.slot) assert state.latest_block_header.slot < block.slot # There may not already be a block in this slot or past it. @@ -60,11 +61,11 @@ def transition_unsigned_block(spec, state, block): spec.process_block(state, block) -def apply_empty_block(spec, state): +def apply_empty_block(spec, state, slot=None): """ Transition via an empty block (on current slot, assuming no block has been applied yet). """ - block = build_empty_block(spec, state) + block = build_empty_block(spec, state, slot) transition_unsigned_block(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 3860a11ce..7e16e13d0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -1,5 +1,5 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.block import sign_block, transition_unsigned_block +from eth2spec.test.helpers.block import apply_empty_block, sign_block, transition_unsigned_block def get_balance(state, index): @@ -31,6 +31,15 @@ def transition_to(spec, state, slot): assert state.slot == slot +def transition_to_slot_via_block(spec, state, slot): + """ + Transition to ``slot`` via an empty block transition + """ + assert state.slot < slot + apply_empty_block(spec, state, slot) + assert state.slot == slot + + def next_epoch(spec, state): """ Transition to the start slot of the next epoch @@ -40,6 +49,13 @@ def next_epoch(spec, state): spec.process_slots(state, slot) +def next_epoch_via_block(spec, state): + """ + Transition to the start slot of the next epoch via a full block transition + """ + apply_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) + + def get_state_root(spec, state, slot) -> bytes: """ Return the state root at a recent ``slot``. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 8663391aa..8752d3747 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -15,10 +15,9 @@ from eth2spec.test.helpers.attestations import ( from eth2spec.test.helpers.state import ( next_slot, next_slots, - next_epoch, - transition_to, + next_epoch_via_block, + transition_to_slot_via_block, ) -from eth2spec.test.helpers.block import apply_empty_block from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -47,9 +46,7 @@ def test_success_multi_proposer_index_iterations(spec, state): @spec_state_test def test_success_previous_epoch(spec, state): attestation = get_valid_attestation(spec, state, signed=True, on_time=False) - transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) yield from run_attestation_processing(spec, state, attestation) @@ -79,8 +76,7 @@ def test_after_epoch_slots(spec, state): attestation = get_valid_attestation(spec, state, signed=True, on_time=False) # increment past latest inclusion slot - transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH + 1) - apply_empty_block(spec, state) + transition_to_slot_via_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH + 1) yield from run_attestation_processing(spec, state, attestation, False) @@ -151,8 +147,8 @@ def test_invalid_index(spec, state): @with_all_phases @spec_state_test def test_mismatched_target_and_slot(spec, state): - next_epoch(spec, state) - next_epoch(spec, state) + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) attestation = get_valid_attestation(spec, state, on_time=False) attestation.data.slot = attestation.data.slot - spec.SLOTS_PER_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 48dc75fd9..11ead6033 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -5,10 +5,9 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ get_indexed_attestation_participants, get_attestation_2_data, get_attestation_1_data -from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import ( get_balance, - next_epoch, + next_epoch_via_block, ) @@ -91,8 +90,7 @@ def test_success_double(spec, state): @with_all_phases @spec_state_test def test_success_surround(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) state.current_justified_checkpoint.epoch += 1 attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 83b0fe325..668561261 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,6 +1,5 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal -from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.helpers.state import next_epoch, get_balance +from eth2spec.test.helpers.state import next_epoch_via_block, get_balance from eth2spec.test.context import ( PHASE0, with_all_phases_except, @@ -64,8 +63,7 @@ def test_reveal_from_current_epoch(spec, state): @spec_state_test @never_bls def test_reveal_from_past_epoch(spec, state): - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index 60bcaddfc..a0678dbb3 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -2,7 +2,10 @@ from copy import deepcopy from eth2spec.utils import bls -from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot, next_epoch +from eth2spec.test.helpers.state import ( + get_balance, state_transition_and_sign_block, + next_slot, next_epoch, next_epoch_via_block, +) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, build_empty_block, sign_block, @@ -48,10 +51,12 @@ def test_same_slot_block_transition(spec, state): yield 'pre', state - signed_block = state_transition_and_sign_block(spec, state, block) + assert state.slot == block.slot + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) yield 'blocks', [signed_block] - yield 'post', state + yield 'post', None @with_all_phases @@ -357,22 +362,19 @@ def test_proposer_after_inactive_index(spec, state): state.validators[inactive_index].exit_epoch = spec.get_current_epoch(state) # skip forward, get brand new proposers - next_epoch(spec, state) - next_epoch(spec, state) - block = build_empty_block_for_next_slot(spec, state) - state_transition_and_sign_block(spec, state, block) - + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) while True: - next_slot(spec, state) proposer_index = spec.get_beacon_proposer_index(state) if proposer_index > inactive_index: # found a proposer that has a higher index than a disabled validator yield 'pre', state # test if the proposer can be recognized correctly after the inactive validator - signed_block = state_transition_and_sign_block(spec, state, build_empty_block(spec, state)) + signed_block = state_transition_and_sign_block(spec, state, build_empty_block_for_next_slot(spec, state)) yield 'blocks', [signed_block] yield 'post', state break + next_slot(spec, state) @with_all_phases @@ -390,16 +392,16 @@ def test_high_proposer_index(spec, state): active_count = len(spec.get_active_validator_indices(state, current_epoch)) while True: - next_slot(spec, state) proposer_index = spec.get_beacon_proposer_index(state) if proposer_index >= active_count: # found a proposer that has a higher index than the active validator count yield 'pre', state # test if the proposer can be recognized correctly, even while it has a high index. - signed_block = state_transition_and_sign_block(spec, state, build_empty_block(spec, state)) + signed_block = state_transition_and_sign_block(spec, state, build_empty_block_for_next_slot(spec, state)) yield 'blocks', [signed_block] yield 'post', state break + next_slot(spec, state) @with_all_phases diff --git a/tests/core/pyspec/eth2spec/test/test_finality.py b/tests/core/pyspec/eth2spec/test/test_finality.py index 0b99cc3f9..55ccde2d9 100644 --- a/tests/core/pyspec/eth2spec/test/test_finality.py +++ b/tests/core/pyspec/eth2spec/test/test_finality.py @@ -1,7 +1,6 @@ from eth2spec.test.context import spec_state_test, never_bls, with_all_phases, with_phases -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.state import next_epoch_via_block from eth2spec.test.helpers.attestations import next_epoch_with_attestations -from eth2spec.test.helpers.block import apply_empty_block def check_finality(spec, @@ -58,10 +57,8 @@ def test_finality_no_updates_at_genesis(spec, state): @never_bls def test_finality_rule_4(spec, state): # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) yield 'pre', state @@ -86,10 +83,8 @@ def test_finality_rule_4(spec, state): @never_bls def test_finality_rule_1(spec, state): # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) yield 'pre', state @@ -116,10 +111,8 @@ def test_finality_rule_1(spec, state): @never_bls def test_finality_rule_2(spec, state): # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) yield 'pre', state @@ -152,10 +145,8 @@ def test_finality_rule_3(spec, state): https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 """ # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) + next_epoch_via_block(spec, state) + next_epoch_via_block(spec, state) yield 'pre', state From 84cea96c42e23c3737bf04096bfb8ee5823ceeb7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 May 2020 15:01:03 +0800 Subject: [PATCH 072/127] Fix typo Co-authored-by: Diederik Loerakker --- ssz/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 8c76ad916..b8a6bc9a2 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -212,7 +212,7 @@ We first define helper functions: * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` * `pack(value)`: given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. Length-delimiting bit for bitlists is excluded. And them pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. The length-delimiting bit for bitlists is excluded. And then pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` * `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: * The merkleization depends on the effective input, which can be padded/limited: From f3448e51dd1faed2e769b26c48ed695b7e2d395c Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 14 May 2020 06:21:36 -0700 Subject: [PATCH 073/127] Update specs/phase1/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fccf157aa..4a75cb076 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -476,7 +476,7 @@ def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD - if source_epoch > SHARD_COMMITTEE_PERIOD: + if source_epoch >= SHARD_COMMITTEE_PERIOD: source_epoch -= SHARD_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) From 23e2b83e20683802ce3a0f1627df88182cf3d7e1 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 14 May 2020 06:21:44 -0700 Subject: [PATCH 074/127] Update specs/phase1/beacon-chain.md Co-authored-by: Hsiao-Wei Wang --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 4a75cb076..b81f12b91 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -494,7 +494,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch > LIGHT_CLIENT_COMMITTEE_PERIOD: + if source_epoch => LIGHT_CLIENT_COMMITTEE_PERIOD: source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) From 183b19788863174e6cb5ffb6a23e2d48e3cec5af Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 May 2020 21:25:36 +0800 Subject: [PATCH 075/127] Update specs/phase1/beacon-chain.md --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index b81f12b91..03794fa2a 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -494,7 +494,7 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch => LIGHT_CLIENT_COMMITTEE_PERIOD: + if source_epoch >= LIGHT_CLIENT_COMMITTEE_PERIOD: source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) From aa436d91b2b3cc2434bccead18c991138139a81d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 May 2020 22:04:13 +0800 Subject: [PATCH 076/127] Use NO_SIGNATURE (0x00...) approach --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/phase1/beacon-chain.md | 34 +++++++++++++++++++++++++------- specs/phase1/shard-transition.md | 4 ++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 60bd1c087..0fb81a1c3 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -156,6 +156,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +# Constant +NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5c1511e6d..6650eadc1 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -156,6 +156,8 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +# Constant +NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 83d63ee2a..a52e2eb68 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -55,6 +55,8 @@ - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) + - [`optional_aggregate_verify`](#optional_aggregate_verify) + - [`optional_fast_aggregate_verify`](#optional_fast_aggregate_verify) - [Block processing](#block-processing) - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) @@ -110,6 +112,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | ## Updated containers @@ -633,6 +636,28 @@ def is_winning_attestation(state: BeaconState, ) ``` +#### `optional_aggregate_verify` + +```python +def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], + messages: Sequence[Bytes32], + signature: BLSSignature) -> bool: + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.AggregateVerify(pubkeys, messages, signature) +``` + +#### `optional_fast_aggregate_verify` + +```python +def optional_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.FastAggregateVerify(pubkeys, message, signature) +``` + ### Block processing ```python @@ -764,7 +789,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for header in headers ] # Verify combined proposer signature - assert bls.AggregateVerify(pubkeys, signing_roots, signature=transition.proposer_signature_aggregate) + assert optional_aggregate_verify(pubkeys, signing_roots, transition.proposer_signature_aggregate) # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] @@ -942,12 +967,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB slot = compute_previous_slot(state.slot) signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) - if len(signer_pubkeys) == 0: - # TODO: handle the empty light_client_signature case? - assert block_body.light_client_signature == BLSSignature() - return - else: - assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) + assert optional_fast_aggregate_verify(signer_pubkeys, signing_root, block_body.light_client_signature) ``` ### Epoch transition diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5b6a72f28..36b3b80ab 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -277,13 +277,13 @@ def get_shard_transition(beacon_state: BeaconState, proposer_signatures = [] for proposal in proposals: shard_block_lengths.append(len(proposal.message.body)) - if proposal.signature != BLSSignature(): + if proposal.signature != NO_SIGNATURE: proposer_signatures.append(proposal.signature) if len(proposer_signatures) > 0: proposer_signature_aggregate = bls.Aggregate(proposer_signatures) else: - proposer_signature_aggregate = BLSSignature() + proposer_signature_aggregate = NO_SIGNATURE return ShardTransition( start_slot=start_slot, From 6a3241be61b8be2437ef8a06bfdcc7b675915723 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 00:48:53 +0800 Subject: [PATCH 077/127] Remove leading space Co-authored-by: Danny Ryan --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index f96842b20..2ab0686ef 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -603,7 +603,7 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures - Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From d27f2350a2079644595be83e5e03cb6a0f8ff696 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 00:45:26 +0800 Subject: [PATCH 078/127] Update BLS test suite to BLS standard draft v2 format 1. Make sure that BLS -Verify APIs would only return `True` or `False` , no exceptions. 2. Use `eth2spec.utils.bls` instead of py_ecc for test generator 3. Add assertions in test generator 4. Add some special test cases for the -Verify APIs 5. Clean up the test format documents --- tests/core/pyspec/eth2spec/utils/bls.py | 26 +++- .../bls/{aggregate_sigs.md => aggregate.md} | 2 +- tests/formats/bls/aggregate_pubkeys.md | 19 --- tests/formats/bls/aggregate_verify.md | 17 +++ tests/formats/bls/fast_aggregate_verify.md | 17 +++ tests/formats/bls/msg_hash_g2_compressed.md | 21 ---- tests/formats/bls/msg_hash_g2_uncompressed.md | 21 ---- tests/formats/bls/priv_to_pub.md | 19 --- tests/formats/bls/{sign_msg.md => sign.md} | 6 - tests/formats/bls/verify.md | 17 +++ tests/generators/bls/main.py | 116 ++++++++++++++---- 11 files changed, 165 insertions(+), 116 deletions(-) rename tests/formats/bls/{aggregate_sigs.md => aggregate.md} (78%) delete mode 100644 tests/formats/bls/aggregate_pubkeys.md create mode 100644 tests/formats/bls/aggregate_verify.md create mode 100644 tests/formats/bls/fast_aggregate_verify.md delete mode 100644 tests/formats/bls/msg_hash_g2_compressed.md delete mode 100644 tests/formats/bls/msg_hash_g2_uncompressed.md delete mode 100644 tests/formats/bls/priv_to_pub.md rename tests/formats/bls/{sign_msg.md => sign.md} (67%) create mode 100644 tests/formats/bls/verify.md diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 7f265b555..acf9f99c7 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -25,17 +25,32 @@ def only_with_bls(alt_return=None): @only_with_bls(alt_return=True) def Verify(PK, message, signature): - return bls.Verify(PK, message, signature) + try: + result = bls.Verify(PK, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) def AggregateVerify(pubkeys, messages, signature): - return bls.AggregateVerify(pubkeys, messages, signature) + try: + result = bls.AggregateVerify(pubkeys, messages, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) def FastAggregateVerify(pubkeys, message, signature): - return bls.FastAggregateVerify(pubkeys, message, signature) + try: + result = bls.FastAggregateVerify(pubkeys, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=STUB_SIGNATURE) @@ -56,3 +71,8 @@ def signature_to_G2(signature): @only_with_bls(alt_return=STUB_PUBKEY) def AggregatePKs(pubkeys): return bls._AggregatePKs(pubkeys) + + +@only_with_bls(alt_return=STUB_SIGNATURE) +def SkToPk(SK): + return bls.SkToPk(SK) diff --git a/tests/formats/bls/aggregate_sigs.md b/tests/formats/bls/aggregate.md similarity index 78% rename from tests/formats/bls/aggregate_sigs.md rename to tests/formats/bls/aggregate.md index 2252dbaa8..0d7e7c631 100644 --- a/tests/formats/bls/aggregate_sigs.md +++ b/tests/formats/bls/aggregate.md @@ -16,4 +16,4 @@ output: BLS Signature -- expected output, single BLS signature ## Condition -The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. +The `aggregate` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_pubkeys.md b/tests/formats/bls/aggregate_pubkeys.md deleted file mode 100644 index 049ad6991..000000000 --- a/tests/formats/bls/aggregate_pubkeys.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS pubkey aggregation - -A BLS pubkey aggregation combines a series of pubkeys into a single pubkey. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: List[BLS Pubkey] -- list of input BLS pubkeys -output: BLS Pubkey -- expected output, single BLS pubkey -``` - -`BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`. - - -## Condition - -The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_verify.md b/tests/formats/bls/aggregate_verify.md new file mode 100644 index 000000000..3985de9f4 --- /dev/null +++ b/tests/formats/bls/aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one messages. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkeys + messages: List[bytes32] -- the messages + signature: bytes96 -- the signature to verify against pubkeys and messages +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/fast_aggregate_verify.md b/tests/formats/bls/fast_aggregate_verify.md new file mode 100644 index 000000000..7e3899a15 --- /dev/null +++ b/tests/formats/bls/fast_aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkeys and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/msg_hash_g2_compressed.md b/tests/formats/bls/msg_hash_g2_compressed.md deleted file mode 100644 index 761e819f2..000000000 --- a/tests/formats/bls/msg_hash_g2_compressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-compressed - -A BLS compressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[bytes48] -- length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/msg_hash_g2_uncompressed.md b/tests/formats/bls/msg_hash_g2_uncompressed.md deleted file mode 100644 index 5ee535a38..000000000 --- a/tests/formats/bls/msg_hash_g2_uncompressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-uncompressed - -A BLS uncompressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[List[bytes48]] -- 3 lists, each a length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/priv_to_pub.md b/tests/formats/bls/priv_to_pub.md deleted file mode 100644 index 29c6b216a..000000000 --- a/tests/formats/bls/priv_to_pub.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS private key to pubkey - -A BLS private key to public key conversion. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: bytes32 -- the private key -output: bytes48 -- the public key -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/sign_msg.md b/tests/formats/bls/sign.md similarity index 67% rename from tests/formats/bls/sign_msg.md rename to tests/formats/bls/sign.md index 6c4f88cd1..1c328755a 100644 --- a/tests/formats/bls/sign_msg.md +++ b/tests/formats/bls/sign.md @@ -10,13 +10,7 @@ The test data is declared in a `data.yaml` file: input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: bytes8 -- the BLS domain output: bytes96 -- expected signature ``` All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`. diff --git a/tests/formats/bls/verify.md b/tests/formats/bls/verify.md new file mode 100644 index 000000000..57ec8a33a --- /dev/null +++ b/tests/formats/bls/verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given one pubkey and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkey: bytes48 -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkey and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 7bb093593..f97be3c90 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -10,7 +10,7 @@ from eth_utils import ( ) from gen_base import gen_runner, gen_typing -from py_ecc import bls +from eth2spec.utils import bls from hashlib import sha256 from eth2spec.test.context import PHASE0 @@ -19,11 +19,6 @@ def hash(x): return sha256(x).digest() -F2Q_COEFF_LEN = 48 -G2_COMPRESSED_Z_LEN = 48 -DST = bls.G2ProofOfPossession.DST - - def int_to_hex(n: int, byte_length: int = None) -> str: byte_value = int_to_big_endian(n) if byte_length: @@ -49,11 +44,15 @@ PRIVKEYS = [ hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), ] +NO_PUBKEY = b'\x00' * 48 +Z1_PUBKEY = b'\xc0' + b'\x00' * 47 +NO_SIGNATURE = b'\x00' * 96 +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 def case01_sign(): for privkey in PRIVKEYS: for message in MESSAGES: - sig = bls.G2ProofOfPossession.Sign(privkey, message) + sig = bls.Sign(privkey, message) identifier = f'{int_to_hex(privkey)}_{encode_hex(message)}' yield f'sign_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -68,9 +67,10 @@ def case02_verify(): for i, privkey in enumerate(PRIVKEYS): for message in MESSAGES: # Valid signature - signature = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.SkToPk(privkey) + signature = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -81,8 +81,9 @@ def case02_verify(): } # Invalid signatures -- wrong pubkey - wrong_pubkey = bls.G2ProofOfPossession.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) + wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' + assert not bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -95,6 +96,7 @@ def case02_verify(): # Invalid signature -- tampered with signature tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert not bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -104,26 +106,37 @@ def case02_verify(): 'output': False, } + # Valid pubkey and signature with the point at infinity + assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + yield f'verify_infinity_pubkey_and_infinity_signature', { + 'input': { + 'pubkey': encode_hex(Z1_PUBKEY), + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': True, + } def case03_aggregate(): for message in MESSAGES: - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in PRIVKEYS] + sigs = [bls.Sign(privkey, message) for privkey in PRIVKEYS] yield f'aggregate_{encode_hex(message)}', { 'input': [encode_hex(sig) for sig in sigs], - 'output': encode_hex(bls.G2ProofOfPossession.Aggregate(sigs)), + 'output': encode_hex(bls.Aggregate(sigs)), } def case04_fast_aggregate_verify(): for i, message in enumerate(MESSAGES): privkeys = PRIVKEYS[:i + 1] - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in privkeys] - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) - pubkeys = [bls.G2ProofOfPossession.SkToPk(privkey) for privkey in privkeys] + sigs = [bls.Sign(privkey, message) for privkey in privkeys] + aggregate_signature = bls.Aggregate(sigs) + pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -134,9 +147,10 @@ def case04_fast_aggregate_verify(): } # Invalid signature -- extra pubkey - pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.SkToPk(PRIVKEYS[-1])] + pubkeys_extra = pubkeys + [bls.SkToPk(PRIVKEYS[-1])] pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -149,6 +163,7 @@ def case04_fast_aggregate_verify(): # Invalid signature -- tampered with signature tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -158,37 +173,86 @@ def case04_fast_aggregate_verify(): 'output': False, } + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } def case05_aggregate_verify(): - pairs = [] + pubekys = [] + pubkeys_serial = [] + messages = [] + messages_serial = [] sigs = [] for privkey, message in zip(PRIVKEYS, MESSAGES): - sig = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.SkToPk(privkey) - pairs.append({ - 'pubkey': encode_hex(pubkey), - 'message': encode_hex(message), - }) + sig = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) + pubekys.append(pubkey) + pubkeys_serial.append(encode_hex(pubkey)) + messages.append(message) + messages_serial.append(encode_hex(message)) sigs.append(sig) - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) + aggregate_signature = bls.Aggregate(sigs) + assert bls.AggregateVerify(pubekys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(aggregate_signature), }, 'output': True, } tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' + assert not bls.AggregateVerify(pubkey, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(tampered_signature), }, 'output': False, } + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'message': [], + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + assert not bls.AggregateVerify([], [], NO_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'messages': [], + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: From f0c4623871d0f66a9f9878c3444379c5ec80f6a3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 01:05:32 +0800 Subject: [PATCH 079/127] Apply PR feedback: add docstring --- specs/phase1/beacon-chain.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index a52e2eb68..9a07e0a77 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -642,6 +642,10 @@ def is_winning_attestation(state: BeaconState, def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes32], signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS AggregateVerify API. + """ if len(pubkeys) == 0: return signature == NO_SIGNATURE else: @@ -652,6 +656,10 @@ def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], ```python def optional_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS FastAggregateVerify API. + """ if len(pubkeys) == 0: return signature == NO_SIGNATURE else: From 9a2559857cc862fe42940f1fcb0de3df28b82845 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 01:17:11 +0800 Subject: [PATCH 080/127] Fix typo and remove unused variable --- tests/generators/bls/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index f97be3c90..092f1247b 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -15,6 +15,7 @@ from hashlib import sha256 from eth2spec.test.context import PHASE0 + def hash(x): return sha256(x).digest() @@ -44,11 +45,11 @@ PRIVKEYS = [ hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), ] -NO_PUBKEY = b'\x00' * 48 Z1_PUBKEY = b'\xc0' + b'\x00' * 47 NO_SIGNATURE = b'\x00' * 96 Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 + def case01_sign(): for privkey in PRIVKEYS: for message in MESSAGES: @@ -117,6 +118,7 @@ def case02_verify(): 'output': True, } + def case03_aggregate(): for message in MESSAGES: sigs = [bls.Sign(privkey, message) for privkey in PRIVKEYS] @@ -195,6 +197,7 @@ def case04_fast_aggregate_verify(): 'output': False, } + def case05_aggregate_verify(): pubekys = [] pubkeys_serial = [] @@ -237,7 +240,7 @@ def case05_aggregate_verify(): yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], - 'message': [], + 'messages': [], 'signature': encode_hex(Z2_SIGNATURE), }, 'output': False, @@ -254,6 +257,7 @@ def case05_aggregate_verify(): 'output': False, } + def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: From ea99f0ab10314b4bc236aef90ec39618b7dc24ff Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 03:03:47 +0800 Subject: [PATCH 081/127] Fix typo Co-authored-by: Danny Ryan --- tests/generators/bls/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 092f1247b..ea7446e9b 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -175,7 +175,7 @@ def case04_fast_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { @@ -186,7 +186,7 @@ def case04_fast_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { 'input': { @@ -235,7 +235,7 @@ def case05_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == Z1_SIGNATURE + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], Z2_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { @@ -246,7 +246,7 @@ def case05_aggregate_verify(): 'output': False, } - # Invalid pubkeys and signature -- len(pubkey) == 0 and signature == 0x00... + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], NO_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_na_signature', { 'input': { From 82073a4a834de77baa44a64c74e608b8b39b50f6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 03:05:23 +0800 Subject: [PATCH 082/127] Fix typo --- tests/generators/bls/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index ea7446e9b..4ad42d0a6 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -199,7 +199,7 @@ def case04_fast_aggregate_verify(): def case05_aggregate_verify(): - pubekys = [] + pubkeys = [] pubkeys_serial = [] messages = [] messages_serial = [] @@ -207,14 +207,14 @@ def case05_aggregate_verify(): for privkey, message in zip(PRIVKEYS, MESSAGES): sig = bls.Sign(privkey, message) pubkey = bls.SkToPk(privkey) - pubekys.append(pubkey) + pubkeys.append(pubkey) pubkeys_serial.append(encode_hex(pubkey)) messages.append(message) messages_serial.append(encode_hex(message)) sigs.append(sig) aggregate_signature = bls.Aggregate(sigs) - assert bls.AggregateVerify(pubekys, messages, aggregate_signature) + assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, From 3dd168335bf5a86a753a5f34de3afd2c1ad36bcf Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 14 May 2020 13:50:29 -0600 Subject: [PATCH 083/127] reformat compute_subnet_for_attestation to not use for loop --- specs/phase0/validator.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 398b882ee..75aae1f2e 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -423,11 +423,12 @@ Finally, the validator broadcasts `attestation` to the associated attestation su ```python def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. + """ slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = sum([ - get_committee_count_at_slot(state, Slot(slot)) - for slot in range(slots_since_epoch_start) - ]) + committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT ``` From ab3cbdae7528caf2d3db6d4d7cd3bd262133b3e1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 04:05:51 +0800 Subject: [PATCH 084/127] Add a note of `len(attestations) > 0` --- specs/phase0/validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 62fdc0a93..d8c164669 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -446,7 +446,7 @@ def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_si If the validator is selected to aggregate (`is_aggregator()`), they construct an aggregate attestation via the following. -Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator, and create an `aggregate_attestation: Attestation` with the following fields. +Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator. If `len(attestations) > 0`, create an `aggregate_attestation: Attestation` with the following fields. ##### Data From 483f9a1d7b7faeafe635229881810247c754371b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 14 May 2020 14:25:32 -0600 Subject: [PATCH 085/127] Update specs/phase0/p2p-interface.md Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 98ee6edd2..7b9d6b737 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -220,12 +220,14 @@ The payload is carried in the `data` field of a gossipsub message, and varies de | Name | Message Type | |------------------------------------------------|-------------------------| -| beacon\_block | SignedBeaconBlock | -| beacon\_aggregate\_and\_proof | SignedAggregateAndProof | -| beacon_attestation\_{subnet\_id} | Attestation | -| voluntary\_exit | SignedVoluntaryExit | -| proposer\_slashing | ProposerSlashing | -| attester\_slashing | AttesterSlashing | +| Name | Message Type | +|----------------------------------|---------------------------| +| `beacon_block` | `SignedBeaconBlock` | +| `beacon_aggregate_and_proof` | `SignedAggregateAndProof` | +| `beacon_attestation_{subnet_id}` | `Attestation` | +| `voluntary_exit` | `SignedVoluntaryExit` | +| `proposer_slashing` | `ProposerSlashing` | +| `attester_slashing` | `AttesterSlashing` | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. From 6f5cbe6182d9e4b35fb29e876e69d9a96aa388a7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 14 May 2020 14:55:29 -0700 Subject: [PATCH 086/127] Update shard-transition.md --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index a8de508fb..e9d97f598 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -260,7 +260,7 @@ def get_shard_state_transition_result( ### Make attestations -Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. +Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `beacon_state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. ```python def get_shard_transition(beacon_state: BeaconState, From a9c4516f382f2b13d94f514b0564b8537cda929d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 18:50:08 +0800 Subject: [PATCH 087/127] PR feedback from proto: revert configs --- configs/mainnet.yaml | 2 -- configs/minimal.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 0fb81a1c3..60bd1c087 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -156,8 +156,6 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Constant -NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 6650eadc1..5c1511e6d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -156,8 +156,6 @@ DOMAIN_SHARD_PROPOSAL: 0x80000000 DOMAIN_SHARD_COMMITTEE: 0x81000000 DOMAIN_LIGHT_CLIENT: 0x82000000 DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 -# Constant -NO_SIGNATURE: 0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 # Phase 1: Upgrade from Phase 0 From d07e594f92ea41c1ed1899422ebaa15d9f20c85b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 23:27:35 +0800 Subject: [PATCH 088/127] Add `Aggregate()` case --- tests/formats/bls/aggregate.md | 6 +++--- tests/generators/bls/main.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/formats/bls/aggregate.md b/tests/formats/bls/aggregate.md index 0d7e7c631..af8444540 100644 --- a/tests/formats/bls/aggregate.md +++ b/tests/formats/bls/aggregate.md @@ -8,11 +8,11 @@ The test data is declared in a `data.yaml` file: ```yaml input: List[BLS Signature] -- list of input BLS signatures -output: BLS Signature -- expected output, single BLS signature +output: BLS Signature -- expected output, single BLS signature or empty. ``` -`BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. - +- `BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. +- No output value if the input is invalid. ## Condition diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 4ad42d0a6..8c6589b36 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -127,6 +127,19 @@ def case03_aggregate(): 'output': encode_hex(bls.Aggregate(sigs)), } + # Invalid pubkeys -- len(pubkeys) == 0 + try: + bls.Aggregate([]) + except Exception: + pass + else: + raise Exception("Should have been INVALID") + + yield f'aggregate_na_pubkeys', { + 'input': [], + 'output': None, + } + def case04_fast_aggregate_verify(): for i, message in enumerate(MESSAGES): From fd3cce0d2c791d59237d25a632110405e7d033f1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 15 May 2020 23:38:25 +0800 Subject: [PATCH 089/127] Update README --- tests/formats/bls/README.md | 11 +++++------ tests/generators/bls/README.md | 18 ++++-------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/formats/bls/README.md b/tests/formats/bls/README.md index 4d95bdfd7..65154ba1c 100644 --- a/tests/formats/bls/README.md +++ b/tests/formats/bls/README.md @@ -5,11 +5,10 @@ We do not recommend rolling your own crypto or using an untested BLS library. The BLS test suite runner has the following handlers: -- [`aggregate_pubkeys`](./aggregate_pubkeys.md) -- [`aggregate_sigs`](./aggregate_sigs.md) -- [`msg_hash_g2_compressed`](./msg_hash_g2_compressed.md) -- [`msg_hash_g2_uncompressed`](./msg_hash_g2_uncompressed.md) -- [`priv_to_pub`](./priv_to_pub.md) -- [`sign_msg`](./sign_msg.md) +- [`aggregate_verify`](./aggregate_verify.md) +- [`aggregate`](./aggregate.md) +- [`fast_aggregate_verify`](./fast_aggregate_verify.md) +- [`sign`](./sign.md) +- [`verify`](./verify.md) *Note*: Signature-verification and aggregate-verify test cases are not yet supported. diff --git a/tests/generators/bls/README.md b/tests/generators/bls/README.md index 878bb156b..24013f88e 100644 --- a/tests/generators/bls/README.md +++ b/tests/generators/bls/README.md @@ -1,21 +1,11 @@ # BLS Test Generator -Explanation of BLS12-381 type hierarchy -The base unit is bytes48 of which only 381 bits are used +The [BLS Signature APIs](../../../specs/phase0/beacon-chain.md#bls-signatures) -- FQ: uint381 modulo field modulus -- FQ2: (FQ, FQ) -- G2: (FQ2, FQ2, FQ2) +Information on the format of the tests can be found in the [BLS test formats documentation](../../formats/bls/README.md). ## Resources -- [Eth2 spec](../../../specs/phase0/beacon-chain.md#bls-signatures) +- [IETF BLS Signature Scheme](https://datatracker.ietf.org/doc/draft-irtf-cfrg-bls-signature/) - [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf) -- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone -- [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381) -- [Trinity implementation](https://github.com/ethereum/trinity/blob/master/eth2/_utils/bls.py) - -## Comments - -Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). -Also note that pubkeys and privkeys are reversed. +- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone From 74c900f814b75d026699d92c51b8f198979a9a93 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 15 May 2020 16:39:26 -0600 Subject: [PATCH 090/127] add conditoin that block must be later than latest_block_header --- specs/phase0/beacon-chain.md | 2 + .../test_process_block_header.py | 21 +++++++++- .../eth2spec/test/sanity/test_blocks.py | 40 +++++++++++++++---- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1fd18336d..4a48e968f 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1494,6 +1494,8 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot # Verify that proposer index is the correct index assert block.proposer_index == get_beacon_proposer_index(state) # Verify that the parent matches diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index a2eb744b9..b57090568 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -9,7 +9,7 @@ def prepare_state_for_header_processing(spec, state): spec.process_slots(state, state.slot + 1) -def run_block_header_processing(spec, state, block, valid=True): +def run_block_header_processing(spec, state, block, prepare_state=True, valid=True): """ Run ``process_block_header``, yielding: - pre-state ('pre') @@ -17,7 +17,8 @@ def run_block_header_processing(spec, state, block, valid=True): - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - prepare_state_for_header_processing(spec, state) + if prepare_state: + prepare_state_for_header_processing(spec, state) yield 'pre', state yield 'block', block @@ -68,6 +69,22 @@ def test_invalid_parent_root(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) +@with_all_phases +@spec_state_test +def test_invalid_multiple_blocks_single_slot(spec, state): + block = build_empty_block_for_next_slot(spec, state) + + prepare_state_for_header_processing(spec, state) + spec.process_block_header(state, block) + + assert state.latest_block_header.slot == state.slot + + child_block = block.copy() + child_block.parent_root = block.hash_tree_root() + + yield from run_block_header_processing(spec, state, child_block, prepare_state=False, valid=False) + + @with_all_phases @spec_state_test def test_proposer_slashed(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py index a0678dbb3..71ee5141d 100644 --- a/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py @@ -79,6 +79,34 @@ def test_empty_block_transition(spec, state): assert spec.get_randao_mix(state, spec.get_current_epoch(state)) != spec.Bytes32() +def process_and_sign_block_without_header_validations(spec, state, block): + """ + Artificially bypass the restrictions in the state transition to transition and sign block + + WARNING UNSAFE: Only use when generating valid-looking invalid blocks for test vectors + """ + + # Perform single mutation in `process_block_header` + state.latest_block_header = spec.BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=spec.Bytes32(), + body_root=block.body.hash_tree_root(), + ) + + # Perform rest of process_block transitions + spec.process_randao(state, block.body) + spec.process_eth1_data(state, block.body) + spec.process_operations(state, block.body) + + # Insert post-state rot + block.state_root = state.hash_tree_root() + + # Sign block + return sign_block(spec, state, block) + + @with_phases(['phase0']) @spec_state_test def test_proposal_for_genesis_slot(spec, state): @@ -95,10 +123,8 @@ def test_proposal_for_genesis_slot(spec, state): lambda: spec.state_transition(failed_state, spec.SignedBeaconBlock(message=block), validate_result=False) ) - # Artifically bypass the restriction in the state transition to transition and sign block for test vectors - spec.process_block(state, block) - block.state_root = state.hash_tree_root() - signed_block = sign_block(spec, state, block) + # Artificially bypass the restriction in the state transition to transition and sign block for test vectors + signed_block = process_and_sign_block_without_header_validations(spec, state, block) yield 'blocks', [signed_block] yield 'post', None @@ -121,10 +147,8 @@ def test_parent_from_same_slot(spec, state): lambda: spec.state_transition(failed_state, spec.SignedBeaconBlock(message=child_block), validate_result=False) ) - # Artifically bypass the restriction in the state transition to transition and sign block for test vectors - spec.process_block(state, child_block) - child_block.state_root = state.hash_tree_root() - signed_child_block = sign_block(spec, state, child_block) + # Artificially bypass the restriction in the state transition to transition and sign block for test vectors + signed_child_block = process_and_sign_block_without_header_validations(spec, state, child_block) yield 'blocks', [signed_parent_block, signed_child_block] yield 'post', None From b975a92e9016650bc90c4ea6619675933d70ddeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 09:29:01 -0600 Subject: [PATCH 091/127] ensure at least one validator is slashed for slashing rewards tests --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index c5eace226..0433e808a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -83,8 +83,10 @@ def exit_random_validators(spec, state, rng): def slash_random_validators(spec, state, rng): # Slash ~1/2 of validators - for validator in state.validators: - validator.slashed = rng.choice([True, False]) + for index in range(len(state.validators)): + # slash at least one validator + if index == 0 or rng.choice([True, False]): + spec.slash_validator(state, index) def run_test_empty(spec, state, runner): From 97b6db497188da5c6fe9c1205b25642eb12232e2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 10:05:06 -0600 Subject: [PATCH 092/127] add rewards tests for validators not yet activated --- .../pyspec/eth2spec/test/helpers/deposits.py | 11 +++++++++ .../pyspec/eth2spec/test/helpers/rewards.py | 23 ++++++++++++++++++- .../test_process_registry_updates.py | 9 +------- .../phase_0/rewards/test_get_head_deltas.py | 6 +++++ .../test_get_inactivity_penalty_deltas.py | 21 +++++++++++++++++ .../test_get_inclusion_delay_deltas.py | 6 +++++ .../phase_0/rewards/test_get_source_deltas.py | 6 +++++ .../phase_0/rewards/test_get_target_deltas.py | 12 ++++++++++ 8 files changed, 85 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index a16f7a7bf..6a2e30497 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -5,6 +5,17 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import List +def mock_deposit(spec, state, index): + """ + Mock validator at ``index`` as having just made a deposit + """ + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + + def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 0433e808a..ac863725c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,6 +2,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import prepare_state_with_attestations +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -61,6 +62,17 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def set_some_new_deposits(spec, state, rng): + num_validators = len(state.validators) + # last 10th of validators are new deposits + for i in range(len(state.validators))[0:num_validators // 10]: + index = num_validators - 1 - i + mock_deposit(spec, state, index) + # Set half to eligible for activation + if i % 2 == 0: + state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) + + def exit_random_validators(spec, state, rng): if spec.get_current_epoch(state) < 5: # Move epochs forward to allow for some validators already exited/withdrawable @@ -69,10 +81,11 @@ def exit_random_validators(spec, state, rng): current_epoch = spec.get_current_epoch(state) # Exit ~1/2 of validators - for validator in state.validators: + for index in spec.get_active_validator_indices(state, current_epoch): if rng.choice([True, False]): continue + validator = state.validators[index] validator.exit_epoch = rng.choice([current_epoch - 1, current_epoch - 2, current_epoch - 3]) # ~1/2 are withdrawable if rng.choice([True, False]): @@ -133,6 +146,13 @@ def run_test_one_attestation_one_correct(spec, state, runner): yield from runner(spec, state) +def run_test_with_not_yet_activated_validators(spec, state, runner, rng=Random(5555)): + set_some_new_deposits(spec, state, rng) + prepare_state_with_attestations(spec, state) + + yield from runner(spec, state) + + def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): exit_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) @@ -190,6 +210,7 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_random(spec, state, runner, rng=Random(8020)): + set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index a5f4d9227..b6597b1cf 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,3 +1,4 @@ +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -7,14 +8,6 @@ def run_process_registry_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_registry_updates') -def mock_deposit(spec, state, index): - assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - - @with_all_phases @spec_state_test def test_add_to_activation_queue(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py index 2e4b9dbbc..e49a8567c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py @@ -48,6 +48,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_head_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py index 4940cdc63..8588b3f52 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py @@ -115,6 +115,27 @@ def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_no_leak(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_not_yet_activated_validators( + spec, + state, + run_get_inactivity_penalty_deltas, + ) + + @with_all_phases @spec_state_test def test_with_exited_validators_no_leak(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py index 64e1a1796..94fef5777 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py @@ -83,6 +83,12 @@ def test_full_but_partial_participation(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_inclusion_delay_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py index 54f8f3b5d..1d7891eca 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py @@ -48,6 +48,12 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_source_deltas) + + @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py index 0ae985086..5a02a246b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py @@ -48,6 +48,18 @@ def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_target_deltas) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_target_deltas) + + @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): From 96b5733086f9d2900077132bd32885474f53e430 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 12:06:24 -0600 Subject: [PATCH 093/127] cleanup set some new deposits helper for rewards tests --- tests/core/pyspec/eth2spec/test/helpers/rewards.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index ac863725c..9c445f968 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -64,13 +64,13 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) - # last 10th of validators are new deposits - for i in range(len(state.validators))[0:num_validators // 10]: - index = num_validators - 1 - i - mock_deposit(spec, state, index) - # Set half to eligible for activation - if i % 2 == 0: - state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) + # Set ~1/10 to just recently deposited + for index in range(num_validators): + if rng.randrange(num_validators) < num_validators // 10: + mock_deposit(spec, state, index) + # Set ~half of selected to eligible for activation + if rng.choice([True, False]): + state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) def exit_random_validators(spec, state, rng): From 8060505743911f6855254a386a07f5efd85cabeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 16:00:59 -0600 Subject: [PATCH 094/127] refactor rewards/penalties tests to use a single structure --- .../pyspec/eth2spec/test/helpers/rewards.py | 250 +++++++++++++++--- ...est_get_source_deltas.py => test_basic.py} | 88 +++--- .../phase_0/rewards/test_get_head_deltas.py | 136 ---------- .../test_get_inactivity_penalty_deltas.py | 231 ---------------- .../test_get_inclusion_delay_deltas.py | 213 --------------- .../phase_0/rewards/test_get_target_deltas.py | 140 ---------- .../test/phase_0/rewards/test_leak.py | 165 ++++++++++++ .../test/phase_0/rewards/test_random.py | 22 ++ tests/formats/rewards/README.md | 57 ++-- tests/generators/rewards/main.py | 24 +- 10 files changed, 504 insertions(+), 822 deletions(-) rename tests/core/pyspec/eth2spec/test/phase_0/rewards/{test_get_source_deltas.py => test_basic.py} (67%) delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py delete mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 9c445f968..fc9f1f93a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -25,17 +25,50 @@ def has_enough_for_reward(spec, state, index): ) -def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): +def run_deltas(spec, state): """ - Run ``component_delta_fn``, yielding: + Run all deltas functions yielding: - pre-state ('pre') - - deltas ('deltas') + - source deltas ('source_deltas') + - target deltas ('target_deltas') + - head deltas ('head_deltas') + - inclusion delay deltas ('inclusion_delay_deltas') + - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + yield from run_attestation_component_deltas( + spec, + state, + spec.get_source_deltas, + spec.get_matching_source_attestations, + 'source_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_target_deltas, + spec.get_matching_target_attestations, + 'target_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_head_deltas, + spec.get_matching_head_attestations, + 'head_deltas', + ) + yield from run_get_inclusion_delay_deltas(spec, state) + yield from run_get_inactivity_penalty_deltas(spec, state) + +def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): + """ + Run ``component_delta_fn``, yielding: + - deltas ('{``deltas_name``}') + """ rewards, penalties = component_delta_fn(state) - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) + yield deltas_name, Deltas(rewards=rewards, penalties=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) @@ -62,6 +95,81 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def run_get_inclusion_delay_deltas(spec, state): + """ + Run ``get_inclusion_delay_deltas``, yielding: + - inclusion delay deltas ('inclusion_delay_deltas') + """ + rewards, penalties = spec.get_inclusion_delay_deltas(state) + + yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) + + eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) + attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) + + rewarded_indices = set() + rewarded_proposer_indices = set() + # Ensure attesters with enough balance are rewarded for attestations + # Track those that are rewarded and track proposers that should be rewarded + for index in range(len(state.validators)): + if index in attesting_indices and has_enough_for_reward(spec, state, index): + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Track proposer of earliest included attestation for the validator defined by index + earliest_attestation = min([ + a for a in eligible_attestations + if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewarded_proposer_indices.add(earliest_attestation.proposer_index) + + # Ensure all expected proposers have been rewarded + # Track rewarde indices + proposing_indices = [a.proposer_index for a in eligible_attestations] + for index in proposing_indices: + if index in rewarded_proposer_indices: + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Ensure all expected non-rewarded indices received no reward + for index in range(len(state.validators)): + assert penalties[index] == 0 + if index not in rewarded_indices: + assert rewards[index] == 0 + + +def run_get_inactivity_penalty_deltas(spec, state): + """ + Run ``get_inactivity_penalty_deltas``, yielding: + - inactivity penalty deltas ('inactivity_penalty_deltas') + """ + rewards, penalties = spec.get_inactivity_penalty_deltas(state) + + yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) + + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + + finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + assert rewards[index] == 0 + if index not in eligible_indices: + assert penalties[index] == 0 + continue + + if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if not has_enough_for_reward(spec, state, index): + assert penalties[index] == 0 + elif index in matching_attesting_indices: + assert penalties[index] == base_penalty + else: + assert penalties[index] > base_penalty + else: + assert penalties[index] == 0 + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited @@ -102,74 +210,74 @@ def slash_random_validators(spec, state, rng): spec.slash_validator(state, index) -def run_test_empty(spec, state, runner): +def run_test_empty(spec, state): # Do not add any attestations to state - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_all_correct(spec, state, runner): +def run_test_full_all_correct(spec, state): prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): +def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_partial(spec, state, fraction_filled, runner): +def run_test_partial(spec, state, fraction_filled): prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_half_full(spec, state, runner): - yield from run_test_partial(spec, state, 0.5, runner) +def run_test_half_full(spec, state): + yield from run_test_partial(spec, state, 0.5) -def run_test_one_attestation_one_correct(spec, state, runner): +def run_test_one_attestation_one_correct(spec, state): prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_not_yet_activated_validators(spec, state, runner, rng=Random(5555)): +def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): +def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_slashed_validators(spec, state, runner, rng=Random(3322)): +def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_attested(spec, state, runner): +def run_test_some_very_low_effective_balances_that_attested(spec, state): state.balances prepare_state_with_attestations(spec, state) @@ -178,10 +286,10 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state, runner) for i, index in enumerate(range(5)): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): +def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): prepare_state_with_attestations(spec, state) # Remove attestation @@ -192,10 +300,10 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, r for i, index in enumerate(indices): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): +def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified @@ -206,10 +314,92 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, if not correct_head: pending_attestation.data.beacon_block_root = b'\x66' * 32 - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_random(spec, state, runner, rng=Random(8020)): +def run_test_full_delay_one_slot(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += 1 + + yield from run_deltas(spec, state) + + +def run_test_full_delay_max_slots(spec, state): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += spec.SLOTS_PER_EPOCH + + yield from run_deltas(spec, state) + + +def run_test_full_mixed_delay(spec, state, rng=Random(1234)): + prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from run_deltas(spec, state) + + +def run_test_proposer_not_in_attestations(spec, state): + prepare_state_with_attestations(spec, state) + + # Get an attestation where the proposer is not in the committee + non_proposer_attestations = [] + for a in state.previous_epoch_attestations: + if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): + non_proposer_attestations.append(a) + + assert any(non_proposer_attestations) + state.previous_epoch_attestations = non_proposer_attestations + + yield from run_deltas(spec, state) + + +def run_test_duplicate_attestations_at_later_slots(spec, state): + prepare_state_with_attestations(spec, state) + + # Remove 2/3 of attestations to make it more interesting + num_attestations = int(len(state.previous_epoch_attestations) * 0.33) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + # Get map of the proposer at each slot to make valid-looking duplicate attestations + per_slot_proposers = { + (a.data.slot + a.inclusion_delay): a.proposer_index + for a in state.previous_epoch_attestations + } + max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) + later_attestations = [] + for a in state.previous_epoch_attestations: + # Only have proposers for previous epoch so do not create later + # duplicate if slot exceeds the max slot in previous_epoch_attestations + if a.data.slot + a.inclusion_delay >= max_slot: + continue + later_a = a.copy() + later_a.inclusion_delay += 1 + later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] + later_attestations.append(later_a) + + assert any(later_attestations) + + state.previous_epoch_attestations = sorted( + state.previous_epoch_attestations + later_attestations, + key=lambda a: a.data.slot + a.inclusion_delay + ) + + yield from run_deltas(spec, state) + + +def run_test_all_balances_too_low_for_reward(spec, state): + prepare_state_with_attestations(spec, state) + + for index in range(len(state.validators)): + state.validators[index].effective_balance = 10 + + yield from run_deltas(spec, state) + + +def run_test_full_random(spec, state, rng=Random(8020)): set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) @@ -228,4 +418,4 @@ def run_test_full_random(spec, state, runner, rng=Random(8020)): # Random inclusion delay pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - yield from runner(spec, state) + yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py similarity index 67% rename from tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py index 1d7891eca..92277fdd7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py @@ -1,89 +1,71 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas import eth2spec.test.helpers.rewards as rewards_helpers -def run_get_source_deltas(spec, state): - """ - Run ``get_source_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_source_deltas, - spec.get_matching_source_attestations, - ) - - @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full(spec, state): + yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_source_deltas - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_source_deltas, - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) # @@ -101,7 +83,6 @@ def test_full_half_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -113,7 +94,6 @@ def test_full_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=1.0, - runner=run_get_source_deltas ) @@ -125,7 +105,6 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): correct_target=False, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -137,11 +116,40 @@ def test_full_half_incorrect_target_correct_head(spec, state): correct_target=False, correct_head=True, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @with_all_phases @spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_source_deltas) +def test_full_delay_one_slot(spec, state): + yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_delay_max_slots(spec, state): + yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_mixed_delay(spec, state): + yield from rewards_helpers.run_test_full_mixed_delay(spec, state) + + +@with_all_phases +@spec_state_test +def test_proposer_not_in_attestations(spec, state): + yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) + + +@with_all_phases +@spec_state_test +def test_duplicate_attestations_at_later_slots(spec, state): + yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_all_balances_too_low_for_reward(spec, state): + yield from rewards_helpers.run_test_all_balances_too_low_for_reward(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py deleted file mode 100644 index e49a8567c..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ /dev/null @@ -1,136 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_head_deltas(spec, state): - """ - Run ``get_head_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_head_deltas, - spec.get_matching_head_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_head_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py deleted file mode 100644 index 8588b3f52..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ /dev/null @@ -1,231 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import has_enough_for_reward -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.rewards import Deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inactivity_penalty_deltas(spec, state): - """ - Run ``get_inactivity_penalty_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inactivity_penalty_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch - eligible_indices = spec.get_eligible_validator_indices(state) - for index in range(len(state.validators)): - assert rewards[index] == 0 - if index not in eligible_indices: - assert penalties[index] == 0 - continue - - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) - if not has_enough_for_reward(spec, state, index): - assert penalties[index] == 0 - elif index in matching_attesting_indices: - assert penalties[index] == base_penalty - else: - assert penalties[index] > base_penalty - else: - assert penalties[index] == 0 - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -@with_all_phases -@spec_state_test -def test_empty_no_leak(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_no_leak(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_no_leak(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_no_leak(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_no_leak(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_not_yet_activated_validators( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_random_no_leak(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py deleted file mode 100644 index 94fef5777..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ /dev/null @@ -1,213 +0,0 @@ -from random import Random - -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import prepare_state_with_attestations -from eth2spec.test.helpers.rewards import Deltas, has_enough_for_reward -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inclusion_delay_deltas(spec, state): - """ - Run ``get_inclusion_delay_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inclusion_delay_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) - attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) - - rewarded_indices = set() - rewarded_proposer_indices = set() - # Ensure attesters with enough balance are rewarded for attestations - # Track those that are rewarded and track proposers that should be rewarded - for index in range(len(state.validators)): - if index in attesting_indices and has_enough_for_reward(spec, state, index): - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Track proposer of earliest included attestation for the validator defined by index - earliest_attestation = min([ - a for a in eligible_attestations - if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) - ], key=lambda a: a.inclusion_delay) - rewarded_proposer_indices.add(earliest_attestation.proposer_index) - - # Ensure all expected proposers have been rewarded - # Track rewarde indices - proposing_indices = [a.proposer_index for a in eligible_attestations] - for index in proposing_indices: - if index in rewarded_proposer_indices: - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Ensure all expected non-rewarded indices received no reward - for index in range(len(state.validators)): - assert penalties[index] == 0 - if index not in rewarded_indices: - assert rewards[index] == 0 - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inclusion_delay_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += 1 - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += spec.SLOTS_PER_EPOCH - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_mixed_delay(spec, state): - rng = Random(1234) - - prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) - - # Get an attestation where the proposer is not in the committee - non_proposer_attestations = [] - for a in state.previous_epoch_attestations: - if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): - non_proposer_attestations.append(a) - - assert any(non_proposer_attestations) - state.previous_epoch_attestations = non_proposer_attestations - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) - - # Remove 2/3 of attestations to make it more interesting - num_attestations = int(len(state.previous_epoch_attestations) * 0.33) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - - # Get map of the proposer at each slot to make valid-looking duplicate attestations - per_slot_proposers = { - (a.data.slot + a.inclusion_delay): a.proposer_index - for a in state.previous_epoch_attestations - } - max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) - later_attestations = [] - for a in state.previous_epoch_attestations: - # Only have proposers for previous epoch so do not create later - # duplicate if slot exceeds the max slot in previous_epoch_attestations - if a.data.slot + a.inclusion_delay >= max_slot: - continue - later_a = a.copy() - later_a.inclusion_delay += 1 - later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] - later_attestations.append(later_a) - - assert any(later_attestations) - - state.previous_epoch_attestations = sorted( - state.previous_epoch_attestations + later_attestations, - key=lambda a: a.data.slot + a.inclusion_delay - ) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) - - for index in range(len(state.validators)): - state.validators[index].effective_balance = 10 - - yield from run_get_inclusion_delay_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py deleted file mode 100644 index 5a02a246b..000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ /dev/null @@ -1,140 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_target_deltas(spec, state): - """ - Run ``get_target_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_target_deltas, - spec.get_matching_target_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_not_yet_activated_validators(spec, state): - yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_target_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_target_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py new file mode 100644 index 000000000..562a99b4b --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -0,0 +1,165 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.state import next_epoch +import eth2spec.test.helpers.rewards as rewards_helpers + + +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +@with_all_phases +@spec_state_test +def test_empty_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_empty(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_all_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_half_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_partial(spec, state, 0.25) + + +@with_all_phases +@spec_state_test +def test_full_but_partial_participation_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) + + +@with_all_phases +@spec_state_test +def test_one_attestation_one_correct_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_exited_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_slashed_validators_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_attested_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) + + +@with_all_phases +@spec_state_test +def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) + + +# +# NOTE: No source incorrect tests +# All PendingAttestations in state have source validated +# We choose to keep this invariant in these tests to not force clients to test with degenerate states +# + + +@with_all_phases +@spec_state_test +def test_full_half_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_correct_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_incorrect_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_half_incorrect_target_correct_head_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +def test_full_random_leak(spec, state): + transition_state_to_leak(spec, state) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_five_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=5) + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_random_ten_epoch_leak(spec, state): + transition_state_to_leak(spec, state, epochs=10) + yield from rewards_helpers.run_test_full_random(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py new file mode 100644 index 000000000..bda0ca687 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -0,0 +1,22 @@ +from random import Random + +from eth2spec.test.context import with_all_phases, spec_state_test +import eth2spec.test.helpers.rewards as rewards_helpers + + +@with_all_phases +@spec_state_test +def test_full_random_0(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(1010)) + + +@with_all_phases +@spec_state_test +def test_full_random_1(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(2020)) + + +@with_all_phases +@spec_state_test +def test_full_random_2(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index f70a20f9c..b00e7a32a 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -1,8 +1,15 @@ # Rewards tests -The different rewards deltas sub-functions are testing individually with the test handlers, each returning the related `rewards`/`penalties`. +All rewards deltas sub-functions are tested for each test case. There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. -Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) +(See test condition documentation on how to run the tests.) + +`Deltas` is defined as: +```python +class Deltas(Container): + rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] + penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] +``` ## Test case format @@ -22,31 +29,47 @@ A YAML-encoded `BeaconState`, the state before running the rewards sub-function. Also available as `pre.ssz`. -### `deltas.yaml` +### `source_deltas.yaml` -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards sub-function +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function -Where `Deltas` is defined as: -```python -class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] -``` +Also available as `source_deltas.ssz`. -Also available as `deltas.ssz`. +### `target_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function + +Also available as `target_deltas.ssz`. + +### `head_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function + +Also available as `head_deltas.ssz`. + +### `inclusion_delay_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function + +Also available as `inclusion_delay_deltas.ssz`. + +### `inactivity_penalty_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function + +Also available as `inactivity_penalty_deltas.ssz`. ## Condition A handler of the `rewards` test-runner should process these cases, - calling the corresponding rewards deltas function (same name in spec). -This excludes all other parts of `process_rewards_and_penalties` + calling the corresponding rewards deltas function for each set of deltas. -The provided pre-state is ready to be input into the designated handler. +The provided pre-state is ready to be input into each rewards deltas function. The provided `deltas` should match the return values of the - handler. Specifically the following must hold true: + deltas function. Specifically the following must hold true for each set of deltas: ```python - deltas.rewards == handler(state)[0] - deltas.penalties == handler(state)[1] + deltas.rewards == deltas_function(state)[0] + deltas.penalties == deltas_function(state)[1] ``` diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index fd95dcfaa..d8dae74fa 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -3,11 +3,9 @@ from typing import Iterable from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.phase_0.rewards import ( - test_get_source_deltas, - test_get_target_deltas, - test_get_head_deltas, - test_get_inclusion_delay_deltas, - test_get_inactivity_penalty_deltas, + test_basic, + test_leak, + test_random, ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests @@ -37,14 +35,10 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('get_source_deltas', test_get_source_deltas, 'minimal'), - create_provider('get_source_deltas', test_get_source_deltas, 'mainnet'), - create_provider('get_target_deltas', test_get_target_deltas, 'minimal'), - create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), - create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), - create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'minimal'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'mainnet'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'minimal'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'mainnet'), + create_provider('get_deltas', test_basic, 'minimal'), + create_provider('get_deltas', test_basic, 'mainnet'), + create_provider('get_deltas', test_leak, 'minimal'), + create_provider('get_deltas', test_leak, 'mainnet'), + create_provider('get_deltas', test_random, 'minimal'), + create_provider('get_deltas', test_random, 'mainnet'), ]) From 0f20d8a9ba9da9ada7c2fee0a7d7cfa08ff6ba64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 01:55:17 +0200 Subject: [PATCH 095/127] leak state decorator, and test pre-state caching --- tests/core/pyspec/eth2spec/test/context.py | 53 ++++++++++------ .../test/phase_0/rewards/test_leak.py | 60 +++++++++++++------ 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd31..303f680fb 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -9,6 +9,8 @@ from .utils import vector_test, with_meta_tags from random import Random from typing import Any, Callable, NewType, Sequence, TypedDict, Protocol +from lru import LRU + from importlib import reload @@ -48,28 +50,45 @@ class SpecForks(TypedDict, total=False): PHASE1: SpecPhase1 +def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], + spec: Spec, phases: SpecForks): + + p0 = phases[PHASE0] + balances = balances_fn(p0) + activation_threshold = threshold_fn(p0) + + state = create_genesis_state(spec=p0, validator_balances=balances, + activation_threshold=activation_threshold) + if spec.fork == PHASE1: + # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. + # Decide based on performance/consistency results later. + state = phases[PHASE1].upgrade_to_phase1(state) + # Shard state slot must lag behind BeaconState slot by at least 1 + # Will handle this more elegantly with fork mechanics + spec.process_slots(state, state.slot + 1) + + return state + + +_custom_state_cache_dict = LRU(size=10) + + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): def deco(fn): + def entry(*args, spec: Spec, phases: SpecForks, **kw): - try: - p0 = phases[PHASE0] - balances = balances_fn(p0) - activation_threshold = threshold_fn(p0) + # Use fork and file path to make a key for th + key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + global _custom_state_cache_dict + if key not in _custom_state_cache_dict: + state = _prepare_state(balances_fn, threshold_fn, spec, phases) + _custom_state_cache_dict[key] = state.get_backing() - state = create_genesis_state(spec=p0, validator_balances=balances, - activation_threshold=activation_threshold) - if spec.fork == PHASE1: - # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. - # Decide based on performance/consistency results later. - state = phases[PHASE1].upgrade_to_phase1(state) - # Shard state slot must lag behind BeaconState slot by at least 1 - # Will handle this more elegantly with fork mechanics - spec.process_slots(state, state.slot + 1) - - kw['state'] = state - except KeyError: - raise TypeError('Spec decorator must come within state decorator to inject spec into state.') + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_custom_state_cache_dict[key]) + kw['state'] = state return fn(*args, spec=spec, phases=phases, **kw) return entry return deco diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 562a99b4b..6080ec751 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,6 +1,7 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.state import next_epoch import eth2spec.test.helpers.rewards as rewards_helpers +from lru import LRU def transition_state_to_leak(spec, state, epochs=None): @@ -12,80 +13,103 @@ def transition_state_to_leak(spec, state, epochs=None): next_epoch(spec, state) +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() + + # Take a copy out of the LRU cache result. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + @with_all_phases @spec_state_test +@leaking() def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test +@leaking() def test_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_half_full(spec, state) @with_all_phases @spec_state_test +@leaking() def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test +@leaking() def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test +@leaking() def test_one_attestation_one_correct_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_not_yet_activated_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test +@leaking() def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) @@ -98,8 +122,8 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -110,8 +134,8 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_correct_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=True, @@ -122,8 +146,8 @@ def test_full_correct_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -134,8 +158,8 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_fraction_incorrect( spec, state, correct_target=False, @@ -146,20 +170,20 @@ def test_full_half_incorrect_target_correct_head_leak(spec, state): @with_all_phases @spec_state_test +@leaking() def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=5) def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) yield from rewards_helpers.run_test_full_random(spec, state) @with_all_phases @spec_state_test +@leaking(epochs=10) def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) yield from rewards_helpers.run_test_full_random(spec, state) From 75a0d60eb3339d457bfc70652cc6cacc8010461f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:25:32 +0200 Subject: [PATCH 096/127] cached epoch attestation preparation --- .../eth2spec/test/helpers/attestations.py | 21 +++++++++++ .../pyspec/eth2spec/test/helpers/rewards.py | 37 +++++++++---------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index e4be6a521..8e0501e4a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -6,6 +6,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +from lru import LRU def run_attestation_processing(spec, state, attestation, valid=True): @@ -373,6 +374,26 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): return attestations +_prep_state_cache_dict = LRU(size=10) + + +def cached_prepare_state_with_attestations(spec, state): + """ + Cached version of prepare_state_with_attestations, + but does not return anything, and does not support a participation fn argument + """ + # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root is fine. + key = (spec.fork, state.hash_tree_root()) + global _prep_state_cache_dict + if key not in _prep_state_cache_dict: + prepare_state_with_attestations(spec, state) + _prep_state_cache_dict[key] = state.get_backing() + + # Put the LRU cache result into the state view, as if we transitioned the original view + state.set_backing(_prep_state_cache_dict[key]) + + def fill_block_shard_transitions_by_attestations(spec, state, block): block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS for attestation in block.body.attestations: diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index fc9f1f93a..42e7a7614 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,7 +1,7 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.helpers.attestations import prepare_state_with_attestations +from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -217,13 +217,13 @@ def run_test_empty(spec, state): def run_test_full_all_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] @@ -232,7 +232,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): def run_test_partial(spec, state, fraction_filled): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) @@ -246,7 +246,7 @@ def run_test_half_full(spec, state): def run_test_one_attestation_one_correct(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] @@ -256,14 +256,14 @@ def run_test_one_attestation_one_correct(spec, state): def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): set_some_new_deposits(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) @@ -272,14 +272,13 @@ def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) yield from run_deltas(spec, state) def run_test_some_very_low_effective_balances_that_attested(spec, state): - state.balances - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Set some balances to be very low (including 0) assert len(state.validators) >= 5 @@ -290,7 +289,7 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove attestation attestation = state.previous_epoch_attestations[0] @@ -304,7 +303,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified num_incorrect = int(fraction_incorrect * len(state.previous_epoch_attestations)) @@ -318,7 +317,7 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, def run_test_full_delay_one_slot(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += 1 @@ -326,7 +325,7 @@ def run_test_full_delay_one_slot(spec, state): def run_test_full_delay_max_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay += spec.SLOTS_PER_EPOCH @@ -334,7 +333,7 @@ def run_test_full_delay_max_slots(spec, state): def run_test_full_mixed_delay(spec, state, rng=Random(1234)): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) @@ -342,7 +341,7 @@ def run_test_full_mixed_delay(spec, state, rng=Random(1234)): def run_test_proposer_not_in_attestations(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Get an attestation where the proposer is not in the committee non_proposer_attestations = [] @@ -357,7 +356,7 @@ def run_test_proposer_not_in_attestations(spec, state): def run_test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) # Remove 2/3 of attestations to make it more interesting num_attestations = int(len(state.previous_epoch_attestations) * 0.33) @@ -391,7 +390,7 @@ def run_test_duplicate_attestations_at_later_slots(spec, state): def run_test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for index in range(len(state.validators)): state.validators[index].effective_balance = 10 @@ -404,7 +403,7 @@ def run_test_full_random(spec, state, rng=Random(8020)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: # ~1/3 have bad target From 97e54b85d6c4c6663f97e4a9e3d51b10c1945300 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 02:46:27 +0200 Subject: [PATCH 097/127] fix cache for config change during runtime --- tests/core/pyspec/eth2spec/test/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 303f680fb..93b474517 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -78,8 +78,9 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def deco(fn): def entry(*args, spec: Spec, phases: SpecForks, **kw): - # Use fork and file path to make a key for th - key = (spec.fork, spec.__file__, balances_fn, threshold_fn) + # make a key for the state + # genesis fork version separates configs during test-generation runtime. + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) global _custom_state_cache_dict if key not in _custom_state_cache_dict: state = _prepare_state(balances_fn, threshold_fn, spec, phases) From 85f6712363802b6b1f4a1a9464f8847f02a57fb1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 03:42:58 +0200 Subject: [PATCH 098/127] fix comments based on PR feedback --- tests/core/pyspec/eth2spec/test/context.py | 2 +- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 7 ++++--- .../pyspec/eth2spec/test/phase_0/rewards/test_leak.py | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 93b474517..20214908e 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -86,7 +86,7 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], state = _prepare_state(balances_fn, threshold_fn, spec, phases) _custom_state_cache_dict[key] = state.get_backing() - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_custom_state_cache_dict[key]) kw['state'] = state diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 8e0501e4a..79f752411 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -382,13 +382,14 @@ def cached_prepare_state_with_attestations(spec, state): Cached version of prepare_state_with_attestations, but does not return anything, and does not support a participation fn argument """ - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # prepare it with attestations, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (spec.fork, state.hash_tree_root()) global _prep_state_cache_dict if key not in _prep_state_cache_dict: prepare_state_with_attestations(spec, state) - _prep_state_cache_dict[key] = state.get_backing() + _prep_state_cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 6080ec751..4e75079c0 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -20,15 +20,16 @@ def leaking(epochs=None): def deco(fn): def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, make it leaking, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root is fine. + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) global _cache_dict if key not in _cache_dict: transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - # Take a copy out of the LRU cache result. + # Take an entry out of the LRU. # No copy is necessary, as we wrap the immutable backing with a new view. state = spec.BeaconState(backing=_cache_dict[key]) return fn(*args, spec=spec, state=state, **kw) From 6d45afeefbf5f76ea61eb0f4d71e90402c3744c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 19:53:46 -0600 Subject: [PATCH 099/127] add some more random reawrds tests --- .../pyspec/eth2spec/test/helpers/rewards.py | 3 +++ .../test/phase_0/rewards/test_random.py | 25 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 42e7a7614..d62fee6ce 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -174,6 +174,9 @@ def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited for index in range(num_validators): + # If not already active, skip + if not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)): + continue if rng.randrange(num_validators) < num_validators // 10: mock_deposit(spec, state, index) # Set ~half of selected to eligible for activation diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py index bda0ca687..83c7f7905 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -1,6 +1,13 @@ from random import Random -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import ( + with_all_phases, + spec_test, + spec_state_test, + with_custom_state, + single_phase, + low_balances, misc_balances, +) import eth2spec.test.helpers.rewards as rewards_helpers @@ -20,3 +27,19 @@ def test_full_random_1(spec, state): @spec_state_test def test_full_random_2(spec, state): yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) + + +@with_all_phases +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_low_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_misc_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) From 5c564f4d7c9463e553641fc1792f879a2aadecb5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 May 2020 20:06:36 -0600 Subject: [PATCH 100/127] clean up reards generator --- tests/generators/rewards/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index d8dae74fa..c90943cab 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -14,7 +14,7 @@ from eth2spec.config import config_util from eth2spec.test.context import PHASE0 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -25,7 +25,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( runner_name='rewards', - handler_name=handler_name, + handler_name='core', src=tests_src, fork_name=PHASE0, ) @@ -34,11 +34,11 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('get_deltas', test_basic, 'minimal'), - create_provider('get_deltas', test_basic, 'mainnet'), - create_provider('get_deltas', test_leak, 'minimal'), - create_provider('get_deltas', test_leak, 'mainnet'), - create_provider('get_deltas', test_random, 'minimal'), - create_provider('get_deltas', test_random, 'mainnet'), + gen_runner.run_generator("rewards", [ + create_provider(test_basic, 'minimal'), + create_provider(test_basic, 'mainnet'), + create_provider(test_leak, 'minimal'), + create_provider(test_leak, 'mainnet'), + create_provider(test_random, 'minimal'), + create_provider(test_random, 'mainnet'), ]) From 65b5aa3c636234d024c89c3f3a9df87700b5258a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 19 May 2020 14:31:28 +0200 Subject: [PATCH 101/127] update test format docs --- tests/formats/epoch_processing/README.md | 2 +- tests/formats/rewards/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 7c5e2dc70..57c9441c8 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -38,7 +38,7 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: - `justification_and_finalization` -- *`rewards_and_penalties` - planned testing extension* +- `rewards_and_penalties` (limited to `minimal` config) - `registry_updates` - `slashings` - `final_updates` diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index b00e7a32a..b229d9f98 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -7,8 +7,8 @@ There is no "change" factor, the rewards/penalties outputs are pure functions wi `Deltas` is defined as: ```python class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] + rewards: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + penalties: List[Gwei, VALIDATOR_REGISTRY_LIMIT] ``` ## Test case format From 724139a1f81526648dd0055d4ab4bdfa34796477 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 10:23:12 -0600 Subject: [PATCH 102/127] change gossipsub protocol ID to /meshsub/1.1.0 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 01d905a38..c8893ead9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -178,7 +178,7 @@ Where Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. -**Protocol ID:** `/meshsub/1.0.0` +**Protocol ID:** `/meshsub/1.1.0` **Gossipsub Parameters** From fdc7e846dfbffe172dd17805550a4980e16e7a36 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 10:25:43 -0600 Subject: [PATCH 103/127] remove incorrect table format --- specs/phase0/p2p-interface.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c8893ead9..0e8699555 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -218,8 +218,6 @@ where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: -| Name | Message Type | -|------------------------------------------------|-------------------------| | Name | Message Type | |----------------------------------|---------------------------| | `beacon_block` | `SignedBeaconBlock` | @@ -552,7 +550,7 @@ Clients MUST order blocks by increasing slot number. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block clients MAY stop in the process of responding, if their fork choice changes the view of the chain in the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot From 85e78223dd361bbdf6e3e567bdf6860820b008ce Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 16:51:46 -0600 Subject: [PATCH 104/127] ensure when performing optimally that you don't lose money during a leak --- specs/phase0/beacon-chain.md | 31 ++++++++--- .../pyspec/eth2spec/test/helpers/rewards.py | 34 +++++++++++++ .../test_process_rewards_and_penalties.py | 51 +++++++++++++++++++ .../test/phase_0/rewards/test_leak.py | 36 +------------ 4 files changed, 110 insertions(+), 42 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 142cf3b02..7bf4cd06e 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1354,6 +1354,19 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) ``` + +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` + + +```python +def in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` + + ```python def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: previous_epoch = get_previous_epoch(state) @@ -1378,8 +1391,11 @@ def get_attestation_component_deltas(state: BeaconState, for index in get_eligible_validator_indices(state): if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) - rewards[index] += reward_numerator // (total_balance // increment) + if in_inactivity_leak(state): + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) else: penalties[index] += get_base_reward(state, index) return rewards, penalties @@ -1428,7 +1444,10 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward - max_attester_reward = get_base_reward(state, index) - proposer_reward + if in_inactivity_leak(state): + max_attester_reward = get_base_reward(state, index) + else: + max_attester_reward = get_base_reward(state, index) - proposer_reward rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # No penalties associated with inclusion delay @@ -1442,16 +1461,14 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S Return inactivity reward/penalty deltas for each validator. """ penalties = [Gwei(0) for _ in range(len(state.validators))] - finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch - - if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + if in_inactivity_leak(state): matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) # No rewards associated with inactivity penalties rewards = [Gwei(0) for _ in range(len(state.validators))] diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index d62fee6ce..034a79fd4 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,4 +1,5 @@ from random import Random +from lru import LRU from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations @@ -170,6 +171,39 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. + + # Take an entry out of the LRU. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index eff286448..52d3b3c06 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -14,6 +14,7 @@ from eth2spec.test.helpers.attestations import ( get_valid_attestation, prepare_state_with_attestations, ) +from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with from random import Random @@ -80,6 +81,56 @@ def test_full_attestations(spec, state): assert state.balances[index] < pre_state.balances[index] +@with_all_phases +@spec_state_test +@leaking() +def test_full_attestations_with_leak(spec, state): + attestations = prepare_state_with_attestations(spec, state) + + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + pre_state = state.copy() + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) == len(pre_state.validators) + for index in range(len(pre_state.validators)): + # Proposers can still make money during a leak + if index in proposer_indices: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +@leaking() +def test_partial_attestations_with_leak(spec, state): + attestations = prepare_state_with_attestations(spec, state) + + attestations = attestations[:len(attestations) // 2] + state.previous_epoch_attestations = state.previous_epoch_attestations[:len(attestations)] + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + pre_state = state.copy() + + yield from run_process_rewards_and_penalties(spec, state) + + attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) + assert len(attesting_indices) < len(pre_state.validators) + for index in range(len(pre_state.validators)): + # Proposers can still make money during a leak + if index in proposer_indices and index in attesting_indices: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 4e75079c0..b0f9767b2 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,40 +1,6 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers -from lru import LRU - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -_cache_dict = LRU(size=10) - - -def leaking(epochs=None): - - def deco(fn): - def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, - # transition it to leak, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root does not affect speed. - key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) - global _cache_dict - if key not in _cache_dict: - transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - - # Take an entry out of the LRU. - # No copy is necessary, as we wrap the immutable backing with a new view. - state = spec.BeaconState(backing=_cache_dict[key]) - return fn(*args, spec=spec, state=state, **kw) - return entry - return deco @with_all_phases From 95c3295eeba373e94b9340b8b0fe8c804858ee08 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 May 2020 17:12:53 -0600 Subject: [PATCH 105/127] move proposer negation to inactivity_penalty deltas --- specs/phase0/beacon-chain.md | 19 ++++++++++++------- .../pyspec/eth2spec/test/helpers/rewards.py | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7bf4cd06e..b0a9724fa 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1355,6 +1355,12 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` + + ```python def get_finality_delay(state: BeaconState) -> uint64: return get_previous_epoch(state) - state.finalized_checkpoint.epoch @@ -1392,6 +1398,7 @@ def get_attestation_component_deltas(state: BeaconState, if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow if in_inactivity_leak(state): + # Full base reward will be cancelled out by inactivity penalty deltas rewards[index] += get_base_reward(state, index) else: reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) @@ -1442,12 +1449,8 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) - proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) - rewards[attestation.proposer_index] += proposer_reward - if in_inactivity_leak(state): - max_attester_reward = get_base_reward(state, index) - else: - max_attester_reward = get_base_reward(state, index) - proposer_reward + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = get_base_reward(state, index) - get_proposer_reward(state, index) rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # No penalties associated with inclusion delay @@ -1465,7 +1468,9 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): - penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 034a79fd4..801434e79 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -160,7 +160,8 @@ def run_get_inactivity_penalty_deltas(spec, state): continue if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + base_reward = spec.get_base_reward(state, index) + base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: From 4c5d2c25b3809273777894c1cadd0069314b4835 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 20 May 2020 14:37:15 +0800 Subject: [PATCH 106/127] Bump remerkleable to 0.1.15 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f0dce763..879ca4d38 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "pycryptodome==3.9.4", "py_ecc==4.0.0", "dataclasses==0.6", - "remerkleable==0.1.13", + "remerkleable==0.1.15", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] From 796e372c5e03b3e0c0a589d3e90a5309f360455e Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 15:30:44 +0200 Subject: [PATCH 107/127] remerkleable 0.1.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 879ca4d38..da56175af 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "pycryptodome==3.9.4", "py_ecc==4.0.0", "dataclasses==0.6", - "remerkleable==0.1.15", + "remerkleable==0.1.16", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] From 943e51aef18bb025fa34b2caeb7810f9d77e9890 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 10:11:47 -0600 Subject: [PATCH 108/127] hww feedback for finality rewards fix --- specs/phase0/beacon-chain.md | 9 +- .../pyspec/eth2spec/test/helpers/rewards.py | 3 +- .../test_process_rewards_and_penalties.py | 116 +++++++----------- 3 files changed, 51 insertions(+), 77 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b0a9724fa..839af5f7b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1368,7 +1368,7 @@ def get_finality_delay(state: BeaconState) -> uint64: ```python -def in_inactivity_leak(state: BeaconState) -> bool: +def is_in_inactivity_leak(state: BeaconState) -> bool: return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY ``` @@ -1397,8 +1397,9 @@ def get_attestation_component_deltas(state: BeaconState, for index in get_eligible_validator_indices(state): if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - if in_inactivity_leak(state): - # Full base reward will be cancelled out by inactivity penalty deltas + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. rewards[index] += get_base_reward(state, index) else: reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) @@ -1464,7 +1465,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S Return inactivity reward/penalty deltas for each validator. """ penalties = [Gwei(0) for _ in range(len(state.validators))] - if in_inactivity_leak(state): + if is_in_inactivity_leak(state): matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 801434e79..c11ba1ec1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -151,7 +151,6 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): assert rewards[index] == 0 @@ -159,7 +158,7 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 continue - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + if spec.is_in_inactivity_leak(state): base_reward = spec.get_base_reward(state, index) base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 52d3b3c06..bafefcad6 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -63,74 +63,6 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_full_attestations(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == len(pre_state.validators) - for index in range(len(pre_state.validators)): - if index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - -@with_all_phases -@spec_state_test -@leaking() -def test_full_attestations_with_leak(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == len(pre_state.validators) - for index in range(len(pre_state.validators)): - # Proposers can still make money during a leak - if index in proposer_indices: - assert state.balances[index] > pre_state.balances[index] - # If not proposer but participated optimally, should have exactly neutral balance - elif index in attesting_indices: - assert state.balances[index] == pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - -@with_all_phases -@spec_state_test -@leaking() -def test_partial_attestations_with_leak(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - attestations = attestations[:len(attestations) // 2] - state.previous_epoch_attestations = state.previous_epoch_attestations[:len(attestations)] - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) < len(pre_state.validators) - for index in range(len(pre_state.validators)): - # Proposers can still make money during a leak - if index in proposer_indices and index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - # If not proposer but participated optimally, should have exactly neutral balance - elif index in attesting_indices: - assert state.balances[index] == pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): @@ -224,6 +156,7 @@ def run_with_participation(spec, state, participation_fn): return att_participants attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] pre_state = state.copy() @@ -233,10 +166,20 @@ def run_with_participation(spec, state, participation_fn): assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - if index in participated: - assert state.balances[index] > pre_state.balances[index] + if spec.is_in_inactivity_leak(state): + # Proposers can still make money during a leak + if index in proposer_indices and index in participated: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] else: - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] @with_all_phases @@ -246,6 +189,14 @@ def test_almost_empty_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_empty_attestations_with_leak(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + @with_all_phases @spec_state_test def test_random_fill_attestations(spec, state): @@ -253,6 +204,14 @@ def test_random_fill_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) +@with_all_phases +@spec_state_test +@leaking() +def test_random_fill_attestations_with_leak(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + @with_all_phases @spec_state_test def test_almost_full_attestations(spec, state): @@ -260,12 +219,27 @@ def test_almost_full_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_full_attestations_with_leak(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + @with_all_phases @spec_state_test def test_full_attestation_participation(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) +@with_all_phases +@spec_state_test +@leaking() +def test_full_attestation_participation_with_leak(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) + + @with_all_phases @spec_state_test def test_duplicate_attestation(spec, state): From c9f21f1f43d44b1cce67363aa8606eaf66e60684 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 10:44:08 -0600 Subject: [PATCH 109/127] clarify that eth1 blocks must be at a safe fllow distance before being considered for genesis --- specs/phase0/beacon-chain.md | 5 ++++- specs/phase0/validator.md | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 142cf3b02..0e4e2b7a0 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -183,6 +183,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | +| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) | | `MAX_COMMITTEES_PER_SLOT` | `2**6` (= 64) | | `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) | | `MAX_VALIDATORS_PER_COMMITTEE` | `2**11` (= 2,048) | @@ -219,6 +220,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | :-: | :-: | | `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | @@ -229,7 +231,6 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | - ### State list lengths | Name | Value | Unit | Duration | @@ -1136,6 +1137,8 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` - `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `MIN_GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. + ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, eth1_timestamp: uint64, diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index adf23c840..80e9d48c5 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -85,11 +85,9 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) | blocks | ~4 hours | | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator From f72d14a7477d388e8e441b5ef7b2ebc1220d93c7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 01:35:40 +0800 Subject: [PATCH 110/127] Bump `milagro_bls_binding` to 1.2.0 Also verify it in BLS test generator --- setup.py | 3 +-- tests/core/pyspec/eth2spec/utils/bls.py | 5 +++-- tests/generators/bls/main.py | 29 ++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index cc5e39865..792ef42fa 100644 --- a/setup.py +++ b/setup.py @@ -503,8 +503,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==4.0.0", - "milagro_bls_binding==1.0.2", - "py_ecc==4.0.0", + "milagro_bls_binding==1.2.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 320e3bc93..778b23da7 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -10,7 +10,8 @@ bls = py_ecc_bls STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 -STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 +STUB_COORDINATES = _signature_to_G2(Z2_SIGNATURE) def only_with_bls(alt_return=None): @@ -67,7 +68,7 @@ def Sign(SK, message): if bls == py_ecc_bls: return bls.Sign(SK, message) else: - return bls.Sign(SK.to_bytes(48, 'big'), message) + return bls.Sign(SK.to_bytes(32, 'big'), message) @only_with_bls(alt_return=STUB_COORDINATES) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 8c6589b36..9e10b4044 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -2,18 +2,22 @@ BLS test vectors generator """ +from hashlib import sha256 from typing import Tuple, Iterable, Any, Callable, Dict from eth_utils import ( encode_hex, int_to_big_endian, ) -from gen_base import gen_runner, gen_typing +import milagro_bls_binding as milagro_bls from eth2spec.utils import bls -from hashlib import sha256 - from eth2spec.test.context import PHASE0 +from gen_base import gen_runner, gen_typing + + +def to_bytes(i): + return i.to_bytes(32, "big") def hash(x): @@ -70,8 +74,15 @@ def case02_verify(): # Valid signature signature = bls.Sign(privkey, message) pubkey = bls.SkToPk(privkey) + + assert milagro_bls.SkToPk(to_bytes(privkey)) == pubkey + assert milagro_bls.Sign(to_bytes(privkey), message) == signature + identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) + assert milagro_bls.Verify(pubkey, message, signature) + yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -85,6 +96,7 @@ def case02_verify(): wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' assert not bls.Verify(wrong_pubkey, message, signature) + assert not milagro_bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -98,6 +110,7 @@ def case02_verify(): tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' assert not bls.Verify(pubkey, message, tampered_signature) + assert not milagro_bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -109,6 +122,7 @@ def case02_verify(): # Valid pubkey and signature with the point at infinity assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + assert milagro_bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) yield f'verify_infinity_pubkey_and_infinity_signature', { 'input': { 'pubkey': encode_hex(Z1_PUBKEY), @@ -152,6 +166,7 @@ def case04_fast_aggregate_verify(): # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) + assert milagro_bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -166,6 +181,7 @@ def case04_fast_aggregate_verify(): pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -179,6 +195,7 @@ def case04_fast_aggregate_verify(): tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -190,6 +207,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, Z2_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -201,6 +219,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, NO_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [], @@ -228,6 +247,7 @@ def case05_aggregate_verify(): aggregate_signature = bls.Aggregate(sigs) assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) + assert milagro_bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, @@ -239,6 +259,7 @@ def case05_aggregate_verify(): tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' assert not bls.AggregateVerify(pubkey, messages, tampered_signature) + assert not milagro_bls.AggregateVerify(pubkeys, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { 'pubkeys': pubkeys_serial, @@ -250,6 +271,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], Z2_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -261,6 +283,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], NO_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], NO_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [], From db1a90d2eee58bd48985d6d39b766686aa638852 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 02:05:22 +0800 Subject: [PATCH 111/127] `test_success_surround` changes the signing data of attestation, so it should be never_bls --- .../phase_0/block_processing/test_process_attester_slashing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 11ead6033..8d7638f51 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases + spec_state_test, expect_assertion_error, always_bls, never_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -89,6 +89,7 @@ def test_success_double(spec, state): @with_all_phases @spec_state_test +@never_bls def test_success_surround(spec, state): next_epoch_via_block(spec, state) From 4ac2fc7eff6ee437cc8cb796ca35e0cea2984dff Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 12:28:08 -0600 Subject: [PATCH 112/127] add missing column description fo SECONDS_PER_ETH1_BLOCK Co-authored-by: Diederik Loerakker --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 0e4e2b7a0..419c88e65 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -220,7 +220,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | :-: | :-: | | `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | -| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | From 607e23949c6e489c2f399bbbbaab7247cd8fc1be Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 19 May 2020 07:51:53 +0200 Subject: [PATCH 113/127] require blocks to be ordered consecutively in block range request Per the spec, if I request range 5-10, it is permissible for a client to answer with block 7, 9 - even if the blocks 5, 6 and 8 exist. Because blocks 7 and 9 cannot be validated as they arrive in such a request, it seems better to close this gap - this update adds the spec language that forbids well-behaving clients from answering this way. --- specs/phase0/p2p-interface.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 0e8699555..fc457c58a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -542,12 +542,14 @@ The response MUST consist of zero or more `response_chunk`. Each _successful_ `r Clients MUST keep a record of signed blocks seen since the since the start of the weak subjectivity period and MUST support serving requests of blocks up to their own `head_block_root`. -Clients MUST respond with at least one block, if they have it and it exists in the range. Clients MAY limit the number of blocks in the response. +Clients MUST respond with at least the first block that exists in the range, if they have it. + +The following blocks MUST be ordered consecutively, with each `parent_root` matching the `hash_tree_root` of the previous block. + +Clients MAY limit the number of blocks in the response. The response MUST contain no more than `count` blocks. -Clients MUST order blocks by increasing slot number. - Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. From a29cbebc0e314892865667b23db8ff75f76a3983 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 20 May 2020 09:51:47 +0200 Subject: [PATCH 114/127] cover `step` parameter in stricter range request --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index fc457c58a..787c94c34 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -544,7 +544,7 @@ Clients MUST keep a record of signed blocks seen since the since the start of th Clients MUST respond with at least the first block that exists in the range, if they have it. -The following blocks MUST be ordered consecutively, with each `parent_root` matching the `hash_tree_root` of the previous block. +The following blocks, where they exist, MUST be send in consecutive order without gaps, as selected by `step`. In particular, when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. Clients MAY limit the number of blocks in the response. From 59a43142c2f9181b05006638019a24dfb5550a35 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 20:39:52 +0200 Subject: [PATCH 115/127] Rebased on latest BlocksByRange spec, fix conflicts, clarify single chain, even with higher step --- specs/phase0/p2p-interface.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 787c94c34..e536c239a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -544,7 +544,7 @@ Clients MUST keep a record of signed blocks seen since the since the start of th Clients MUST respond with at least the first block that exists in the range, if they have it. -The following blocks, where they exist, MUST be send in consecutive order without gaps, as selected by `step`. In particular, when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. +The following blocks, where they exist, MUST be send in consecutive order. Clients MAY limit the number of blocks in the response. @@ -552,7 +552,10 @@ The response MUST contain no more than `count` blocks. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +This applies to any `step` value. In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. + +After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot From 522e34efcfca10f0c8f53e7757cdd172e119b5e7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 20:46:08 +0200 Subject: [PATCH 116/127] Fix markdown, use multiple lines for change-control, and add step >= 1 case --- specs/phase0/p2p-interface.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index e536c239a..118be8839 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -532,7 +532,10 @@ Response Content: ) ``` -Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. +`step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. +In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +A request MUST NOT have a 0 slot increment, i.e. `step >= 1`. `BeaconBlocksByRange` is primarily used to sync historical blocks. From 763d74bbf559e6ed73d50180a617f33ddcfac3b2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 02:51:38 +0800 Subject: [PATCH 117/127] Just learned bls was disabled by default; fixing the tests --- .../test/validator/test_validator_unittest.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 5bb246ed5..f21be8cd3 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.context import spec_state_test, always_bls, with_all_phases from eth2spec.test.helpers.attestations import build_attestation_data from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -8,9 +8,11 @@ from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey, signing_ssz_object=None): + if signing_ssz_object is None: + signing_ssz_object = obj signature = get_signature_fn(state, obj, privkey) - signing_root = spec.compute_signing_root(obj, domain) + signing_root = spec.compute_signing_root(signing_ssz_object, domain) assert bls.Verify(pubkey, signing_root, signature) @@ -55,7 +57,6 @@ def get_mock_aggregate(spec): @with_all_phases @spec_state_test -@never_bls def test_check_if_validator_active(spec, state): active_validator_index = len(state.validators) - 1 assert spec.check_if_validator_active(state, active_validator_index) @@ -73,7 +74,6 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 @@ -82,7 +82,6 @@ def test_get_committee_assignment_current_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_next_epoch(spec, state): epoch = spec.get_current_epoch(state) + 1 validator_index = len(state.validators) - 1 @@ -91,7 +90,6 @@ def test_get_committee_assignment_next_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_out_bound_epoch(spec, state): epoch = spec.get_current_epoch(state) + 2 validator_index = len(state.validators) - 1 @@ -100,7 +98,6 @@ def test_get_committee_assignment_out_bound_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_is_proposer(spec, state): proposer_index = spec.get_beacon_proposer_index(state) assert spec.is_proposer(state, proposer_index) @@ -132,6 +129,7 @@ def test_get_epoch_signature(spec, state): get_signature_fn=spec.get_epoch_signature, privkey=privkey, pubkey=pubkey, + signing_ssz_object=spec.compute_epoch_at_slot(block.slot), ) @@ -256,6 +254,7 @@ def test_compute_new_state_root(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -277,6 +276,7 @@ def test_get_block_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_attestation_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -298,6 +298,7 @@ def test_get_attestation_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -316,6 +317,7 @@ def test_get_slot_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_is_aggregator(spec, state): # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` # if we have more validators and larger committeee size @@ -334,9 +336,10 @@ def test_is_aggregator(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_signature(spec, state): attestations = [] - pubkeys = [] + attesting_pubkeys = [] slot = state.slot committee_index = 0 attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) @@ -348,24 +351,26 @@ def test_get_aggregate_signature(spec, state): committee_size = len(beacon_committee) aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) for i, validator_index in enumerate(beacon_committee): - bits = aggregation_bits + bits = aggregation_bits.copy() bits[i] = True attestations.append( spec.Attestation( data=attestation_data, aggregation_bits=bits, + signature=spec.get_attestation_signature(state, attestation_data, privkeys[validator_index]), ) ) - pubkeys.append(state.validators[validator_index].pubkey) - pubkey = bls.AggregatePKs(pubkeys) + attesting_pubkeys.append(state.validators[validator_index].pubkey) + assert len(attestations) > 0 signature = spec.get_aggregate_signature(attestations) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + assert bls.FastAggregateVerify(attesting_pubkeys, signing_root, signature) @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof(spec, state): privkey = privkeys[0] aggregator_index = spec.ValidatorIndex(10) @@ -378,6 +383,7 @@ def test_get_aggregate_and_proof(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] From d92efdf071920f0717f4609e30e8549cdd270720 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 May 2020 03:02:02 +0800 Subject: [PATCH 118/127] Should have signed the attestions in `test_filtered_block_tree` test --- tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..4371bffd0 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -183,7 +183,7 @@ def test_filtered_block_tree(spec, state): for i in range(spec.SLOTS_PER_EPOCH): slot = rogue_block.slot + i for index in range(spec.get_committee_count_at_slot(non_viable_state, slot)): - attestation = get_valid_attestation(spec, non_viable_state, rogue_block.slot + i, index) + attestation = get_valid_attestation(spec, non_viable_state, slot, index, signed=True) attestations.append(attestation) # tick time forward to be able to include up to the latest attestation From 96f785e84be320229ab921476dc83108105505e3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 13:23:59 -0600 Subject: [PATCH 119/127] ensure only forward progress with eth1data voting --- setup.py | 5 +- specs/phase0/validator.md | 14 ++++-- .../test/validator/test_validator_unittest.py | 46 +++++++++++++++++-- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index da56175af..6d7d47767 100644 --- a/setup.py +++ b/setup.py @@ -150,7 +150,10 @@ def get_eth1_data(block: Eth1Block) -> Eth1Data: """ A stub function return mocking Eth1Data. """ - return Eth1Data(block_hash=hash_tree_root(block)) + return Eth1Data( + deposit_root=block.deposit_root, + deposit_count=block.deposit_count, + block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 80e9d48c5..5e8ddc977 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -252,11 +252,13 @@ The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 d ###### `Eth1Block` -Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` field available. +Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` and depost contract data available. ```python class Eth1Block(Container): timestamp: uint64 + deposit_root: Root + deposit_count: uint64 # All other eth1 block fields ``` @@ -289,8 +291,14 @@ def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: period_start = voting_period_start_time(state) # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [get_eth1_data(block) for block in eth1_chain if - is_candidate_block(block, period_start)] + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] # Valid votes already cast during this period valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 5bb246ed5..9ef499313 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -190,8 +190,14 @@ def test_get_eth1_vote_consensus_vote(spec, state): assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -218,8 +224,14 @@ def test_get_eth1_vote_tie(spec, state): assert votes_length > 0 and votes_length % 2 == 0 state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] # Half votes are for block_1, another half votes are for block_2 @@ -237,6 +249,32 @@ def test_get_eth1_vote_tie(spec, state): assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() +@with_all_phases +@spec_state_test +def test_get_eth1_vote_chain_in_past(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count - 1, # Chain prior to current eth1data + ) + eth1_chain = [block_1] + eth1_data_votes = [] + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + + # Should be default vote + assert eth1_data == state.eth1_data + + @with_all_phases @spec_state_test def test_compute_new_state_root(spec, state): From 61336a9c0b254b8dd3ffdaa7a0e7a0d480c0251a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 20 May 2020 14:03:16 -0600 Subject: [PATCH 120/127] add deposit_root to validator unit tets --- .../eth2spec/test/validator/test_validator_unittest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 9ef499313..f79fd3fad 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -193,10 +193,12 @@ def test_get_eth1_vote_consensus_vote(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, ) block_2 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, - deposit_count=state.eth1_data.deposit_count, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -227,10 +229,12 @@ def test_get_eth1_vote_tie(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, ) block_2 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, - deposit_count=state.eth1_data.deposit_count, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -264,6 +268,7 @@ def test_get_eth1_vote_chain_in_past(spec, state): block_1 = spec.Eth1Block( timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, deposit_count=state.eth1_data.deposit_count - 1, # Chain prior to current eth1data + deposit_root=b'\x42' * 32, ) eth1_chain = [block_1] eth1_data_votes = [] From 87005c6f10cd67f919c1b286dbfcc767172f416d Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 20 May 2020 22:32:47 +0200 Subject: [PATCH 121/127] milagro bls 1.3 with improved error handling --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 792ef42fa..c88edfe56 100644 --- a/setup.py +++ b/setup.py @@ -503,7 +503,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==4.0.0", - "milagro_bls_binding==1.2.0", + "milagro_bls_binding==1.3.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", From 5e8457e62d1272e36de0f55ce3e83543258cddf1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 23 May 2020 21:53:55 +0800 Subject: [PATCH 122/127] Fix phase1 on-time sign_indexed_attestation --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 5 ++++- .../block_processing/test_process_attester_slashing.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 79f752411..4d246c8c6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -211,7 +211,10 @@ def sign_indexed_attestation(spec, state, indexed_attestation): indexed_attestation.attestation.aggregation_bits, ) data = indexed_attestation.attestation.data - indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) + if any(indexed_attestation.attestation.custody_bits_blocks): + sign_on_time_attestation(spec, state, indexed_attestation.attestation) + else: + indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) def sign_on_time_attestation(spec, state, attestation): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 8d7638f51..11ead6033 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,6 +1,6 @@ from eth2spec.test.context import ( PHASE0, PHASE1, - spec_state_test, expect_assertion_error, always_bls, never_bls, with_all_phases, with_phases + spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ @@ -89,7 +89,6 @@ def test_success_double(spec, state): @with_all_phases @spec_state_test -@never_bls def test_success_surround(spec, state): next_epoch_via_block(spec, state) From ce1d22d71c3758b77d3b2e3a0b00efc9d5c5d4cd Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sat, 23 May 2020 15:22:49 -0700 Subject: [PATCH 123/127] Use helper `compute_previous_slot` --- specs/phase1/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 3bb01e262..d14e3bce4 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -801,7 +801,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] - state.shard_states[shard].slot = state.slot - 1 + state.shard_states[shard].slot = compute_previous_slot(state.slot) ``` ###### `process_crosslink_for_shard` @@ -953,7 +953,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it for shard in range(get_active_shard_count(state)): - if state.shard_states[shard].slot != state.slot - 1: + if state.shard_states[shard].slot != compute_previous_slot(state.slot): assert block_body.shard_transitions[shard] == ShardTransition() ``` From d3c26d6b8b16f130a340568b3caf93a31ae2203e Mon Sep 17 00:00:00 2001 From: ericsson Date: Mon, 25 May 2020 18:45:38 +0300 Subject: [PATCH 124/127] `compute_shard_transition_digest` expects `Root` as a fourth parameter --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 55b867faa..da3aa648c 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -81,7 +81,7 @@ def shard_state_transition(beacon_state: BeaconState, beacon_state, shard_state, block.beacon_parent_root, - block.body, + hash_tree_root(block.body), ) shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) shard_state.slot = block.slot From 75787d92a8645751338350ea7d95cf27cb8119fa Mon Sep 17 00:00:00 2001 From: Ali Atiia <42751398+aliatiia@users.noreply.github.com> Date: Mon, 25 May 2020 16:45:44 -0400 Subject: [PATCH 125/127] broken link broken link to custory-game.md --- specs/phase1/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 3bb01e262..321b828cc 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -997,7 +997,7 @@ def process_epoch(state: BeaconState) -> None: #### Custody game updates -`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./1_custody-game.md), +`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./custody-game.md), #### Online-tracking From 75633cfcf1139858d002fe01c68d5f37d22bc28b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 26 May 2020 11:00:02 -0700 Subject: [PATCH 126/127] Update link so gossipsub encodings match the correct section --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 118be8839..a994b0c66 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -203,7 +203,7 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot - `current_fork_version` is the fork version of the epoch of the message to be sent on the topic - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` - `Name` - see table below -- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details. +- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encodings) section for further details. *Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. From c437578280a3be99c5f2e9c61ad39ff14dbbdaa7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 28 May 2020 21:32:27 +0800 Subject: [PATCH 127/127] Add `shard` field to `ShardBlock` --- specs/phase1/beacon-chain.md | 5 ++++- specs/phase1/shard-transition.md | 1 + tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 0ae4bae2b..39a73c0aa 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -302,6 +302,7 @@ class ShardBlock(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body: ByteList[MAX_SHARD_BLOCK_SIZE] ``` @@ -321,6 +322,7 @@ class ShardBlockHeader(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body_root: Root ``` @@ -781,8 +783,9 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr header = ShardBlockHeader( shard_parent_root=shard_parent_root, beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]), - proposer_index=proposal_index, slot=offset_slots[i], + shard=shard, + proposer_index=proposal_index, body_root=transition.shard_data_roots[i] ) shard_parent_root = hash_tree_root(header) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index da3aa648c..53402bb2a 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,7 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index ef65d2427..805b955f7 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -38,6 +38,7 @@ def build_shard_block(spec, shard_parent_root=shard_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, + shard=shard, proposer_index=proposer_index, body=body, )