From 34cd96be54ba59c5715900b9b55f16e069b42daa Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Wed, 13 Mar 2019 02:47:29 -0500 Subject: [PATCH 01/32] Added light client related files --- specs/light_client/merkle_proofs.md | 134 ++++++++++++++++++++++ specs/light_client/sync_protocol.md | 172 ++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 specs/light_client/merkle_proofs.md create mode 100644 specs/light_client/sync_protocol.md diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md new file mode 100644 index 000000000..cf4dad2e3 --- /dev/null +++ b/specs/light_client/merkle_proofs.md @@ -0,0 +1,134 @@ +### Generalized Merkle tree index + +In a binary Merkle tree, we define a "generalized index" of a node as `2**depth + index`. Visually, this looks as follows: + +``` + 1 + 2 3 +4 5 6 7 + ... +``` + +Note that the generalized index has the convenient property that the two children of node `k` are `2k` and `2k+1`, and also that it equals the position of a node in the linear representation of the Merkle tree that's computed by this function: + +```python +def merkle_tree(leaves): + o = [0] * len(leaves) + leaves + for i in range(len(leaves)-1, 0, -1): + o[i] = hash(o[i*2] + o[i*2+1]) + return o +``` + +We will define Merkle proofs in terms of generalized indices. + +### SSZ object to index + +We can describe the hash tree of any SSZ object, rooted in `hash_tree_root(object)`, as a binary Merkle tree whose depth may vary. For example, an object `{x: bytes32, y: List[uint64]}` would look as follows: + +``` + root + / \ + x y_root + / \ +y_data_root len(y) + / \ + /\ /\ + ....... +``` + +We can now define a concept of a "path", a way of describing a function that takes as input an SSZ object and outputs some specific (possibly deeply nested) member. For example, `foo -> foo.x` is a path, as are `foo -> len(foo.y)` and `foo -> foo[5]`. We'll describe paths as lists: in these three cases they are `["x"]`, `["y", "len"]` and `["y", 5]` respectively. We can now define a function `get_generalized_indices(object: Any, path: List[str OR int], root=1: int) -> int` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access. + +```python +def get_generalized_indices(obj: Any, path: List[str or int], root=1) -> List[int]: + if len(path) == 0: + return [root] + elif isinstance(obj, StaticList): + items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 + new_root = root * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk + return get_generalized_indices(obj[path[0]], path[1:], new_root) + elif isinstance(obj, DynamicList) and path[0] == "len": + return [root * 2 + 1] + elif isinstance(obj, DynamicList) and isinstance(path[0], int): + assert path[0] < len(obj) + items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1 + new_root = root * 2 * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk + return [root *2 + 1] + get_generalized_indices(obj[path[0]], path[1:], new_root) + elif hasattr(obj, "fields"): + index = list(fields.keys()).index(path[0]) + new_root = root * next_power_of_2(len(fields)) + index + return get_generalized_indices(getattr(obj, path[0]), path[1:], new_root) + else: + raise Exception("Unknown type / path") +``` + +### Merkle multiproofs + +We define a Merkle multiproof as a minimal subset of nodes in a Merkle tree needed to fully authenticate that a set of nodes actually are part of a Merkle tree with some specified root, at a particular set of generalized indices. For example, here is the Merkle multiproof for positions 0, 1, 6 in an 8-node Merkle tree (ie. generalized indices 8, 9, 14): + +``` + . + . . + . * * . +x x . . . . x * +``` + +. are unused nodes, * are used nodes, x are the values we are trying to prove. Notice how despite being a multiproof for 3 values, it requires only 3 auxiliary nodes, only one node more than would be required to prove a single value. Normally the efficiency gains are not quite that extreme, but the savings relative to individual Merkle proofs are still significant. As a rule of thumb, a multiproof for k nodes at the same level of an n-node tree has size `k * (n/k + log(n/k))`. + +Here is code for creating and verifying a multiproof. First a helper: + +```python +def log2(x): + return 0 if x == 1 else 1 + log2(x//2) +``` + +First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require: + +```python +def get_proof_indices(tree_indices: List[int]) -> List[int]: + # Get all indices touched by the proof + maximal_indices = set({}) + for i in tree_indices: + x = i + while x > 1: + maximal_indices.add(x ^ 1) + x //= 2 + maximal_indices = tree_indices + sorted(list(maximal_indices))[::-1] + # Get indices that cannot be recalculated from earlier indices + redundant_indices = set({}) + proof = [] + for index in maximal_indices: + if index not in redundant_indices: + proof.append(index) + while index > 1: + redundant_indices.add(index) + if (index ^ 1) not in redundant_indices: + break + index //= 2 + return [i for i in proof if i not in tree_indices] +```` + +Generating a proof is simply a matter of taking the node of the SSZ hash tree with the union of the given generalized indices for each index given by `get_proof_indices`, and outputting the list of nodes in the same order. + +```python +def verify_multi_proof(root, indices, leaves, proof): + tree = {} + for index, leaf in zip(indices, leaves): + tree[index] = leaf + for index, proofitem in zip(get_proof_indices(indices), proof): + tree[index] = proofitem + indexqueue = sorted(tree.keys())[:-1] + i = 0 + while i < len(indexqueue): + index = indexqueue[i] + if index >= 2 and index^1 in tree: + tree[index//2] = hash(tree[index - index%2] + tree[index - index%2 + 1]) + indexqueue.append(index//2) + i += 1 + return (indices == []) or (1 in tree and tree[1] == root) +``` + +#### Proofs for execution + +We define `MerklePartial(f, arg1, arg2...)` as being a list of Merkle multiproofs of the sets of nodes in the hash trees of the SSZ objects that are needed to authenticate the values needed to compute some function `f(arg1, arg2...)`. An individual Merkle multiproof is given as a dynamic sized list of `bytes32` values, a `MerklePartial` is a fixed-size list of objects `{proof: ["bytes32"], value: "bytes32"}`, one for each `arg` to `f` (if some `arg` is a base type, then the multiproof is empty). + +Ideally, any function which accepts an SSZ object should also be able to accept a `MerklePartial` object as a substitute. diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md new file mode 100644 index 000000000..8878545bb --- /dev/null +++ b/specs/light_client/sync_protocol.md @@ -0,0 +1,172 @@ +# Beacon chain light client syncing + +One of the design goals of the eth2 beacon chain is light-client friendlines, both to allow low-resource clients (mobile phones, IoT, etc) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the eth2 beacon chain and other chains. + +### Preliminaries + +We define an "expansion" of an object as an object where a field in an object that is meant to represent the `hash_tree_root` of another object is replaced by the object. Note that defining expansions is not a consensus-layer-change; it is merely a "re-interpretation" of the object. Particularly, the `hash_tree_root` of an expansion of an object is identical to that of the original object, and we can define expansions where, given a complete history, it is always possible to compute the expansion of any object in the history. The opposite of an expansion is a "summary" (eg. `BeaconBlockHeader` is a summary of `BeaconBlock`). + +We define two expansions: + +* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState` +* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])` + +Note that there is now a new way to compute `get_active_validator_indices`: + +```python +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: + return state.latest_active_indices[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] +``` + +Note that it takes `state` instead of `state.validator_registry` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments. + +A `MerklePartial(f, *args)` is an object that contains a minimal Merkle proof needed to compute `f(*args)`. A `MerklePartial` can be used in place of a regular SSZ object, though a computation would return an error if it attempts to access part of the object that is not contained in the proof. + +We add a data type `PeriodData` and four helpers: + +```python +{ + 'validator_count': 'uint64', + 'seed': 'bytes32', + 'committee': [Validator] +} +``` + +```python +def get_earlier_start_epoch(slot: Slot) -> int: + return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD * 2 + +def get_later_start_epoch(slot: Slot) -> int: + return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD + +def get_earlier_period_data(block: ExtendedBeaconBlock, shard_id: Shard) -> PeriodData: + period_start = get_earlier_start_epoch(header.slot) + validator_count = len(get_active_validator_indices(state, period_start)) + committee_count = validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE) + 1 + indices = get_shuffled_committee(block.state, shard_id, period_start, 0, committee_count) + return PeriodData( + validator_count, + generate_seed(block.state, period_start), + [block.state.validator_registry[i] for i in indices] + ) + +def get_later_period_data(block: ExtendedBeaconBlock, shard_id: Shard) -> PeriodData: + period_start = get_later_start_epoch(header.slot) + validator_count = len(get_active_validator_indices(state, period_start)) + committee_count = validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE) + 1 + indices = get_shuffled_committee(block.state, shard_id, period_start, 0, committee_count) + return PeriodData( + validator_count, + generate_seed(block.state, period_start), + [block.state.validator_registry[i] for i in indices] + ) +``` + +### Light client state + +A light client will keep track of: + +* A random `shard_id` in `[0...SHARD_COUNT-1]` (selected once and retained forever) +* A block header that they consider to be finalized (`finalized_header`) and do not expect to revert. +* `later_period_data = get_maximal_later_committee(finalized_header, shard_id)` +* `earlier_period_data = get_maximal_earlier_committee(finalized_header, shard_id)` + +We use the struct `validator_memory` to keep track of these variables. + +### Updating the shuffled committee + +If a client's `validator_memory.finalized_header` changes so that `header.slot // PERSISTENT_COMMITTEE_PERIOD` increases, then the client can ask the network for a `new_committee_proof = MerklePartial(get_maximal_later_committee, validator_memory.finalized_header, shard_id)`. It can then compute: + +```python +earlier_period_data = later_period_data +later_period_data = get_later_period_data(new_committee_proof, finalized_header, shard_id) +``` + +The maximum size of a proof is `128 * ((22-7) * 32 + 110) = 75520` bytes for validator records and `(22-7) * 32 + 128 * 8 = 1504` for the active index proof (much smaller because the relevant active indices are all beside each other in the Merkle tree). This needs to be done once per `PERSISTENT_COMMITTEE_PERIOD` epochs (2048 epochs / 9 days), or ~38 bytes per epoch. + +### Computing the current committee + +Here is a helper to compute the committee at a slot given the maximal earlier and later committees: + +```python +def compute_committee(header: BeaconBlockHeader, + validator_memory: ValidatorMemory): + + earlier_validator_count = validator_memory.earlier_period_data.validator_count + later_validator_count = validator_memory.later_period_data.validator_count + earlier_committee = validator_memory.earlier_period_data.committee + later_committee = validator_memory.later_period_data.committee + earlier_start_epoch = get_earlier_start_epoch(header.slot) + later_start_epoch = get_later_start_epoch(header.slot) + epoch = slot_to_epoch(header.slot) + + actual_committee_count = max( + earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), + later_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE), + ) + 1 + + def get_offset(count, end:bool): + return get_split_offset(count, + SHARD_COUNT * committee_count, + validator_memory.shard_id * committee_count + (1 if end else 0)) + + actual_earlier_committee = maximal_earlier_committee[ + 0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False) + ] + actual_later_committee = maximal_later_committee[ + 0:get_offset(later_validator_count, True) - get_offset(later_validator_count, False) + ] + def get_switchover_epoch(index): + return ( + bytes_to_int(hash(validator_memory.earlier_period_data.seed + bytes3(index))[0:8]) % + PERSISTENT_COMMITTEE_PERIOD + ) + + # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from + # later committee; return a sorted list of the union of the two, deduplicated + return sorted(list(set( + [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] + ))) + +``` + +Note that this method makes use of the fact that the committee for any given shard always starts and ends at the same validator index independently of the committee count (this is because the validator set is split into `SHARD_COUNT * committee_count` slices but the first slice of a shard is a multiple `committee_count * i`, so the start of the slice is `n * committee_count * i // (SHARD_COUNT * committee_count) = n * i // SHARD_COUNT`, using the slightly nontrivial algebraic identity `(x * a) // ab == x // b`). + +### Verifying blocks + +If a client wants to update its `finalized_header` it asks the network for a `BlockValidityProof`, which is simply: + +```python +{ + 'header': BlockHeader, + 'shard_aggregate_signature': 'bytes96', + 'shard_bitfield': 'bytes', + 'shard_parent_block': ShardBlock +} +``` + +The verification procedure is as follows: + +```python +def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: ValidatorMemory) -> bool: + assert proof.shard_parent_block.beacon_chain_ref == hash_tree_root(proof.header) + committee = compute_committee(proof.header, validator_memory) + # Verify that we have >=50% support + support_balance = sum([c.high_balance for i, c in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + total_balance = sum([c.high_balance for i, c in enumerate(committee)] + assert support_balance * 2 > total_balance + # Verify shard attestations + group_public_key = bls_aggregate_pubkeys([ + v.pubkey for v, index in enumerate(committee) if + get_bitfield_bit(proof.shard_bitfield, i) is True + ]) + assert bls_verify( + pubkey=group_public_key, + message_hash=hash_tree_root(shard_parent_block), + signature=shard_aggregate_signature, + domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER) + ) +``` + +The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_ref, ShardBlock)`, which would cut off ~220 bytes. From 24468de23bf1e0e4059ff5eeb5f9a5c621bc4f75 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 14 Mar 2019 20:28:44 -0500 Subject: [PATCH 02/32] Change get_shuffling to compute_committee See #729 and #774 The behavior now is that the first committee will consist of `get_permuted_index(0..n-1)`, the second committee `get_permuted_index(n....2n-1)`, etc. --- specs/core/0_beacon-chain.md | 47 +++++++++++++++--------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index daa1bc108..206aebf76 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -63,7 +63,7 @@ - [`get_permuted_index`](#get_permuted_index) - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - - [`get_shuffling`](#get_shuffling) + - [`compute_committee`](#compute_committee) - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count) - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count) @@ -803,28 +803,26 @@ def get_epoch_committee_count(active_validator_count: int) -> int: ) * SLOTS_PER_EPOCH ``` -### `get_shuffling` +### `compute_committee` ```python -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: +def compute_committee(validator_indices: [int], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: """ - Shuffle active validators and split into crosslink committees. - Return a list of committees (each a list of validator indices). + Return the index'th shuffled committee out of a total `total_committees` + using the given validator_indices and seed """ - # Shuffle active validator indices - active_validator_indices = get_active_validator_indices(validators, epoch) - length = len(active_validator_indices) - shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] - - # Split the shuffled active validator indices - return split(shuffled_indices, get_epoch_committee_count(length)) + start_offset = get_split_offset(len(validator_indices), total_committees, index) + end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) + return [ + validator_indices[get_permuted_index(i, len(validator_indices), seed)] + for i in range(start_offset, end_offset) + ] ``` -**Invariant**: if `get_shuffling(seed, validators, epoch)` returns some value `x` for some `epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY`, it should return the same value `x` for the same `seed` and `epoch` and possible future modifications of `validators` forever in phase 0, and until the ~1 year deletion delay in phase 2 and in the future. - -**Note**: this definition and the next few definitions make heavy use of repetitive computing. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. +**Note**: this definition and the next few definitions are highly inefficient as algorithms as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. ### `get_previous_epoch_committee_count` @@ -916,22 +914,17 @@ def get_crosslink_committees_at_slot(state: BeaconState, shuffling_epoch = state.current_shuffling_epoch shuffling_start_shard = state.current_shuffling_start_shard - shuffling = get_shuffling( - seed, - state.validator_registry, - shuffling_epoch, - ) - offset = slot % SLOTS_PER_EPOCH - committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT - + indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) + committee_count = get_epoch_committee_count(len(indices)) + committees_per_slot = committee_count // EPOCH_LENGTH return [ ( - shuffling[committees_per_slot * offset + i], + compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) ] + ``` ### `get_block_root` From 5d327b63646d8831412853e7c972f866b78e628e Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 09:43:38 +0000 Subject: [PATCH 03/32] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 206aebf76..2dfeb7d69 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -806,7 +806,7 @@ def get_epoch_committee_count(active_validator_count: int) -> int: ### `compute_committee` ```python -def compute_committee(validator_indices: [int], +def compute_committee(validator_indices: List[ValidatorIndex], seed: Bytes32, index: int, total_committees: int) -> List[ValidatorIndex]: From 68d1c74784b8d5a1daa05b8098fa8bfb2e17b009 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 09:45:20 +0000 Subject: [PATCH 04/32] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2dfeb7d69..be3544ab8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -811,8 +811,8 @@ def compute_committee(validator_indices: List[ValidatorIndex], index: int, total_committees: int) -> List[ValidatorIndex]: """ - Return the index'th shuffled committee out of a total `total_committees` - using the given validator_indices and seed + Return the ``index``'th shuffled committee out of a total ``total_committees`` + using ``validator_indices`` and ``seed``. """ start_offset = get_split_offset(len(validator_indices), total_committees, index) end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) From 96ab535704fb18b3bbcf585159bf499a87d277bf Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 12:40:52 +0000 Subject: [PATCH 05/32] Simplify and cleanup process_attestation Improve readability and testability (by avoiding untriggerable `assert`). Fix #753. --- specs/core/0_beacon-chain.md | 86 +++++++++++++----------------------- 1 file changed, 30 insertions(+), 56 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index daa1bc108..53695aeea 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2375,65 +2375,39 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: Process ``Attestation`` transaction. Note that this function mutates ``state``. """ - # Can't submit attestations that are too far in history (or in prehistory) - assert attestation.data.slot >= GENESIS_SLOT - assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH - # Can't submit attestations too quickly - assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot - # Verify that the justified epoch and root is correct - if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state): - # Case 1: current epoch attestations - assert attestation.data.source_epoch == state.current_justified_epoch - assert attestation.data.source_root == state.current_justified_root - else: - # Case 2: previous epoch attestations - assert attestation.data.source_epoch == state.previous_justified_epoch - assert attestation.data.source_root == state.previous_justified_root - # Check that the crosslink data is valid - acceptable_crosslink_data = { - # Case 1: Latest crosslink matches the one in the state - attestation.data.previous_crosslink, - # Case 2: State has already been updated, state's latest crosslink matches the crosslink - # the attestation is trying to create - Crosslink( - crosslink_data_root=attestation.data.crosslink_data_root, - epoch=slot_to_epoch(attestation.data.slot) - ) - } - assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data - # Attestation must be nonempty! - assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield) - # Custody must be empty (to be removed in phase 1) - assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) - # Get the committee for the specific shard that this attestation is for - crosslink_committee = [ - committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot) - if shard == attestation.data.shard - ][0] - # Custody bitfield must be a subset of the attestation bitfield - for i in range(len(crosslink_committee)): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0: - assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0 - # Verify aggregate signature - participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield) - custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants] + assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot + assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)), - ], + # Check source epoch and root match current or previous justified epoch and root + assert (slot_to_epoch(attestation.data.slot), attestation.data.source_epoch, attestation.data.source_root) in { + (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), + (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), + } + + # Check crosslink data + assert attestation.data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] + assert state.latest_crosslinks[attestation.data.shard] in { + attestation.data.previous_crosslink, # Case 1: latest crosslink matches previous crosslink + Crosslink( # Case 2: latest crosslink matches current crosslink + crosslink_data_root=attestation.data.crosslink_data_root, + epoch=slot_to_epoch(attestation.data.slot), + ), + } + + # Check custody bits [to be generalised in phase 1] + assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) + + # Check aggregate signature [to be generalised in phase 1] + participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) + assert len(participants) != 0 + assert bls_verify( + pubkey=bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in participants]), + message_hash=hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), signature=attestation.aggregate_signature, domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), ) - # Crosslink data root is zero (to be removed in phase 1) - assert attestation.data.crosslink_data_root == ZERO_HASH - # Apply the attestation + + # Cache pending attestation pending_attestation = PendingAttestation( data=attestation.data, aggregation_bitfield=attestation.aggregation_bitfield, @@ -2442,7 +2416,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ) if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): state.current_epoch_attestations.append(pending_attestation) - elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state): + else: state.previous_epoch_attestations.append(pending_attestation) ``` From d8d653dd949e92e4baf368040afa0b8216922a55 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 12:51:46 +0000 Subject: [PATCH 06/32] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 53695aeea..766bdf53c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2378,8 +2378,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY - # Check source epoch and root match current or previous justified epoch and root - assert (slot_to_epoch(attestation.data.slot), attestation.data.source_epoch, attestation.data.source_root) in { + # Check target epoch, source epoch, and source root + target_epoch = slot_to_epoch(attestation.data.slot) + assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), } @@ -2390,7 +2391,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: attestation.data.previous_crosslink, # Case 1: latest crosslink matches previous crosslink Crosslink( # Case 2: latest crosslink matches current crosslink crosslink_data_root=attestation.data.crosslink_data_root, - epoch=slot_to_epoch(attestation.data.slot), + epoch=target_epoch, ), } @@ -2404,7 +2405,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pubkey=bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in participants]), message_hash=hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), signature=attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), + domain=get_domain(state.fork, target_epoch, DOMAIN_ATTESTATION), ) # Cache pending attestation @@ -2414,7 +2415,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: custody_bitfield=attestation.custody_bitfield, inclusion_slot=state.slot ) - if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): + if target_epoch == get_current_epoch(state): state.current_epoch_attestations.append(pending_attestation) else: state.previous_epoch_attestations.append(pending_attestation) From 3944fd4b1554ea928b625c6e7ae47fc6d6076737 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 18:18:37 +0000 Subject: [PATCH 07/32] Clarify empty sums in BLS spec (#782) Fix #775. --- specs/bls_signature.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index b0490b7ae..14a4f1cb7 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -110,11 +110,11 @@ def modular_squareroot(value: Fq2) -> Fq2: ### `bls_aggregate_pubkeys` -Let `bls_aggregate_pubkeys(pubkeys: List[Bytes48]) -> Bytes48` return `pubkeys[0] + .... + pubkeys[len(pubkeys)-1]`, where `+` is the elliptic curve addition operation over the G1 curve. +Let `bls_aggregate_pubkeys(pubkeys: List[Bytes48]) -> Bytes48` return `pubkeys[0] + .... + pubkeys[len(pubkeys)-1]`, where `+` is the elliptic curve addition operation over the G1 curve. (When `len(pubkeys) == 0` the empty sum is the G1 point at infinity.) ### `bls_aggregate_signatures` -Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `signatures[0] + .... + signatures[len(signatures)-1]`, where `+` is the elliptic curve addition operation over the G2 curve. +Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `signatures[0] + .... + signatures[len(signatures)-1]`, where `+` is the elliptic curve addition operation over the G2 curve. (When `len(signatures) == 0` the empty sum is the G2 point at infinity.) ## Signature verification From 6b118d2398d5506fdd5d1659e85b93b1cf9e2bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Steiger?= Date: Sat, 16 Mar 2019 05:16:47 +0100 Subject: [PATCH 08/32] Add trailing comma (#789) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index daa1bc108..f2e639a96 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -628,7 +628,7 @@ The types are defined topologically to aid in facilitating an executable version # Ethereum 1.0 chain data 'latest_eth1_data': Eth1Data, 'eth1_data_votes': [Eth1DataVote], - 'deposit_index': 'uint64' + 'deposit_index': 'uint64', } ``` From 1a0938169bce4388c4443809a221862a259d9b69 Mon Sep 17 00:00:00 2001 From: NIC Lin Date: Sat, 16 Mar 2019 20:45:39 +0800 Subject: [PATCH 09/32] Fix `get_split_offset` (#790) --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 1713c6cbf..b2f567ed8 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -118,9 +118,9 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md def get_split_offset(list_size: int, chunks: int, index: int) -> int: """ Returns a value such that for a list L, chunk count k and index i, - split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k+1, i)] + split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k, i+1)] """ - return (len(list_size) * index) // chunks + return (list_size * index) // chunks ```` #### `get_shuffled_committee` From 919b99e0aea2a8338cafcdd984a5531cfbfe08fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Steiger?= Date: Sat, 16 Mar 2019 13:46:45 +0100 Subject: [PATCH 10/32] Add missing word (#788) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f2e639a96..1d53f1c3f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -186,7 +186,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | `MAX_EXIT_DEQUEUES_PER_EPOCH` | `2**2` (= 4) | | `SHUFFLE_ROUND_COUNT` | 90 | -* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Deposit contract From 65162e037110be66dd793a37ecbc0a285e36e8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Steiger?= Date: Sat, 16 Mar 2019 21:56:10 +0100 Subject: [PATCH 11/32] Update 0_beacon-chain.md (#791) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1d53f1c3f..5ab3da052 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1042,7 +1042,7 @@ def get_attestation_participants(state: BeaconState, attestation_data: AttestationData, bitfield: bytes) -> List[ValidatorIndex]: """ - Return the participant indices at for the ``attestation_data`` and ``bitfield``. + Return the participant indices corresponding to ``attestation_data`` and ``bitfield``. """ # Find the committee in the list with the desired shard crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) From 506fdf40424e05876dfbd32bc2cece4895330185 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 17 Mar 2019 06:33:19 -0500 Subject: [PATCH 12/32] Added FixedSizeList wrappers (#777) * Added FixedSizeList wrappers Requires corresponding changes to the spec pythonizer. * FixedSizeList -> Vector --- specs/core/0_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5ab3da052..454cf105b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1528,7 +1528,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], validator_registry_update_epoch=GENESIS_EPOCH, # Randomness and committees - latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)], + latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), previous_shuffling_start_shard=GENESIS_START_SHARD, current_shuffling_start_shard=GENESIS_START_SHARD, previous_shuffling_epoch=GENESIS_EPOCH, @@ -1548,11 +1548,11 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], finalized_root=ZERO_HASH, # Recent state - latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)], - latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)], - latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)], + latest_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), + latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), + latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), + latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]), + latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]), latest_block_header=get_temporary_block_header(get_empty_block()), historical_roots=[], From 390ece7fbeb09e285f3bd79d89fcf6d9a5f75dc4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 17 Mar 2019 11:33:29 +0000 Subject: [PATCH 13/32] Rename "vector" to "tuple" in SSZ spec (#794) To be done in combination with #777. Also: * Define "fixed-size" and "variable-size" more rigorously * Use `"` vs `'` consistently * Add missing `"` --- specs/simple-serialize.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 862d13edf..378a1a7cb 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -12,7 +12,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) - - [Tuples, containers, lists](#tuples-containers-lists) + - [Vectors, containers, lists](#vectors-containers-lists) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - [Self-signed containers](#self-signed-containers) @@ -34,12 +34,14 @@ This is a **work in progress** describing typing, serialization and Merkleizatio ### Composite types * **container**: ordered heterogenous collection of values - * key-pair curly bracket notation `{}`, e.g. `{'foo': "uint64", 'bar': "bool"}` -* **tuple**: ordered fixed-length homogeneous collection of values + * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` +* **vector**: ordered fixed-length homogeneous collection of values * angle bracket notation `[type, N]`, e.g. `["uint64", N]` * **list**: ordered variable-length homogenous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` +We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size". + ### Aliases For convenience we alias: @@ -54,34 +56,34 @@ We recursively define the `serialize` function which consumes an object `value` *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, etc.) objects implicitly carry their type. -### `uintN` +### `"uintN"` ```python assert N in [8, 16, 32, 64, 128, 256] -return value.to_bytes(N // 8, 'little') +return value.to_bytes(N // 8, "little") ``` -### `bool` +### `"bool"` ```python assert value in (True, False) -return b'\x01' if value is True else b'\x00' +return b"\x01" if value is True else b"\x00" ``` -### Tuples, containers, lists +### Vectors, containers, lists -If `value` is fixed-length (i.e. does not embed a list): +If `value` is fixed-size: ```python -return ''.join([serialize(element) for element in value]) +return "".join([serialize(element) for element in value]) ``` -If `value` is variable-length (i.e. embeds a list): +If `value` is variable-size: ```python -serialized_bytes = ''.join([serialize(element) for element in value]) +serialized_bytes = "".join([serialize(element) for element in value]) assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) -serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') +serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, "little") return serialized_length + serialized_bytes ``` @@ -99,9 +101,9 @@ We first define helper functions: We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: -* `merkleize(pack(value))` if `value` is a basic object or a tuple of basic objects +* `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects * `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects -* `merkleize([hash_tree_root(element) for element in value])` if `value` is a tuple of composite objects or a container +* `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container * `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects ## Self-signed containers From 6b82f5e9995acc7ed9f2e24bb8edf213767c60e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 17 Mar 2019 19:33:42 +0800 Subject: [PATCH 14/32] Set `GENESIS_FORK_VERSION` to a `bytes4` constant (#792) * Set `GENESIS_FORK_VERSION` to a `bytes4` constant * Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 454cf105b..a631bf2fc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -208,7 +208,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | Name | Value | | - | - | -| `GENESIS_FORK_VERSION` | `0` | +| `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` | | `GENESIS_SLOT` | `2**32` | | `GENESIS_EPOCH` | `slot_to_epoch(GENESIS_SLOT)` | | `GENESIS_START_SHARD` | `0` | @@ -1517,8 +1517,8 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], slot=GENESIS_SLOT, genesis_time=genesis_time, fork=Fork( - previous_version=int_to_bytes4(GENESIS_FORK_VERSION), - current_version=int_to_bytes4(GENESIS_FORK_VERSION), + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, epoch=GENESIS_EPOCH, ), From 91a0c1ba5f6c4439345b4476c8a1637140b48f28 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 17 Mar 2019 06:44:19 -0500 Subject: [PATCH 15/32] Persistent committee size per slot reduced to max 128 (#734) * Persistent committee size per slot target 128 max 256 Cuts down the cost of verifying a shard chain and aggregating signatures for a shard chain, and also makes the shard chain signatures more usable by light clients for verification as they would only need to keep track of a max 256-sized committee. --- specs/core/1_shard-data-chains.md | 85 +++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b2f567ed8..c76f9ba08 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -17,41 +17,51 @@ At the current stage, Phase 1, while fundamentally feature-complete, is still su - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - - [Shard chains and crosslink data](#shard-chains-and-crosslink-data) - - [Helper functions](#helper-functions) +- [Shard chains and crosslink data](#shard-chains-and-crosslink-data) + - [Helper functions](#helper-functions) - [`get_split_offset`](#get_split_offset) - [`get_shuffled_committee`](#get_shuffled_committee) - [`get_persistent_committee`](#get_persistent_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - - [Data Structures](#data-structures) + - [Data Structures](#data-structures) - [Shard chain blocks](#shard-chain-blocks) - - [Shard block processing](#shard-block-processing) + - [Shard block processing](#shard-block-processing) - [Verifying shard block data](#verifying-shard-block-data) - [Verifying a crosslink](#verifying-a-crosslink) - [Shard block fork choice rule](#shard-block-fork-choice-rule) - - [Updates to the beacon chain](#updates-to-the-beacon-chain) - - [Data structures](#data-structures) +- [Updates to the beacon chain](#updates-to-the-beacon-chain) + - [Data structures](#data-structures) - [`Validator`](#validator) - [`BeaconBlockBody`](#beaconblockbody) + - [`BeaconState`](#beaconstate) - [`BranchChallenge`](#branchchallenge) - [`BranchResponse`](#branchresponse) - [`BranchChallengeRecord`](#branchchallengerecord) + - [`InteractiveCustodyChallengeRecord`](#interactivecustodychallengerecord) + - [`InteractiveCustodyChallengeInitiation`](#interactivecustodychallengeinitiation) + - [`InteractiveCustodyChallengeResponse`](#interactivecustodychallengeresponse) + - [`InteractiveCustodyChallengeContinuation`](#interactivecustodychallengecontinuation) - [`SubkeyReveal`](#subkeyreveal) - [Helpers](#helpers) - - [`get_attestation_data_merkle_depth`](#get_attestation_data_merkle_depth) + - [`get_branch_challenge_record_by_id`](#get_branch_challenge_record_by_id) + - [`get_custody_challenge_record_by_id`](#get_custody_challenge_record_by_id) + - [`get_attestation_merkle_depth`](#get_attestation_merkle_depth) - [`epoch_to_custody_period`](#epoch_to_custody_period) - [`slot_to_custody_period`](#slot_to_custody_period) - [`get_current_custody_period`](#get_current_custody_period) - [`verify_custody_subkey_reveal`](#verify_custody_subkey_reveal) - - [`prepare_validator_for_withdrawal`](#prepare_validator_for_withdrawal) + - [`verify_signed_challenge_message`](#verify_signed_challenge_message) - [`penalize_validator`](#penalize_validator) - - [Per-slot processing](#per-slot-processing) + - [Per-slot processing](#per-slot-processing) - [Operations](#operations) - [Branch challenges](#branch-challenges) - [Branch responses](#branch-responses) - [Subkey reveals](#subkey-reveals) - - [Per-epoch processing](#per-epoch-processing) - - [One-time phase 1 initiation transition](#one-time-phase-1-initiation-transition) + - [Interactive custody challenge initiations](#interactive-custody-challenge-initiations) + - [Interactive custody challenge responses](#interactive-custody-challenge-responses) + - [Interactive custody challenge continuations](#interactive-custody-challenge-continuations) + - [Per-epoch processing](#per-epoch-processing) + - [One-time phase 1 initiation transition](#one-time-phase-1-initiation-transition) @@ -128,16 +138,27 @@ def get_split_offset(list_size: int, chunks: int, index: int) -> int: ```python def get_shuffled_committee(state: BeaconState, shard: Shard, - committee_start_epoch: Epoch) -> List[ValidatorIndex]: + committee_start_epoch: Epoch, + index: int, + committee_count: int) -> List[ValidatorIndex]: """ Return shuffled committee. """ - validator_indices = get_active_validator_indices(state.validators, committee_start_epoch) + active_validator_indices = get_active_validator_indices(state.validator_registry, committee_start_epoch) + length = len(active_validator_indices) seed = generate_seed(state, committee_start_epoch) - start_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard) - end_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard + 1) + start_offset = get_split_offset( + length, + SHARD_COUNT * committee_count, + shard * committee_count + index, + ) + end_offset = get_split_offset( + length, + SHARD_COUNT * committee_count, + shard * committee_count + index + 1, + ) return [ - validator_indices[get_permuted_index(i, len(validator_indices), seed)] + active_validator_indices[get_permuted_index(i, length, seed)] for i in range(start_offset, end_offset) ] ``` @@ -147,15 +168,24 @@ def get_shuffled_committee(state: BeaconState, ```python def get_persistent_committee(state: BeaconState, shard: Shard, - epoch: Epoch) -> List[ValidatorIndex]: + slot: Slot) -> List[ValidatorIndex]: """ - Return the persistent committee for the given ``shard`` at the given ``epoch``. + Return the persistent committee for the given ``shard`` at the given ``slot``. """ - earlier_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - earlier_committee = get_shuffled_committee(state, shard, earlier_committee_start_epoch) + + earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 + later_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD - later_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD - later_committee = get_shuffled_committee(state, shard, later_committee_start_epoch) + committee_count = max( + len(get_active_validator_indices(state.validator_registry, earlier_start_epoch)) // + (SHARD_COUNT * TARGET_COMMITTEE_SIZE), + len(get_active_validator_indices(state.validator_registry, later_start_epoch)) // + (SHARD_COUNT * TARGET_COMMITTEE_SIZE), + ) + 1 + + index = slot % committee_count + earlier_committee = get_shuffled_committee(state, shard, earlier_start_epoch, index, committee_count) + later_committee = get_shuffled_committee(state, shard, later_start_epoch, index, committee_count) def get_switchover_epoch(index): return ( @@ -170,6 +200,7 @@ def get_persistent_committee(state: BeaconState, [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) ``` + #### `get_shard_proposer_index` ```python @@ -181,14 +212,14 @@ def get_shard_proposer_index(state: BeaconState, int_to_bytes8(shard) + int_to_bytes8(slot) ) - persistent_committee = get_persistent_committee(state, shard, slot_to_epoch(slot)) + persistent_committee = get_persistent_committee(state, shard, slot) # Default proposer index = bytes_to_int(seed[0:8]) % len(persistent_committee) # If default proposer exits, try the other proposers in order; if all are exited # return None (ie. no block can be proposed) validators_to_try = persistent_committee[index:] + persistent_committee[:index] for index in validators_to_try: - if is_active_validator(state.validators[index], get_current_epoch(state)): + if is_active_validator(state.validator_registry[index], get_current_epoch(state)): return index return None ``` @@ -233,14 +264,14 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Verify that `shard_block.beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. * Verify that `shard_block.beacon_chain_ref` is equal to or a descendant of the `shard_block.beacon_chain_ref` specified in the `ShardBlock` pointed to by `shard_block.parent_root`. * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. -* Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, slot_to_epoch(shard_block.slot))`. +* Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, shard_block.slot)`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` -* For every `i in range(len(persistent_committee))` where `is_active_validator(state.validators[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0` +* For every `i in range(len(persistent_committee))` where `is_active_validator(state.validator_registry[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0` * Let `proposer_index = get_shard_proposer_index(state, shard_block.shard_id, shard_block.slot)`. * Verify that `proposer_index` is not `None`. * Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. * Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_PROPOSER))` passes. -* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. +* Let `group_public_key = bls_aggregate_pubkeys([state.validator_registry[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. * Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER))` passes. ### Verifying shard block data From 828dd455ba158aaf187c2805a9561d90bbd0b20e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 10:18:57 -0600 Subject: [PATCH 16/32] add basic dependencies and build script for phase0 testing --- .gitignore | 5 + Makefile | 15 +++ requirements.txt | 6 + scripts/__init__.py | 0 scripts/phase0/__init__.py | 0 scripts/phase0/bls_stub.py | 12 ++ scripts/phase0/build_spec.py | 43 +++++++ scripts/phase0/function_puller.py | 46 +++++++ scripts/phase0/minimal_ssz.py | 190 +++++++++++++++++++++++++++++ scripts/phase0/monkey_patches.py | 29 +++++ scripts/phase0/state_transition.py | 84 +++++++++++++ tests/phase0/conftest.py | 6 + 12 files changed, 436 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 requirements.txt create mode 100644 scripts/__init__.py create mode 100644 scripts/phase0/__init__.py create mode 100644 scripts/phase0/bls_stub.py create mode 100644 scripts/phase0/build_spec.py create mode 100644 scripts/phase0/function_puller.py create mode 100644 scripts/phase0/minimal_ssz.py create mode 100644 scripts/phase0/monkey_patches.py create mode 100644 scripts/phase0/state_transition.py create mode 100644 tests/phase0/conftest.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..36c14f343 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +/__pycache__ +/venv + +/build \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..724a0392e --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +SPEC_DIR = ./specs +SCRIPT_DIR = ./scripts +BUILD_DIR = ./build + +.PHONY: clean all + + +clean: + rm -rf $(BUILD_DIR) + + +$(BUILD_DIR)/phase0: + mkdir -p $@ + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \ + $(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..9145e951e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +eth-utils>=1.3.0,<2 +eth-typing>=2.1.0,<3.0.0 +oyaml==0.7 +pycryptodome==3.7.3 +py_ecc>=1.6.0 +pytest>=3.6,<3.7 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/__init__.py b/scripts/phase0/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/bls_stub.py b/scripts/phase0/bls_stub.py new file mode 100644 index 000000000..7e3a6a308 --- /dev/null +++ b/scripts/phase0/bls_stub.py @@ -0,0 +1,12 @@ + + +def bls_verify(pubkey, message_hash, signature, domain): + return True + + +def bls_verify_multiple(pubkeys, message_hashes, signature, domain): + return True + + +def bls_aggregate_pubkeys(pubkeys): + return b'\x42'*96 diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py new file mode 100644 index 000000000..c4f8ab38c --- /dev/null +++ b/scripts/phase0/build_spec.py @@ -0,0 +1,43 @@ +import sys +import function_puller + +code_lines = [] + +for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) +code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var +code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") + +code_lines.append(""" +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None +""") + + +code_lines += function_puller.get_lines(sys.argv[1]) + +print(open(sys.argv[2]).read()) +print(open(sys.argv[3]).read()) + +for line in code_lines: + print(line) + +print(open(sys.argv[4]).read()) +print(open(sys.argv[5]).read()) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py new file mode 100644 index 000000000..8d1c1a0cc --- /dev/null +++ b/scripts/phase0/function_puller.py @@ -0,0 +1,46 @@ +import sys + + +def get_lines(file_name): + code_lines = [] + pulling_from = None + current_name = None + processing_typedef = False + for linenum, line in enumerate(open(sys.argv[1]).readlines()): + line = line.rstrip() + if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': + current_name = line[line[:-1].rfind('`')+1: -1] + if line[:9] == '```python': + assert pulling_from is None + pulling_from = linenum + 1 + elif line[:3] == '```': + if pulling_from is None: + pulling_from = linenum + else: + if processing_typedef: + assert code_lines[-1] == '}' + code_lines[-1] = '})' + pulling_from = None + processing_typedef = False + else: + if pulling_from == linenum and line == '{': + code_lines.append('%s = SSZType({' % current_name) + processing_typedef = True + elif pulling_from is not None: + code_lines.append(line) + elif pulling_from is None and len(line) > 0 and line[0] == '|': + row = line[1:].split('|') + if len(row) >= 2: + for i in range(2): + row[i] = row[i].strip().strip('`') + if '`' in row[i]: + row[i] = row[i][:row[i].find('`')] + eligible = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + eligible = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + eligible = False + if eligible: + code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890'))) + return code_lines diff --git a/scripts/phase0/minimal_ssz.py b/scripts/phase0/minimal_ssz.py new file mode 100644 index 000000000..5caaf8f09 --- /dev/null +++ b/scripts/phase0/minimal_ssz.py @@ -0,0 +1,190 @@ +from utils.hash import hash + + +BYTES_PER_CHUNK = 32 +BYTES_PER_LENGTH_PREFIX = 4 +ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK + +def SSZType(fields): + class SSZObject(): + def __init__(self, **kwargs): + for f in fields: + if f not in kwargs: + raise Exception("Missing constructor argument: %s" % f) + setattr(self, f, kwargs[f]) + + def __eq__(self, other): + return ( + self.fields == other.fields and + self.serialize() == other.serialize() + ) + + def __hash__(self): + return int.from_bytes(self.hash_tree_root(), byteorder="little") + + def __str__(self): + output = [] + for field in self.fields: + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def serialize(self): + return serialize_value(self, self.__class__) + + def hash_tree_root(self): + return hash_tree_root(self, self.__class__) + + SSZObject.fields = fields + return SSZObject + +class Vector(list): + def __init__(self, x): + list.__init__(self, x) + self.length = len(x) + + def append(*args): + raise Exception("Cannot change the length of a vector") + + remove = clear = extend = pop = insert = append + +def is_basic(typ): + return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte') + +def is_constant_sized(typ): + if is_basic(typ): + return True + elif isinstance(typ, list) and len(typ) == 1: + return is_constant_sized(typ[0]) + elif isinstance(typ, list) and len(typ) == 2: + return False + elif isinstance(typ, str) and typ[:5] == 'bytes': + return len(typ) > 5 + elif hasattr(typ, 'fields'): + for subtype in typ.fields.values(): + if not is_constant_sized(subtype): + return False + return True + else: + raise Exception("Type not recognized") + +def coerce_to_bytes(x): + if isinstance(x, str): + o = x.encode('utf-8') + assert len(o) == len(x) + return o + elif isinstance(x, bytes): + return x + else: + raise Exception("Expecting bytes") + +def serialize_value(value, typ=None): + if typ is None: + typ = infer_type(value) + if isinstance(typ, str) and typ[:4] == 'uint': + length = int(typ[4:]) + assert length in (8, 16, 32, 64, 128, 256) + return value.to_bytes(length // 8, 'little') + elif typ == 'bool': + assert value in (True, False) + return b'\x01' if value is True else b'\x00' + elif (isinstance(typ, list) and len(typ) == 1) or typ == 'bytes': + serialized_bytes = coerce_to_bytes(value) if typ == 'bytes' else b''.join([serialize_value(element, typ[0]) for element in value]) + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + elif isinstance(typ, list) and len(typ) == 2: + assert len(value) == typ[1] + return b''.join([serialize_value(element, typ[0]) for element in value]) + elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': + assert len(value) == int(typ[5:]), (value, int(typ[5:])) + return coerce_to_bytes(value) + elif hasattr(typ, 'fields'): + serialized_bytes = b''.join([serialize_value(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + if is_constant_sized(typ): + return serialized_bytes + else: + assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX) + serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little') + return serialized_length + serialized_bytes + else: + print(value, typ) + raise Exception("Type not recognized") + +def chunkify(bytez): + bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) + return [bytez[i:i+32] for i in range(0, len(bytez), 32)] + +def pack(values, subtype): + return chunkify(b''.join([serialize_value(value, subtype) for value in values])) + +def is_power_of_two(x): + return x > 0 and x & (x-1) == 0 + +def merkleize(chunks): + tree = chunks[::] + while not is_power_of_two(len(tree)): + tree.append(ZERO_CHUNK) + tree = [ZERO_CHUNK] * len(tree) + tree + for i in range(len(tree)//2-1, 0, -1): + tree[i] = hash(tree[i*2] + tree[i*2+1]) + return tree[1] + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + +def infer_type(value): + if hasattr(value.__class__, 'fields'): + return value.__class__ + elif isinstance(value, Vector): + return [infer_type(value[0]) if len(value) > 0 else 'uint64', len(value)] + elif isinstance(value, list): + return [infer_type(value[0])] if len(value) > 0 else ['uint64'] + elif isinstance(value, (bytes, str)): + return 'bytes' + elif isinstance(value, int): + return 'uint64' + else: + raise Exception("Failed to infer type") + +def hash_tree_root(value, typ=None): + if typ is None: + typ = infer_type(value) + if is_basic(typ): + return merkleize(pack([value], typ)) + elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): + return mix_in_length(merkleize(pack(value, typ[0])), len(value)) + elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): + return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) + elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): + assert len(value) == typ[1] + return merkleize(pack(value, typ[0])) + elif typ == 'bytes': + return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) + elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: + assert len(value) == int(typ[5:]) + return merkleize(chunkify(coerce_to_bytes(value))) + elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): + return merkleize([hash_tree_root(element, typ[0]) for element in value]) + elif hasattr(typ, 'fields'): + return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) + else: + raise Exception("Type not recognized") + +def truncate(container): + field_keys = list(container.fields.keys()) + truncated_fields = { + key: container.fields[key] + for key in field_keys[:-1] + } + truncated_class = SSZType(truncated_fields) + kwargs = { + field: getattr(container, field) + for field in field_keys[:-1] + } + return truncated_class(**kwargs) + +def signed_root(container): + return hash_tree_root(truncate(container)) + +def serialize(ssz_object): + return getattr(ssz_object, 'serialize')() diff --git a/scripts/phase0/monkey_patches.py b/scripts/phase0/monkey_patches.py new file mode 100644 index 000000000..8a35b8f27 --- /dev/null +++ b/scripts/phase0/monkey_patches.py @@ -0,0 +1,29 @@ +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret diff --git a/scripts/phase0/state_transition.py b/scripts/phase0/state_transition.py new file mode 100644 index 000000000..f78119cf2 --- /dev/null +++ b/scripts/phase0/state_transition.py @@ -0,0 +1,84 @@ + + +def process_transaction_type(state: BeaconState, + transactions: List[Any], + max_transactions: int, + tx_fn: Callable[[BeaconState, Any], None]) -> None: + assert len(transactions) <= max_transactions + for transaction in transactions: + tx_fn(state, transaction) + + +def process_transactions(state: BeaconState, block: BeaconBlock) -> None: + process_transaction_type( + state, + block.body.proposer_slashings, + MAX_PROPOSER_SLASHINGS, + process_proposer_slashing, + ) + process_transaction_type( + state, + block.body.attester_slashings, + MAX_ATTESTER_SLASHINGS, + process_attester_slashing, + ) + process_transaction_type( + state, + block.body.attestations, + MAX_ATTESTATIONS, + process_attestation, + ) + process_transaction_type( + state, + block.body.deposits, + MAX_DEPOSITS, + process_deposit, + ) + process_transaction_type( + state, + block.body.voluntary_exits, + MAX_VOLUNTARY_EXITS, + process_voluntary_exit, + ) + assert len(block.body.transfers) == len(set(block.body.transfers)) + process_transaction_type( + state, + block.body.transfers, + MAX_TRANSFERS, + process_transfer, + ) + + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + process_block_header(state, block) + process_randao(state, block) + process_eth1_data(state, block) + process_transactions(state, block) + if verify_state_root: + verify_block_state_root(state, block) + + +def process_epoch_transition(state: BeaconState) -> None: + update_justification_and_finalization(state) + process_crosslinks(state) + maybe_reset_eth1_period(state) + apply_rewards(state) + process_ejections(state) + update_registry_and_shuffling_data(state) + process_slashings(state) + process_exit_queue(state) + finish_epoch_update(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + while state.slot < block.slot: + cache_state(state) + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + advance_slot(state) + if block.slot == state.slot: + process_block(state, block) diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py new file mode 100644 index 000000000..d3ebabaa2 --- /dev/null +++ b/tests/phase0/conftest.py @@ -0,0 +1,6 @@ +import pytest +from build.phase0 import spec + + +# @pytest.fixture(autouse=True) +# def build_clean(): \ No newline at end of file From 839590b5f41f4b07a755ef00f7a96563858b3e7a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 12:51:52 -0600 Subject: [PATCH 17/32] initial pytests passing --- .gitignore | 3 +- Makefile | 8 +- build/__init__.py | 0 build/phase0/__init__.py | 0 build/phase0/spec.py | 1620 +++++++++++++++++ {scripts => build}/phase0/state_transition.py | 72 +- build/utils/__init__.py | 0 {scripts/phase0 => build/utils}/bls_stub.py | 0 build/utils/hash_function.py | 6 + build/utils/merkle_minimal.py | 28 + .../phase0 => build/utils}/minimal_ssz.py | 2 +- .../phase0 => build/utils}/monkey_patches.py | 0 scripts/phase0/build_spec.py | 64 +- tests/__init__.py | 0 tests/conftest.py | 0 tests/phase0/conftest.py | 132 +- tests/phase0/test_sanity.py | 632 +++++++ 17 files changed, 2514 insertions(+), 53 deletions(-) create mode 100644 build/__init__.py create mode 100644 build/phase0/__init__.py create mode 100644 build/phase0/spec.py rename {scripts => build}/phase0/state_transition.py (56%) create mode 100644 build/utils/__init__.py rename {scripts/phase0 => build/utils}/bls_stub.py (100%) create mode 100644 build/utils/hash_function.py create mode 100644 build/utils/merkle_minimal.py rename {scripts/phase0 => build/utils}/minimal_ssz.py (99%) rename {scripts/phase0 => build/utils}/monkey_patches.py (100%) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/phase0/test_sanity.py diff --git a/.gitignore b/.gitignore index 36c14f343..5e19cd2a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *.pyc /__pycache__ /venv - -/build \ No newline at end of file +/.pytest_cache diff --git a/Makefile b/Makefile index 724a0392e..745f8f901 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,7 @@ BUILD_DIR = ./build .PHONY: clean all -clean: - rm -rf $(BUILD_DIR) - - $(BUILD_DIR)/phase0: mkdir -p $@ - python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/minimal_ssz.py \ - $(SCRIPT_DIR)/phase0/bls_stub.py $(SCRIPT_DIR)/phase0/state_transition.py $(SCRIPT_DIR)/phase0/monkey_patches.py > $@/spec.py + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py + touch $(BUILD_DIR)/__init__.py $(BUILD_DIR)/phase0/__init__.py diff --git a/build/__init__.py b/build/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build/phase0/__init__.py b/build/phase0/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/build/phase0/spec.py b/build/phase0/spec.py new file mode 100644 index 000000000..8c05b1208 --- /dev/null +++ b/build/phase0/spec.py @@ -0,0 +1,1620 @@ +from build.utils.minimal_ssz import * +from build.utils.bls_stub import * +def int_to_bytes1(x): return x.to_bytes(1, 'little') +def int_to_bytes2(x): return x.to_bytes(2, 'little') +def int_to_bytes3(x): return x.to_bytes(3, 'little') +def int_to_bytes4(x): return x.to_bytes(4, 'little') +def int_to_bytes8(x): return x.to_bytes(8, 'little') +def int_to_bytes32(x): return x.to_bytes(32, 'little') +def int_to_bytes48(x): return x.to_bytes(48, 'little') +def int_to_bytes96(x): return x.to_bytes(96, 'little') +SLOTS_PER_EPOCH = 64 +def slot_to_epoch(x): return x // SLOTS_PER_EPOCH + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None + +SHARD_COUNT = 2**10 +TARGET_COMMITTEE_SIZE = 2**7 +MAX_BALANCE_CHURN_QUOTIENT = 2**5 +MAX_INDICES_PER_SLASHABLE_VOTE = 2**12 +MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 +SHUFFLE_ROUND_COUNT = 90 +DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 +DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 +MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 +MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 +FORK_CHOICE_BALANCE_INCREMENT = 2**0 * 10**9 +EJECTION_BALANCE = 2**4 * 10**9 +GENESIS_FORK_VERSION = 0 +GENESIS_SLOT = 2**32 +GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) +GENESIS_START_SHARD = 0 +FAR_FUTURE_EPOCH = 2**64 - 1 +ZERO_HASH = int_to_bytes32(0) +EMPTY_SIGNATURE = int_to_bytes96(0) +BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) +SECONDS_PER_SLOT = 6 +MIN_ATTESTATION_INCLUSION_DELAY = 2**2 +SLOTS_PER_EPOCH = 2**6 +MIN_SEED_LOOKAHEAD = 2**0 +ACTIVATION_EXIT_DELAY = 2**2 +EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 +SLOTS_PER_HISTORICAL_ROOT = 2**13 +MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 +PERSISTENT_COMMITTEE_PERIOD = 2**11 +LATEST_RANDAO_MIXES_LENGTH = 2**13 +LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 +LATEST_SLASHED_EXIT_LENGTH = 2**13 +BASE_REWARD_QUOTIENT = 2**5 +WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 +ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 +INACTIVITY_PENALTY_QUOTIENT = 2**24 +MIN_PENALTY_QUOTIENT = 2**5 +MAX_PROPOSER_SLASHINGS = 2**4 +MAX_ATTESTER_SLASHINGS = 2**0 +MAX_ATTESTATIONS = 2**7 +MAX_DEPOSITS = 2**4 +MAX_VOLUNTARY_EXITS = 2**4 +MAX_TRANSFERS = 2**4 +DOMAIN_BEACON_BLOCK = 0 +DOMAIN_RANDAO = 1 +DOMAIN_ATTESTATION = 2 +DOMAIN_DEPOSIT = 3 +DOMAIN_VOLUNTARY_EXIT = 4 +DOMAIN_TRANSFER = 5 +Fork = SSZType({ + # Previous fork version + 'previous_version': 'bytes4', + # Current fork version + 'current_version': 'bytes4', + # Fork epoch number + 'epoch': 'uint64', +}) +Crosslink = SSZType({ + # Epoch number + 'epoch': 'uint64', + # Shard data since the previous crosslink + 'crosslink_data_root': 'bytes32', +}) +Eth1Data = SSZType({ + # Root of the deposit tree + 'deposit_root': 'bytes32', + # Block hash + 'block_hash': 'bytes32', +}) +Eth1DataVote = SSZType({ + # Data being voted for + 'eth1_data': Eth1Data, + # Vote count + 'vote_count': 'uint64', +}) +AttestationData = SSZType({ + # LMD GHOST vote + 'slot': 'uint64', + 'beacon_block_root': 'bytes32', + + # FFG vote + 'source_epoch': 'uint64', + 'source_root': 'bytes32', + 'target_root': 'bytes32', + + # Crosslink vote + 'shard': 'uint64', + 'previous_crosslink': Crosslink, + 'crosslink_data_root': 'bytes32', +}) +AttestationDataAndCustodyBit = SSZType({ + # Attestation data + 'data': AttestationData, + # Custody bit + 'custody_bit': 'bool', +}) +SlashableAttestation = SSZType({ + # Validator indices + 'validator_indices': ['uint64'], + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Aggregate signature + 'aggregate_signature': 'bytes96', +}) +DepositInput = SSZType({ + # BLS pubkey + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # A BLS signature of this `DepositInput` + 'proof_of_possession': 'bytes96', +}) +DepositData = SSZType({ + # Amount in Gwei + 'amount': 'uint64', + # Timestamp from deposit contract + 'timestamp': 'uint64', + # Deposit input + 'deposit_input': DepositInput, +}) +BeaconBlockHeader = SSZType({ + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'block_body_root': 'bytes32', + 'signature': 'bytes96', +}) +Validator = SSZType({ + # BLS public key + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # Epoch when validator activated + 'activation_epoch': 'uint64', + # Epoch when validator exited + 'exit_epoch': 'uint64', + # Epoch when validator is eligible to withdraw + 'withdrawable_epoch': 'uint64', + # Did the validator initiate an exit + 'initiated_exit': 'bool', + # Was the validator slashed + 'slashed': 'bool', +}) +PendingAttestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Inclusion slot + 'inclusion_slot': 'uint64', +}) +HistoricalBatch = SSZType({ + # Block roots + 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + # State roots + 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], +}) +ProposerSlashing = SSZType({ + # Proposer index + 'proposer_index': 'uint64', + # First block header + 'header_1': BeaconBlockHeader, + # Second block header + 'header_2': BeaconBlockHeader, +}) +AttesterSlashing = SSZType({ + # First slashable attestation + 'slashable_attestation_1': SlashableAttestation, + # Second slashable attestation + 'slashable_attestation_2': SlashableAttestation, +}) +Attestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # BLS aggregate signature + 'aggregate_signature': 'bytes96', +}) +Deposit = SSZType({ + # Branch in the deposit tree + 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + # Index in the deposit tree + 'index': 'uint64', + # Data + 'deposit_data': DepositData, +}) +VoluntaryExit = SSZType({ + # Minimum epoch for processing exit + 'epoch': 'uint64', + # Index of the exiting validator + 'validator_index': 'uint64', + # Validator signature + 'signature': 'bytes96', +}) +Transfer = SSZType({ + # Sender index + 'sender': 'uint64', + # Recipient index + 'recipient': 'uint64', + # Amount in Gwei + 'amount': 'uint64', + # Fee in Gwei for block proposer + 'fee': 'uint64', + # Inclusion slot + 'slot': 'uint64', + # Sender withdrawal pubkey + 'pubkey': 'bytes48', + # Sender signature + 'signature': 'bytes96', +}) +BeaconBlockBody = SSZType({ + 'randao_reveal': 'bytes96', + 'eth1_data': Eth1Data, + 'proposer_slashings': [ProposerSlashing], + 'attester_slashings': [AttesterSlashing], + 'attestations': [Attestation], + 'deposits': [Deposit], + 'voluntary_exits': [VoluntaryExit], + 'transfers': [Transfer], +}) +BeaconBlock = SSZType({ + # Header + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'body': BeaconBlockBody, + 'signature': 'bytes96', +}) +BeaconState = SSZType({ + # Misc + 'slot': 'uint64', + 'genesis_time': 'uint64', + 'fork': Fork, # For versioning hard forks + + # Validator registry + 'validator_registry': [Validator], + 'validator_balances': ['uint64'], + 'validator_registry_update_epoch': 'uint64', + + # Randomness and committees + 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], + 'previous_shuffling_start_shard': 'uint64', + 'current_shuffling_start_shard': 'uint64', + 'previous_shuffling_epoch': 'uint64', + 'current_shuffling_epoch': 'uint64', + 'previous_shuffling_seed': 'bytes32', + 'current_shuffling_seed': 'bytes32', + + # Finality + 'previous_epoch_attestations': [PendingAttestation], + 'current_epoch_attestations': [PendingAttestation], + 'previous_justified_epoch': 'uint64', + 'current_justified_epoch': 'uint64', + 'previous_justified_root': 'bytes32', + 'current_justified_root': 'bytes32', + 'justification_bitfield': 'uint64', + 'finalized_epoch': 'uint64', + 'finalized_root': 'bytes32', + + # Recent state + 'latest_crosslinks': [Crosslink, SHARD_COUNT], + 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period + 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily + 'historical_roots': ['bytes32'], + + # Ethereum 1.0 chain data + 'latest_eth1_data': Eth1Data, + 'eth1_data_votes': [Eth1DataVote], + 'deposit_index': 'uint64' +}) +def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: + return bytes(a ^ b for a, b in zip(bytes1, bytes2)) +def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: + """ + Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. + """ + return BeaconBlockHeader( + slot=block.slot, + previous_block_root=block.previous_block_root, + state_root=ZERO_HASH, + block_body_root=hash_tree_root(block.body), + signature=block.signature, + ) +def slot_to_epoch(slot: Slot) -> Epoch: + """ + Return the epoch number of the given ``slot``. + """ + return slot // SLOTS_PER_EPOCH +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch of the given ``state``. + """ + return get_current_epoch(state) - 1 +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch of the given ``state``. + """ + return slot_to_epoch(state.slot) +def get_epoch_start_slot(epoch: Epoch) -> Slot: + """ + Return the starting slot of the given ``epoch``. + """ + return epoch * SLOTS_PER_EPOCH +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: + """ + Get indices of active validators from ``validators``. + """ + return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: + """ + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ + assert index < list_size + assert list_size <= 2**40 + + for round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size + flip = (pivot - index) % list_size + position = max(index, flip) + source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index +def split(values: List[Any], split_count: int) -> List[List[Any]]: + """ + Splits ``values`` into ``split_count`` pieces. + """ + list_length = len(values) + return [ + values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] + for i in range(split_count) + ] +def get_epoch_committee_count(active_validator_count: int) -> int: + """ + Return the number of committees in one epoch. + """ + return max( + 1, + min( + SHARD_COUNT // SLOTS_PER_EPOCH, + active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + ) + ) * SLOTS_PER_EPOCH +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + """ + Shuffle active validators and split into crosslink committees. + Return a list of committees (each a list of validator indices). + """ + # Shuffle active validator indices + active_validator_indices = get_active_validator_indices(validators, epoch) + length = len(active_validator_indices) + shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] + + # Split the shuffled active validator indices + return split(shuffled_indices, get_epoch_committee_count(length)) +def get_previous_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the previous epoch of the given ``state``. + """ + previous_active_validators = get_active_validator_indices( + state.validator_registry, + state.previous_shuffling_epoch, + ) + return get_epoch_committee_count(len(previous_active_validators)) +def get_current_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the current epoch of the given ``state``. + """ + current_active_validators = get_active_validator_indices( + state.validator_registry, + state.current_shuffling_epoch, + ) + return get_epoch_committee_count(len(current_active_validators)) +def get_next_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the next epoch of the given ``state``. + """ + next_active_validators = get_active_validator_indices( + state.validator_registry, + get_current_epoch(state) + 1, + ) + return get_epoch_committee_count(len(next_active_validators)) +def get_crosslink_committees_at_slot(state: BeaconState, + slot: Slot, + registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]: + """ + Return the list of ``(committee, shard)`` tuples for the ``slot``. + + Note: There are two possible shufflings for crosslink committees for a + ``slot`` in the next epoch -- with and without a `registry_change` + """ + epoch = slot_to_epoch(slot) + current_epoch = get_current_epoch(state) + previous_epoch = get_previous_epoch(state) + next_epoch = current_epoch + 1 + + assert previous_epoch <= epoch <= next_epoch + + if epoch == current_epoch: + committees_per_epoch = get_current_epoch_committee_count(state) + seed = state.current_shuffling_seed + shuffling_epoch = state.current_shuffling_epoch + shuffling_start_shard = state.current_shuffling_start_shard + elif epoch == previous_epoch: + committees_per_epoch = get_previous_epoch_committee_count(state) + seed = state.previous_shuffling_seed + shuffling_epoch = state.previous_shuffling_epoch + shuffling_start_shard = state.previous_shuffling_start_shard + elif epoch == next_epoch: + epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch + if registry_change: + committees_per_epoch = get_next_epoch_committee_count(state) + seed = generate_seed(state, next_epoch) + shuffling_epoch = next_epoch + current_committees_per_epoch = get_current_epoch_committee_count(state) + shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT + elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): + committees_per_epoch = get_next_epoch_committee_count(state) + seed = generate_seed(state, next_epoch) + shuffling_epoch = next_epoch + shuffling_start_shard = state.current_shuffling_start_shard + else: + committees_per_epoch = get_current_epoch_committee_count(state) + seed = state.current_shuffling_seed + shuffling_epoch = state.current_shuffling_epoch + shuffling_start_shard = state.current_shuffling_start_shard + + shuffling = get_shuffling( + seed, + state.validator_registry, + shuffling_epoch, + ) + offset = slot % SLOTS_PER_EPOCH + committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH + slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT + + return [ + ( + shuffling[committees_per_slot * offset + i], + (slot_start_shard + i) % SHARD_COUNT, + ) + for i in range(committees_per_slot) + ] +def get_block_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +def get_state_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +def get_randao_mix(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) + return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] +def get_active_index_root(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the index root at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY + return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] +def generate_seed(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Generate a seed for the given ``epoch``. + """ + return hash( + get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + + get_active_index_root(state, epoch) + + int_to_bytes32(epoch) + ) +def get_beacon_proposer_index(state: BeaconState, + slot: Slot, + registry_change: bool=False) -> ValidatorIndex: + """ + Return the beacon proposer index for the ``slot``. + """ + epoch = slot_to_epoch(slot) + current_epoch = get_current_epoch(state) + previous_epoch = get_previous_epoch(state) + next_epoch = current_epoch + 1 + + assert previous_epoch <= epoch <= next_epoch + + first_committee, _ = get_crosslink_committees_at_slot(state, slot, registry_change)[0] + return first_committee[epoch % len(first_committee)] +def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: + """ + Verify that the given ``leaf`` is on the merkle branch ``proof`` + starting with the given ``root``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(proof[i] + value) + else: + value = hash(value + proof[i]) + return value == root +def get_attestation_participants(state: BeaconState, + attestation_data: AttestationData, + bitfield: bytes) -> List[ValidatorIndex]: + """ + Return the participant indices at for the ``attestation_data`` and ``bitfield``. + """ + # Find the committee in the list with the desired shard + crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) + + assert attestation_data.shard in [shard for _, shard in crosslink_committees] + crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] + + assert verify_bitfield(bitfield, len(crosslink_committee)) + + # Find the participating attesters in the committee + participants = [] + for i, validator_index in enumerate(crosslink_committee): + aggregation_bit = get_bitfield_bit(bitfield, i) + if aggregation_bit == 0b1: + participants.append(validator_index) + return participants +def is_power_of_two(value: int) -> bool: + """ + Check if ``value`` is a power of two integer. + """ + return (value > 0) and (value & (value - 1) == 0) +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') +def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: + """ + Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + """ + return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of an array of ``validators``. + """ + return sum([get_effective_balance(state, i) for i in validators]) +def get_fork_version(fork: Fork, + epoch: Epoch) -> bytes: + """ + Return the fork version of the given ``epoch``. + """ + if epoch < fork.epoch: + return fork.previous_version + else: + return fork.current_version +def get_domain(fork: Fork, + epoch: Epoch, + domain_type: int) -> int: + """ + Get the domain number that represents the fork meta and signature domain. + """ + return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: + """ + Verify ``bitfield`` against the ``committee_size``. + """ + if len(bitfield) != (committee_size + 7) // 8: + return False + + # Check `bitfield` is padded with zero bits only + for i in range(committee_size, len(bitfield) * 8): + if get_bitfield_bit(bitfield, i) == 0b1: + return False + + return True +def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: + """ + Verify validity of ``slashable_attestation`` fields. + """ + if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] + return False + + if len(slashable_attestation.validator_indices) == 0: + return False + + for i in range(len(slashable_attestation.validator_indices) - 1): + if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: + return False + + if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): + return False + + if len(slashable_attestation.validator_indices) > MAX_INDICES_PER_SLASHABLE_VOTE: + return False + + custody_bit_0_indices = [] + custody_bit_1_indices = [] + for i, validator_index in enumerate(slashable_attestation.validator_indices): + if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: + custody_bit_0_indices.append(validator_index) + else: + custody_bit_1_indices.append(validator_index) + + return bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), + ], + signature=slashable_attestation.aggregate_signature, + domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), + ) +def is_double_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. + """ + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + return target_epoch_1 == target_epoch_2 +def is_surround_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` surrounds ``attestation_data_2``. + """ + source_epoch_1 = attestation_data_1.source_epoch + source_epoch_2 = attestation_data_2.source_epoch + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + + return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 +def integer_squareroot(n: int) -> int: + """ + The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. + """ + assert n >= 0 + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + """ + return epoch + 1 + ACTIVATION_EXIT_DELAY +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + """ + Process a deposit from Ethereum 1.0. + Note that this function mutates ``state``. + """ + deposit_input = deposit.deposit_data.deposit_input + + # Should equal 8 bytes for deposit_data.amount + + # 8 bytes for deposit_data.timestamp + + # 176 bytes for deposit_data.deposit_input + # It should match the deposit_data in the eth1.0 deposit contract + serialized_deposit_data = serialize(deposit.deposit_data) + # Deposits must be processed in order + assert deposit.index == state.deposit_index + + # Verify the Merkle branch + merkle_branch_is_valid = verify_merkle_branch( + leaf=hash(serialized_deposit_data), + proof=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH, + index=deposit.index, + root=state.latest_eth1_data.deposit_root, + ) + assert merkle_branch_is_valid + + # Increment the next deposit index we are expecting. Note that this + # needs to be done here because while the deposit contract will never + # create an invalid Merkle branch, it may admit an invalid deposit + # object, and we need to be able to skip over it + state.deposit_index += 1 + + validator_pubkeys = [v.pubkey for v in state.validator_registry] + pubkey = deposit_input.pubkey + amount = deposit.deposit_data.amount + withdrawal_credentials = deposit_input.withdrawal_credentials + + if pubkey not in validator_pubkeys: + # Verify the proof of possession + proof_is_valid = bls_verify( + pubkey=deposit_input.pubkey, + message_hash=signed_root(deposit_input), + signature=deposit_input.proof_of_possession, + domain=get_domain( + state.fork, + get_current_epoch(state), + DOMAIN_DEPOSIT, + ) + ) + if not proof_is_valid: + return + + # Add new validator + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + initiated_exit=False, + slashed=False, + ) + + # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. + state.validator_registry.append(validator) + state.validator_balances.append(amount) + else: + # Increase balance by deposit amount + state.validator_balances[validator_pubkeys.index(pubkey)] += amount +def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: + """ + Activate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + + validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.initiated_exit = True +def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Exit the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + delayed_activation_exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) + + # The following updates only occur if not previous exited + if validator.exit_epoch <= delayed_activation_exit_epoch: + return + else: + validator.exit_epoch = delayed_activation_exit_epoch +def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Slash the validator with index ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) # [TO BE REMOVED IN PHASE 2] + exit_validator(state, index) + state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) + + whistleblower_index = get_beacon_proposer_index(state, state.slot) + whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT + state.validator_balances[whistleblower_index] += whistleblower_reward + state.validator_balances[index] -= whistleblower_reward + validator.slashed = True + validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH +def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: + """ + Set the validator with the given ``index`` as withdrawable + ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY +def get_empty_block() -> BeaconBlock: + """ + Get an empty ``BeaconBlock``. + """ + return BeaconBlock( + slot=GENESIS_SLOT, + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + body=BeaconBlockBody( + randao_reveal=EMPTY_SIGNATURE, + eth1_data=Eth1Data( + deposit_root=ZERO_HASH, + block_hash=ZERO_HASH, + ), + proposer_slashings=[], + attester_slashings=[], + attestations=[], + deposits=[], + voluntary_exits=[], + transfers=[], + ), + signature=EMPTY_SIGNATURE, + ) +def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], + genesis_time: int, + genesis_eth1_data: Eth1Data) -> BeaconState: + """ + Get the genesis ``BeaconState``. + """ + state = BeaconState( + # Misc + slot=GENESIS_SLOT, + genesis_time=genesis_time, + fork=Fork( + previous_version=int_to_bytes4(GENESIS_FORK_VERSION), + current_version=int_to_bytes4(GENESIS_FORK_VERSION), + epoch=GENESIS_EPOCH, + ), + + # Validator registry + validator_registry=[], + validator_balances=[], + validator_registry_update_epoch=GENESIS_EPOCH, + + # Randomness and committees + latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)], + previous_shuffling_start_shard=GENESIS_START_SHARD, + current_shuffling_start_shard=GENESIS_START_SHARD, + previous_shuffling_epoch=GENESIS_EPOCH, + current_shuffling_epoch=GENESIS_EPOCH, + previous_shuffling_seed=ZERO_HASH, + current_shuffling_seed=ZERO_HASH, + + # Finality + previous_epoch_attestations=[], + current_epoch_attestations=[], + previous_justified_epoch=GENESIS_EPOCH, + current_justified_epoch=GENESIS_EPOCH, + previous_justified_root=ZERO_HASH, + current_justified_root=ZERO_HASH, + justification_bitfield=0, + finalized_epoch=GENESIS_EPOCH, + finalized_root=ZERO_HASH, + + # Recent state + latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)], + latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], + latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], + latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)], + latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)], + latest_block_header=get_temporary_block_header(get_empty_block()), + historical_roots=[], + + # Ethereum 1.0 chain data + latest_eth1_data=genesis_eth1_data, + eth1_data_votes=[], + deposit_index=0, + ) + + # Process genesis deposits + for deposit in genesis_validator_deposits: + process_deposit(state, deposit) + + # Process genesis activations + for validator_index, _ in enumerate(state.validator_registry): + if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: + activate_validator(state, validator_index, is_genesis=True) + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) + for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH) + + return state +def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: + """ + Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. + """ + if block.slot == slot: + return block + elif block.slot < slot: + return None + else: + return get_ancestor(store, store.get_parent(block), slot) +def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: + """ + Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. + """ + validators = start_state.validator_registry + active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) + attestation_targets = [ + (validator_index, get_latest_attestation_target(store, validator_index)) + for validator_index in active_validator_indices + ] + + def get_vote_count(block: BeaconBlock) -> int: + return sum( + get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT + for validator_index, target in attestation_targets + if get_ancestor(store, target, block.slot) == block + ) + + head = start_block + while 1: + children = get_children(store, head) + if len(children) == 0: + return head + head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) +def cache_state(state: BeaconState) -> None: + previous_slot_state_root = hash_tree_root(state) + + # store the previous slot's post state transition root + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root + + # cache state root in stored latest_block_header if empty + if state.latest_block_header.state_root == ZERO_HASH: + state.latest_block_header.state_root = previous_slot_state_root + + # store latest known block for previous slot + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header) +def get_current_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) +def get_previous_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) +def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + output = set() + for a in attestations: + output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) + return sorted(list(output)) +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: + return get_total_balance(state, get_attesting_indices(state, attestations)) +def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.current_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) + ] +def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + ] +def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.beacon_block_root == get_block_root(state, a.data.slot) + ] +def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: + all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations + valid_attestations = [ + a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] + ] + all_roots = [a.data.crosslink_data_root for a in valid_attestations] + + # handle when no attestations for shard available + if len(all_roots) == 0: + return ZERO_HASH, [] + + def get_attestations_for(root) -> List[PendingAttestation]: + return [a for a in valid_attestations if a.data.crosslink_data_root == root] + + # Winning crosslink root is the root with the most votes for it, ties broken in favor of + # lexicographically higher hash + winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) + + return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) +def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: + return min([ + a for a in state.previous_epoch_attestations if + validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) + ], key=lambda a: a.inclusion_slot) +def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: + return earliest_attestation(state, validator_index).inclusion_slot +def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: + attestation = earliest_attestation(state, validator_index) + return attestation.inclusion_slot - attestation.data.slot +def update_justification_and_finalization(state: BeaconState) -> None: + new_justified_epoch = state.current_justified_epoch + new_finalized_epoch = state.finalized_epoch + + # Rotate the justification bitfield up one epoch to make room for the current epoch + state.justification_bitfield <<= 1 + # If the previous epoch gets justified, fill the second last bit + previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) + if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) - 1 + state.justification_bitfield |= 2 + # If the current epoch gets justified, fill the last bit + current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) + if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) + state.justification_bitfield |= 1 + + # Process finalizations + bitfield = state.justification_bitfield + current_epoch = get_current_epoch(state) + # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source + if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: + new_finalized_epoch = state.previous_justified_epoch + # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source + if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.previous_justified_epoch + # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source + if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.current_justified_epoch + # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source + if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: + new_finalized_epoch = state.current_justified_epoch + + # Update state jusification/finality fields + state.previous_justified_epoch = state.current_justified_epoch + state.previous_justified_root = state.current_justified_root + if new_justified_epoch != state.current_justified_epoch: + state.current_justified_epoch = new_justified_epoch + state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) + if new_finalized_epoch != state.finalized_epoch: + state.finalized_epoch = new_finalized_epoch + state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) +def process_crosslinks(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + previous_epoch = current_epoch - 1 + next_epoch = current_epoch + 1 + for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + if 3 * participating_balance >= 2 * total_balance: + state.latest_crosslinks[shard] = Crosslink( + epoch=slot_to_epoch(slot), + crosslink_data_root=winning_root + ) +def maybe_reset_eth1_period(state: BeaconState) -> None: + if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + for eth1_data_vote in state.eth1_data_votes: + # If a majority of all votes were for a particular eth1_data value, + # then set that as the new canonical value + if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.latest_eth1_data = eth1_data_vote.eth1_data + state.eth1_data_votes = [] +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + if get_previous_total_balance(state) == 0: + return 0 + + adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT + return get_effective_balance(state, index) // adjusted_quotient // 5 +def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: + return ( + get_base_reward(state, index) + + get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 + ) +def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch + if epochs_since_finality <= 4: + return compute_normal_justification_and_finalization_deltas(state) + else: + return compute_inactivity_leak_deltas(state) +def compute_normal_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + # Some helper variables + boundary_attestations = get_previous_epoch_boundary_attestations(state) + boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) + total_balance = get_previous_total_balance(state) + total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) + matching_head_attestations = get_previous_epoch_matching_head_attestations(state) + matching_head_balance = get_attesting_balance(state, matching_head_attestations) + # Process rewards or penalties for all validators + for index in get_active_validator_indices(state.validator_registry, get_previous_epoch(state)): + # Expected FFG source + if index in get_attesting_indices(state, state.previous_epoch_attestations): + deltas[0][index] += get_base_reward(state, index) * total_attesting_balance // total_balance + # Inclusion speed bonus + deltas[0][index] += ( + get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_distance(state, index) + ) + else: + deltas[1][index] += get_base_reward(state, index) + # Expected FFG target + if index in get_attesting_indices(state, boundary_attestations): + deltas[0][index] += get_base_reward(state, index) * boundary_attesting_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + # Expected head + if index in get_attesting_indices(state, matching_head_attestations): + deltas[0][index] += get_base_reward(state, index) * matching_head_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + # Proposer bonus + if index in get_attesting_indices(state, state.previous_epoch_attestations): + proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) + deltas[0][proposer_index] += get_base_reward(state, index) // ATTESTATION_INCLUSION_REWARD_QUOTIENT + return deltas +def compute_inactivity_leak_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + boundary_attestations = get_previous_epoch_boundary_attestations(state) + matching_head_attestations = get_previous_epoch_matching_head_attestations(state) + active_validator_indices = get_active_validator_indices(state.validator_registry, get_previous_epoch(state)) + epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch + for index in active_validator_indices: + if index not in get_attesting_indices(state, state.previous_epoch_attestations): + deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) + else: + # If a validator did attest, apply a small penalty for getting attestations included late + deltas[0][index] += ( + get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_distance(state, index) + ) + deltas[1][index] += get_base_reward(state, index) + if index not in get_attesting_indices(state, boundary_attestations): + deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) + if index not in get_attesting_indices(state, matching_head_attestations): + deltas[1][index] += get_base_reward(state, index) + # Penalize slashed-but-inactive validators as though they were active but offline + for index in range(len(state.validator_registry)): + eligible = ( + index not in active_validator_indices and + state.validator_registry[index].slashed and + get_current_epoch(state) < state.validator_registry[index].withdrawable_epoch + ) + if eligible: + deltas[1][index] += ( + 2 * get_inactivity_penalty(state, index, epochs_since_finality) + + get_base_reward(state, index) + ) + return deltas +def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + # deltas[0] for rewards + # deltas[1] for penalties + deltas = [ + [0 for index in range(len(state.validator_registry))], + [0 for index in range(len(state.validator_registry))] + ] + previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + for slot in range(previous_epoch_start_slot, current_epoch_start_slot): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + for index in crosslink_committee: + if index in participants: + deltas[0][index] += get_base_reward(state, index) * participating_balance // total_balance + else: + deltas[1][index] += get_base_reward(state, index) + return deltas +def apply_rewards(state: BeaconState) -> None: + deltas1 = get_justification_and_finalization_deltas(state) + deltas2 = get_crosslink_deltas(state) + for i in range(len(state.validator_registry)): + state.validator_balances[i] = max( + 0, + state.validator_balances[i] + deltas1[0][i] + deltas2[0][i] - deltas1[1][i] - deltas2[1][i] + ) +def process_ejections(state: BeaconState) -> None: + """ + Iterate through the validator registry + and eject active validators with balance below ``EJECTION_BALANCE``. + """ + for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): + if state.validator_balances[index] < EJECTION_BALANCE: + exit_validator(state, index) +def should_update_validator_registry(state: BeaconState) -> bool: + # Must have finalized a new block + if state.finalized_epoch <= state.validator_registry_update_epoch: + return False + # Must have processed new crosslinks on all shards of the current epoch + shards_to_check = [ + (state.current_shuffling_start_shard + i) % SHARD_COUNT + for i in range(get_current_epoch_committee_count(state)) + ] + for shard in shards_to_check: + if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch: + return False + return True +def update_validator_registry(state: BeaconState) -> None: + """ + Update validator registry. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + # The active validators + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + # The total effective balance of active validators + total_balance = get_total_balance(state, active_validator_indices) + + # The maximum balance churn in Gwei (for deposits and exits separately) + max_balance_churn = max( + MAX_DEPOSIT_AMOUNT, + total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) + ) + + # Activate validators within the allowable balance churn + balance_churn = 0 + for index, validator in enumerate(state.validator_registry): + if validator.activation_epoch == FAR_FUTURE_EPOCH and state.validator_balances[index] >= MAX_DEPOSIT_AMOUNT: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Activate validator + activate_validator(state, index, is_genesis=False) + + # Exit validators within the allowable balance churn + balance_churn = 0 + for index, validator in enumerate(state.validator_registry): + if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Exit validator + exit_validator(state, index) + + state.validator_registry_update_epoch = current_epoch +def update_registry_and_shuffling_data(state: BeaconState) -> None: + # First set previous shuffling data to current shuffling data + state.previous_shuffling_epoch = state.current_shuffling_epoch + state.previous_shuffling_start_shard = state.current_shuffling_start_shard + state.previous_shuffling_seed = state.current_shuffling_seed + current_epoch = get_current_epoch(state) + next_epoch = current_epoch + 1 + # Check if we should update, and if so, update + if should_update_validator_registry(state): + update_validator_registry(state) + # If we update the registry, update the shuffling data and shards as well + state.current_shuffling_epoch = next_epoch + state.current_shuffling_start_shard = ( + state.current_shuffling_start_shard + + get_current_epoch_committee_count(state) % SHARD_COUNT + ) + state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) + else: + # If processing at least one crosslink keeps failing, then reshuffle every power of two, + # but don't update the current_shuffling_start_shard + epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch + if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): + state.current_shuffling_epoch = next_epoch + state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) +def process_slashings(state: BeaconState) -> None: + """ + Process the slashings. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + total_balance = get_total_balance(state, active_validator_indices) + + # Compute `total_penalties` + total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] + total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + total_penalties = total_at_end - total_at_start + + for index, validator in enumerate(state.validator_registry): + if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: + penalty = max( + get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, + get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT + ) + state.validator_balances[index] -= penalty +def process_exit_queue(state: BeaconState) -> None: + """ + Process the exit queue. + Note that this function mutates ``state``. + """ + def eligible(index): + validator = state.validator_registry[index] + # Filter out dequeued validators + if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: + return False + # Dequeue if the minimum amount of time has passed + else: + return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + + eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) + # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index + sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) + for dequeues, index in enumerate(sorted_indices): + if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: + break + prepare_validator_for_withdrawal(state, index) +def finish_epoch_update(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = current_epoch + 1 + # Set active index root + index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH + state.latest_active_index_roots[index_root_position] = hash_tree_root( + get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) + ) + # Set total slashed balances + state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( + state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + ) + # Set randao mix + state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) + # Set historical root accumulator + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch( + block_roots=state.latest_block_roots, + state_roots=state.latest_state_roots, + ) + state.historical_roots.append(hash_tree_root(historical_batch)) + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] +def advance_slot(state: BeaconState) -> None: + state.slot += 1 +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the parent matches + assert block.previous_block_root == hash_tree_root(state.latest_block_header) + # Save current block as the new latest block + state.latest_block_header = get_temporary_block_header(block) + # Verify proposer signature + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(block), + signature=block.signature, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) + ) +def process_randao(state: BeaconState, block: BeaconBlock) -> None: + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + # Verify that the provided randao value is valid + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=hash_tree_root(get_current_epoch(state)), + signature=block.body.randao_reveal, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) + ) + # Mix it in + state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( + xor(get_randao_mix(state, get_current_epoch(state)), + hash(block.body.randao_reveal)) + ) +def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: + for eth1_data_vote in state.eth1_data_votes: + # If someone else has already voted for the same hash, add to its counter + if eth1_data_vote.eth1_data == block.body.eth1_data: + eth1_data_vote.vote_count += 1 + return + # If we're seeing this hash for the first time, make a new counter + state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) +def process_proposer_slashing(state: BeaconState, + proposer_slashing: ProposerSlashing) -> None: + """ + Process ``ProposerSlashing`` transaction. + Note that this function mutates ``state``. + """ + proposer = state.validator_registry[proposer_slashing.proposer_index] + # Verify that the epoch is the same + assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) + # But the headers are different + assert proposer_slashing.header_1 != proposer_slashing.header_2 + # Proposer is not yet slashed + assert proposer.slashed is False + # Signatures are valid + for header in (proposer_slashing.header_1, proposer_slashing.header_2): + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(header), + signature=header.signature, + domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) + ) + slash_validator(state, proposer_slashing.proposer_index) +def process_attester_slashing(state: BeaconState, + attester_slashing: AttesterSlashing) -> None: + """ + Process ``AttesterSlashing`` transaction. + Note that this function mutates ``state``. + """ + attestation1 = attester_slashing.slashable_attestation_1 + attestation2 = attester_slashing.slashable_attestation_2 + # Check that the attestations are conflicting + assert attestation1.data != attestation2.data + assert ( + is_double_vote(attestation1.data, attestation2.data) or + is_surround_vote(attestation1.data, attestation2.data) + ) + assert verify_slashable_attestation(state, attestation1) + assert verify_slashable_attestation(state, attestation2) + slashable_indices = [ + index for index in attestation1.validator_indices + if ( + index in attestation2.validator_indices and + state.validator_registry[index].slashed is False + ) + ] + assert len(slashable_indices) >= 1 + for index in slashable_indices: + slash_validator(state, index) +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + """ + Process ``Attestation`` transaction. + Note that this function mutates ``state``. + """ + # Can't submit attestations that are too far in history (or in prehistory) + assert attestation.data.slot >= GENESIS_SLOT + assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH + # Can't submit attestations too quickly + assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot + # Verify that the justified epoch and root is correct + if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state): + # Case 1: current epoch attestations + assert attestation.data.source_epoch == state.current_justified_epoch + assert attestation.data.source_root == state.current_justified_root + else: + # Case 2: previous epoch attestations + assert attestation.data.source_epoch == state.previous_justified_epoch + assert attestation.data.source_root == state.previous_justified_root + # Check that the crosslink data is valid + acceptable_crosslink_data = { + # Case 1: Latest crosslink matches the one in the state + attestation.data.previous_crosslink, + # Case 2: State has already been updated, state's latest crosslink matches the crosslink + # the attestation is trying to create + Crosslink( + crosslink_data_root=attestation.data.crosslink_data_root, + epoch=slot_to_epoch(attestation.data.slot) + ) + } + assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data + # Attestation must be nonempty! + assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield) + # Custody must be empty (to be removed in phase 1) + assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) + # Get the committee for the specific shard that this attestation is for + crosslink_committee = [ + committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot) + if shard == attestation.data.shard + ][0] + # Custody bitfield must be a subset of the attestation bitfield + for i in range(len(crosslink_committee)): + if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0: + assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0 + # Verify aggregate signature + participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) + custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield) + custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants] + + assert bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)), + ], + signature=attestation.aggregate_signature, + domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), + ) + # Crosslink data root is zero (to be removed in phase 1) + assert attestation.data.crosslink_data_root == ZERO_HASH + # Apply the attestation + pending_attestation = PendingAttestation( + data=attestation.data, + aggregation_bitfield=attestation.aggregation_bitfield, + custody_bitfield=attestation.custody_bitfield, + inclusion_slot=state.slot + ) + if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state): + state.previous_epoch_attestations.append(pending_attestation) +def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: + """ + Process ``VoluntaryExit`` transaction. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[exit.validator_index] + # Verify the validator has not yet exited + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Verify the validator has not initiated an exit + assert validator.initiated_exit is False + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= exit.epoch + # Must have been in the validator set long enough + assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD + # Verify signature + assert bls_verify( + pubkey=validator.pubkey, + message_hash=signed_root(exit), + signature=exit.signature, + domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) + ) + # Run the exit + initiate_validator_exit(state, exit.validator_index) +def process_transfer(state: BeaconState, transfer: Transfer) -> None: + """ + Process ``Transfer`` transaction. + Note that this function mutates ``state``. + """ + # Verify the amount and fee aren't individually too big (for anti-overflow purposes) + assert state.validator_balances[transfer.sender] >= max(transfer.amount, transfer.fee) + # Verify that we have enough ETH to send, and that after the transfer the balance will be either + # exactly zero or at least MIN_DEPOSIT_AMOUNT + assert ( + state.validator_balances[transfer.sender] == transfer.amount + transfer.fee or + state.validator_balances[transfer.sender] >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT + ) + # A transfer is valid in only one slot + assert state.slot == transfer.slot + # Only withdrawn or not-yet-deposited accounts can transfer + assert ( + get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or + state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH + ) + # Verify that the pubkey is valid + assert ( + state.validator_registry[transfer.sender].withdrawal_credentials == + BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] + ) + # Verify that the signature is valid + assert bls_verify( + pubkey=transfer.pubkey, + message_hash=signed_root(transfer), + signature=transfer.signature, + domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) + ) + # Process the transfer + state.validator_balances[transfer.sender] -= transfer.amount + transfer.fee + state.validator_balances[transfer.recipient] += transfer.amount + state.validator_balances[get_beacon_proposer_index(state, state.slot)] += transfer.fee +def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: + assert block.state_root == hash_tree_root(state) + +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + \ No newline at end of file diff --git a/scripts/phase0/state_transition.py b/build/phase0/state_transition.py similarity index 56% rename from scripts/phase0/state_transition.py rename to build/phase0/state_transition.py index f78119cf2..2bd33f6d6 100644 --- a/scripts/phase0/state_transition.py +++ b/build/phase0/state_transition.py @@ -1,3 +1,18 @@ +import build.phase0.spec as spec + + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + +from build.phase0.spec import ( + BeaconState, + BeaconBlock, +) def process_transaction_type(state: BeaconState, @@ -13,72 +28,73 @@ def process_transactions(state: BeaconState, block: BeaconBlock) -> None: process_transaction_type( state, block.body.proposer_slashings, - MAX_PROPOSER_SLASHINGS, - process_proposer_slashing, + spec.MAX_PROPOSER_SLASHINGS, + spec.process_proposer_slashing, ) process_transaction_type( state, block.body.attester_slashings, - MAX_ATTESTER_SLASHINGS, - process_attester_slashing, + spec.MAX_ATTESTER_SLASHINGS, + spec.process_attester_slashing, ) process_transaction_type( state, block.body.attestations, - MAX_ATTESTATIONS, - process_attestation, + spec.MAX_ATTESTATIONS, + spec.process_attestation, ) process_transaction_type( state, block.body.deposits, - MAX_DEPOSITS, - process_deposit, + spec.MAX_DEPOSITS, + spec.process_deposit, ) process_transaction_type( state, block.body.voluntary_exits, - MAX_VOLUNTARY_EXITS, - process_voluntary_exit, + spec.MAX_VOLUNTARY_EXITS, + spec.process_voluntary_exit, ) assert len(block.body.transfers) == len(set(block.body.transfers)) process_transaction_type( state, block.body.transfers, - MAX_TRANSFERS, - process_transfer, + spec.MAX_TRANSFERS, + spec.process_transfer, ) def process_block(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> None: - process_block_header(state, block) - process_randao(state, block) - process_eth1_data(state, block) + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + process_transactions(state, block) if verify_state_root: - verify_block_state_root(state, block) + spec.verify_block_state_root(state, block) def process_epoch_transition(state: BeaconState) -> None: - update_justification_and_finalization(state) - process_crosslinks(state) - maybe_reset_eth1_period(state) - apply_rewards(state) - process_ejections(state) - update_registry_and_shuffling_data(state) - process_slashings(state) - process_exit_queue(state) - finish_epoch_update(state) + spec.update_justification_and_finalization(state) + spec.process_crosslinks(state) + spec.maybe_reset_eth1_period(state) + spec.apply_rewards(state) + spec.process_ejections(state) + spec.update_registry_and_shuffling_data(state) + spec.process_slashings(state) + spec.process_exit_queue(state) + spec.finish_epoch_update(state) def state_transition(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> BeaconState: while state.slot < block.slot: - cache_state(state) - if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: process_epoch_transition(state) - advance_slot(state) + spec.advance_slot(state) if block.slot == state.slot: process_block(state, block) diff --git a/build/utils/__init__.py b/build/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/phase0/bls_stub.py b/build/utils/bls_stub.py similarity index 100% rename from scripts/phase0/bls_stub.py rename to build/utils/bls_stub.py diff --git a/build/utils/hash_function.py b/build/utils/hash_function.py new file mode 100644 index 000000000..da5b4d979 --- /dev/null +++ b/build/utils/hash_function.py @@ -0,0 +1,6 @@ +from hashlib import sha256 +from eth_utils import keccak + + +# def hash(x): return sha256(x).digest() +def hash(x): return keccak(x) diff --git a/build/utils/merkle_minimal.py b/build/utils/merkle_minimal.py new file mode 100644 index 000000000..a811350ce --- /dev/null +++ b/build/utils/merkle_minimal.py @@ -0,0 +1,28 @@ +from .hash_function import hash + + +zerohashes = [b'\x00' * 32] +for i in range(1, 32): + zerohashes.append(hash(zerohashes[i-1] + zerohashes[i-1])) + +# Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree +def calc_merkle_tree_from_leaves(values): + values = list(values) + tree = [values[::]] + for h in range(32): + if len(values) % 2 == 1: + values.append(zerohashes[h]) + # print(values) + values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] + tree.append(values[::]) + return tree + +def get_merkle_root(values): + return calc_merkle_tree_from_leaves(values)[-1][0] + +def get_merkle_proof(tree, item_index): + proof = [] + for i in range(32): + subindex = (item_index//2**i)^1 + proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) + return proof diff --git a/scripts/phase0/minimal_ssz.py b/build/utils/minimal_ssz.py similarity index 99% rename from scripts/phase0/minimal_ssz.py rename to build/utils/minimal_ssz.py index 5caaf8f09..845de18c3 100644 --- a/scripts/phase0/minimal_ssz.py +++ b/build/utils/minimal_ssz.py @@ -1,4 +1,4 @@ -from utils.hash import hash +from .hash_function import hash BYTES_PER_CHUNK = 32 diff --git a/scripts/phase0/monkey_patches.py b/build/utils/monkey_patches.py similarity index 100% rename from scripts/phase0/monkey_patches.py rename to build/utils/monkey_patches.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c4f8ab38c..8b5941b62 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -1,14 +1,18 @@ import sys import function_puller -code_lines = [] -for i in (1, 2, 3, 4, 8, 32, 48, 96): - code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) -code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var -code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") +def build_spec(sourcefile, outfile): + code_lines = [] -code_lines.append(""" + code_lines.append("from build.utils.minimal_ssz import *") + code_lines.append("from build.utils.bls_stub import *") + for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) + code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var + code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") + + code_lines.append(""" from typing import ( Any, Callable, @@ -28,16 +32,48 @@ BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 BLSSignature = NewType('BLSSignature', bytes) # bytes96 Any = None Store = None -""") + """) -code_lines += function_puller.get_lines(sys.argv[1]) + code_lines += function_puller.get_lines(sourcefile) -print(open(sys.argv[2]).read()) -print(open(sys.argv[3]).read()) + code_lines.append(""" +# Monkey patch validator shuffling cache +_get_shuffling = get_shuffling +shuffling_cache = {} +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: -for line in code_lines: - print(line) + param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) -print(open(sys.argv[4]).read()) -print(open(sys.argv[5]).read()) + if param_hash in shuffling_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return shuffling_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _get_shuffling(seed, validators, epoch) + shuffling_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + """) + + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print("Error: spec source and outfile must defined") + build_spec(sys.argv[1], sys.argv[2]) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index d3ebabaa2..7e2800afd 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -1,6 +1,134 @@ import pytest + +from py_ecc import bls + from build.phase0 import spec +from build.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) +from build.phase0.spec import ( + Deposit, + DepositData, + DepositInput, + Eth1Data, + get_genesis_beacon_state, + verify_merkle_branch, + hash, +) -# @pytest.fixture(autouse=True) -# def build_clean(): \ No newline at end of file + +privkeys_list = [i+1 for i in range(1000)] +pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} + + +@pytest.fixture +def privkeys(): + return privkeys_list + + +@pytest.fixture +def pubkeys(): + return pubkeys_list + + +def overwrite_spec_config(config): + for field in config: + setattr(spec, field, config[field]) + if field == "LATEST_RANDAO_MIXES_LENGTH": + spec.BeaconState.fields['latest_randao_mixes'][1] = config[field] + elif field == "SHARD_COUNT": + spec.BeaconState.fields['latest_crosslinks'][1] = config[field] + elif field == "SLOTS_PER_HISTORICAL_ROOT": + spec.BeaconState.fields['latest_block_roots'][1] = config[field] + spec.BeaconState.fields['latest_state_roots'][1] = config[field] + spec.HistoricalBatch.fields['block_roots'][1] = config[field] + spec.HistoricalBatch.fields['state_roots'][1] = config[field] + elif field == "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": + spec.BeaconState.fields['latest_active_index_roots'][1] = config[field] + elif field == "LATEST_SLASHED_EXIT_LENGTH": + spec.BeaconState.fields['latest_slashed_balances'][1] = config[field] + + +@pytest.fixture +def config(): + return { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, + } + + +@pytest.fixture(autouse=True) +def overwrite_config(config): + overwrite_spec_config(config) + + +def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): + deposit_timestamp = 0 + proof_of_possession = b'\x33' * 96 + + deposit_data_list = [] + for i in range(num_validators): + pubkey = pubkeys_list[i] + privkey = pubkey_to_privkey[pubkey] + deposit_data = DepositData( + amount=spec.MAX_DEPOSIT_AMOUNT, + timestamp=deposit_timestamp, + deposit_input=DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=proof_of_possession, + ), + ) + item = hash(deposit_data.serialize()) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=i)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) + deposit_data_list.append(deposit_data) + + genesis_validator_deposits = [] + for i in range(num_validators): + genesis_validator_deposits.append(Deposit( + proof=list(get_merkle_proof(tree, item_index=i)), + index=i, + deposit_data=deposit_data_list[i] + )) + return genesis_validator_deposits, root + + +def create_genesis_state(num_validators, deposit_data_leaves): + initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) + return get_genesis_beacon_state( + initial_deposits, + genesis_time=0, + genesis_eth1_data=Eth1Data( + deposit_root=deposit_root, + block_hash=spec.ZERO_HASH, + ), + ) + +@pytest.fixture +def num_validators(): + return 100 + + +@pytest.fixture +def deposit_data_leaves(): + return list() + + +@pytest.fixture +def state(num_validators, deposit_data_leaves): + return create_genesis_state(num_validators, deposit_data_leaves) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py new file mode 100644 index 000000000..573c3ba21 --- /dev/null +++ b/tests/phase0/test_sanity.py @@ -0,0 +1,632 @@ +import os +import sys +import time + +from copy import deepcopy +from py_ecc import bls +import build.phase0.spec as spec + +from build.utils.minimal_ssz import signed_root +from build.phase0.spec import ( + # SSZ + Attestation, + AttestationData, + AttestationDataAndCustodyBit, + BeaconBlockHeader, + Deposit, + DepositData, + DepositInput, + Eth1Data, + Transfer, + ProposerSlashing, + Validator, + VoluntaryExit, + # functions + int_to_bytes32, + int_to_bytes48, + get_active_validator_indices, + get_attestation_participants, + get_block_root, + get_crosslink_committees_at_slot, + get_current_epoch, + get_domain, + get_empty_block, + get_epoch_start_slot, + get_genesis_beacon_state, + get_state_root, + advance_slot, + slot_to_epoch, + cache_state, + verify_merkle_branch, + hash, +) +from build.phase0.state_transition import ( + state_transition, +) +from build.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) +# from state_test_gen import ( + # generate_from_test, + # dump_json, + # dump_yaml, +# ) + + +def get_empty_root(): + return get_merkle_root((spec.ZERO_HASH,)) + + +def construct_empty_block_for_next_slot(state): + empty_block = get_empty_block() + empty_block.slot = state.slot + 1 + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = previous_block_header.hash_tree_root() + return empty_block + + +def create_deposit_data(state, pubkey, privkey, amount): + deposit_input = DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=b'00'*96, + ) + proof_of_possession = bls.sign( + message_hash=signed_root(deposit_input), + privkey=privkey, + domain=get_domain( + state.fork, + get_current_epoch(state), + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_input.proof_of_possession = proof_of_possession + deposit_data = DepositData( + amount=amount, + timestamp=0, + deposit_input=deposit_input, + ) + return deposit_data + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + block_root = construct_empty_block_for_next_slot(state).previous_block_root + + epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if epoch_start_slot == slot: + epoch_boundary_root = block_root + else: + get_block_root(state, epoch_start_slot) + + if slot < epoch_start_slot: + justified_block_root = state.previous_justified_root + else: + justified_block_root = state.current_justified_root + + return AttestationData( + slot=slot, + shard=shard, + beacon_block_root=block_root, + source_epoch=state.current_justified_epoch, + source_root=justified_block_root, + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink=deepcopy(state.latest_crosslinks[shard]), + ) + + +def test_slot_transition(state): + test_state = deepcopy(state) + cache_state(test_state) + advance_slot(test_state) + assert test_state.slot == state.slot + 1 + assert get_state_root(test_state, state.slot) == state.hash_tree_root() + return test_state + + +def test_empty_block_transition(state): + test_state = deepcopy(state) + + block = construct_empty_block_for_next_slot(test_state) + state_transition(test_state, block) + + assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 + assert get_block_root(test_state, state.slot) == block.previous_block_root + + return state, [block], test_state + + +def test_skipped_slots(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += 3 + + state_transition(test_state, block) + + assert test_state.slot == block.slot + for slot in range(state.slot, test_state.slot): + assert get_block_root(test_state, slot) == block.previous_block_root + + return state, [block], test_state + + +def test_empty_epoch_transition(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += spec.SLOTS_PER_EPOCH + + state_transition(test_state, block) + + assert test_state.slot == block.slot + for slot in range(state.slot, test_state.slot): + assert get_block_root(test_state, slot) == block.previous_block_root + + return state, [block], test_state + + +def test_empty_epoch_transition_not_finalizing(state): + test_state = deepcopy(state) + block = construct_empty_block_for_next_slot(test_state) + block.slot += spec.SLOTS_PER_EPOCH * 5 + + state_transition(test_state, block) + + assert test_state.slot == block.slot + assert test_state.finalized_epoch < get_current_epoch(test_state) - 4 + + return state, [block], test_state + + +def test_proposer_slashing(state, pubkeys, privkeys): + test_state = deepcopy(state) + current_epoch = get_current_epoch(test_state) + validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1] + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + slot = spec.GENESIS_SLOT + header_1 = BeaconBlockHeader( + slot=slot, + previous_block_root=b'\x00'*32, + state_root=b'\x00'*32, + block_body_root=b'\x00'*32, + signature=b'\x00'*96 + ) + header_2 = deepcopy(header_1) + header_2.previous_block_root = b'\x02'*32 + header_2.slot = slot + 1 + + domain = get_domain( + fork=test_state.fork, + epoch=get_current_epoch(test_state), + domain_type=spec.DOMAIN_BEACON_BLOCK, + ) + header_1.signature = bls.sign( + message_hash=signed_root(header_1), + privkey=privkey, + domain=domain, + ) + header_2.signature = bls.sign( + message_hash=signed_root(header_2), + privkey=privkey, + domain=domain, + ) + + proposer_slashing = ProposerSlashing( + proposer_index=validator_index, + header_1=header_1, + header_2=header_2, + ) + + # + # Add to state via block transition + # + block = construct_empty_block_for_next_slot(test_state) + block.body.proposer_slashings.append(proposer_slashing) + state_transition(test_state, block) + + assert not state.validator_registry[validator_index].initiated_exit + assert not state.validator_registry[validator_index].slashed + + slashed_validator = test_state.validator_registry[validator_index] + assert not slashed_validator.initiated_exit + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert test_state.validator_balances[validator_index] < state.validator_balances[validator_index] + + return state, [block], test_state + + +def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): + pre_state = deepcopy(state) + test_deposit_data_leaves = deepcopy(deposit_data_leaves) + + index = len(test_deposit_data_leaves) + pubkey = pubkeys[index] + privkey = privkeys[index] + deposit_data = create_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) + + item = hash(deposit_data.serialize()) + test_deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) + root = get_merkle_root((tuple(test_deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + + deposit = Deposit( + proof=list(proof), + index=index, + deposit_data=deposit_data, + ) + + pre_state.latest_eth1_data.deposit_root = root + post_state = deepcopy(pre_state) + block = construct_empty_block_for_next_slot(post_state) + block.body.deposits.append(deposit) + + state_transition(post_state, block) + assert len(post_state.validator_registry) == len(state.validator_registry) + 1 + assert len(post_state.validator_balances) == len(state.validator_balances) + 1 + assert post_state.validator_registry[index].pubkey == pubkeys[index] + + return pre_state, [block], post_state + + +def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): + pre_state = deepcopy(state) + test_deposit_data_leaves = deepcopy(deposit_data_leaves) + + validator_index = 0 + amount = spec.MAX_DEPOSIT_AMOUNT // 4 + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit_data = create_deposit_data(pre_state, pubkey, privkey, amount) + + merkle_index = len(test_deposit_data_leaves) + item = hash(deposit_data.serialize()) + test_deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) + root = get_merkle_root((tuple(test_deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=merkle_index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) + + deposit = Deposit( + proof=list(proof), + index=merkle_index, + deposit_data=deposit_data, + ) + + pre_state.latest_eth1_data.deposit_root = root + block = construct_empty_block_for_next_slot(pre_state) + block.body.deposits.append(deposit) + + pre_balance = pre_state.validator_balances[validator_index] + post_state = deepcopy(pre_state) + state_transition(post_state, block) + assert len(post_state.validator_registry) == len(pre_state.validator_registry) + assert len(post_state.validator_balances) == len(pre_state.validator_balances) + assert post_state.validator_balances[validator_index] == pre_balance + amount + + return pre_state, [block], post_state + + +def test_attestation(state, pubkeys, privkeys): + test_state = deepcopy(state) + slot = state.slot + shard = state.current_shuffling_start_shard + attestation_data = build_attestation_data(state, slot, shard) + + crosslink_committees = get_crosslink_committees_at_slot(state, slot) + crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0] + + committee_size = len(crosslink_committee) + bitfield_length = (committee_size + 7) // 8 + aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1) + custody_bitfield = b'\x00' * bitfield_length + attestation = Attestation( + aggregation_bitfield=aggregation_bitfield, + data=attestation_data, + custody_bitfield=custody_bitfield, + aggregate_signature=b'\x00'*96, + ) + participants = get_attestation_participants( + test_state, + attestation.data, + attestation.aggregation_bitfield, + ) + assert len(participants) == 1 + + validator_index = participants[0] + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + + message_hash = AttestationDataAndCustodyBit( + data=attestation.data, + custody_bit=0b0, + ).hash_tree_root() + + attestation.aggregation_signature = bls.sign( + message_hash=message_hash, + privkey=privkey, + domain=get_domain( + fork=test_state.fork, + epoch=get_current_epoch(test_state), + domain_type=spec.DOMAIN_ATTESTATION, + ) + ) + + # + # Add to state via block transition + # + attestation_block = construct_empty_block_for_next_slot(test_state) + attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + attestation_block.body.attestations.append(attestation) + state_transition(test_state, attestation_block) + + assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 + + # + # Epoch transition should move to previous_epoch_attestations + # + pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) + + epoch_block = construct_empty_block_for_next_slot(test_state) + epoch_block.slot += spec.SLOTS_PER_EPOCH + state_transition(test_state, epoch_block) + + assert len(test_state.current_epoch_attestations) == 0 + assert test_state.previous_epoch_attestations == pre_current_epoch_attestations + + return state, [attestation_block, epoch_block], test_state + + +def test_voluntary_exit(state, pubkeys, privkeys): + pre_state = deepcopy(state) + validator_index = get_active_validator_indices(pre_state.validator_registry, get_current_epoch(pre_state))[-1] + pubkey = pubkeys[validator_index] + + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # artificially trigger registry update at next epoch transition + pre_state.validator_registry_update_epoch -= 1 + + post_state = deepcopy(pre_state) + + voluntary_exit = VoluntaryExit( + epoch=get_current_epoch(pre_state), + validator_index=validator_index, + signature=b'\x00'*96, + ) + voluntary_exit.signature = bls.sign( + message_hash=signed_root(voluntary_exit), + privkey=privkeys[validator_index], + domain=get_domain( + fork=pre_state.fork, + epoch=get_current_epoch(pre_state), + domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + ) + ) + + # + # Add to state via block transition + # + initiate_exit_block = construct_empty_block_for_next_slot(post_state) + initiate_exit_block.body.voluntary_exits.append(voluntary_exit) + state_transition(post_state, initiate_exit_block) + + assert not pre_state.validator_registry[validator_index].initiated_exit + assert post_state.validator_registry[validator_index].initiated_exit + assert post_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + + # + # Process within epoch transition + # + exit_block = construct_empty_block_for_next_slot(post_state) + exit_block.slot += spec.SLOTS_PER_EPOCH + state_transition(post_state, exit_block) + + assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + + return pre_state, [initiate_exit_block, exit_block], post_state + + +def test_transfer(state, pubkeys, privkeys): + pre_state = deepcopy(state) + current_epoch = get_current_epoch(pre_state) + sender_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1] + recipient_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0] + transfer_pubkey = pubkeys[-1] + transfer_privkey = privkeys[-1] + amount = pre_state.validator_balances[sender_index] + pre_transfer_recipient_balance = pre_state.validator_balances[recipient_index] + transfer = Transfer( + sender=sender_index, + recipient=recipient_index, + amount=amount, + fee=0, + slot=pre_state.slot + 1, + pubkey=transfer_pubkey, + signature=b'\x00'*96, + ) + transfer.signature = bls.sign( + message_hash=signed_root(transfer), + privkey=transfer_privkey, + domain=get_domain( + fork=pre_state.fork, + epoch=get_current_epoch(pre_state), + domain_type=spec.DOMAIN_TRANSFER, + ) + ) + + # ensure withdrawal_credentials reproducable + pre_state.validator_registry[sender_index].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:] + ) + # un-activate so validator can transfer + pre_state.validator_registry[sender_index].activation_epoch = spec.FAR_FUTURE_EPOCH + + post_state = deepcopy(pre_state) + # + # Add to state via block transition + # + block = construct_empty_block_for_next_slot(post_state) + block.body.transfers.append(transfer) + state_transition(post_state, block) + + sender_balance = post_state.validator_balances[sender_index] + recipient_balance = post_state.validator_balances[recipient_index] + assert sender_balance == 0 + assert recipient_balance == pre_transfer_recipient_balance + amount + + return pre_state, [block], post_state + + +def test_ejection(state): + pre_state = deepcopy(state) + + current_epoch = get_current_epoch(pre_state) + validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1] + + assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH + + # set validator balance to below ejection threshold + pre_state.validator_balances[validator_index] = spec.EJECTION_BALANCE - 1 + + post_state = deepcopy(pre_state) + # + # trigger epoch transition + # + block = construct_empty_block_for_next_slot(post_state) + block.slot += spec.SLOTS_PER_EPOCH + state_transition(post_state, block) + + assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + + return pre_state, [block], post_state + + +def test_historical_batch(state): + pre_state = deepcopy(state) + pre_state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (pre_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 + + post_state = deepcopy(pre_state) + + block = construct_empty_block_for_next_slot(post_state) + + state_transition(post_state, block) + + assert post_state.slot == block.slot + assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 + assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1 + + return pre_state, [block], post_state + + +def sanity_tests(num_validators=100, config=None): + print(f"Buidling state with {num_validators} validators...") + if config: + overwrite_spec_config(config) + genesis_state = create_genesis_state(num_validators=num_validators) + print("done!") + print() + + test_cases = [] + + print("Running some sanity check tests...\n") + test_slot_transition(genesis_state) + print("Passed slot transition test\n") + test_cases.append( + generate_from_test(test_empty_block_transition, genesis_state, config=config, fields=['slot']) + ) + print("Passed empty block transition test\n") + test_cases.append( + generate_from_test(test_skipped_slots, genesis_state, config=config, fields=['slot', 'latest_block_roots']) + ) + print("Passed skipped slot test\n") + test_cases.append( + generate_from_test(test_empty_epoch_transition, genesis_state, config=config, fields=['slot', 'latest_block_roots']) + ) + print("Passed empty epoch transition test\n") + test_cases.append( + generate_from_test(test_empty_epoch_transition_not_finalizing, genesis_state, config=config, fields=['slot', 'finalized_epoch']) + ) + print("Passed non-finalizing epoch test\n") + test_cases.append( + generate_from_test(test_proposer_slashing, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed proposer slashing test\n") + test_cases.append( + generate_from_test(test_attestation, genesis_state, config=config, fields=['previous_epoch_attestations', 'current_epoch_attestations']) + ) + print("Passed attestation test\n") + test_cases.append( + generate_from_test(test_deposit_in_block, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed deposit test\n") + test_cases.append( + generate_from_test(test_deposit_top_up, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) + ) + print("Passed deposit top up test\n") + test_cases.append( + generate_from_test(test_voluntary_exit, genesis_state, config=config, fields=['validator_registry']) + ) + print("Passed voluntary exit test\n") + test_cases.append( + generate_from_test(test_transfer, genesis_state, config=config, fields=['validator_balances']) + ) + print("Passed transfer test\n") + test_cases.append( + generate_from_test(test_ejection, genesis_state, config=config, fields=['validator_registry']) + ) + print("Passed ejection test\n") + test_cases.append( + generate_from_test(test_historical_batch, genesis_state, config=config, fields=['historical_roots']) + ) + print("Passed historical batch test\n") + print("done!") + + return test_cases + + +if __name__ == "__main__": + config = { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, + } + + test_cases = sanity_tests(32, config) + # uncomment below to run/generate against the default config + # test_cases = sanity_tests(100) + + test = {} + metadata = {} + metadata['title'] = "Sanity tests" + metadata['summary'] = "Basic sanity checks from phase 0 spec pythonization. All tests are run with `verify_signatures` as set to False." + metadata['test_suite'] = "beacon_state" + metadata['fork'] = "tchaikovsky" + metadata['version'] = "v0.5.0" + test['metadata'] = metadata + test['test_cases'] = test_cases + + if '--output-json' in sys.argv: + os.makedirs('output', exist_ok=True) + with open("output/sanity_check_tests.json", "w+") as outfile: + dump_json(test, outfile) + if '--output-yaml' in sys.argv: + os.makedirs('output', exist_ok=True) + with open("output/sanity_check_tests.yaml", "w+") as outfile: + dump_yaml(test, outfile) From f41caa713b167d986acfdbeb61fb9f1bb8d1ce81 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 13:06:45 -0600 Subject: [PATCH 18/32] add circleci config --- .circleci/config.yml | 46 ++++++++++++++++++++++++++++++++++++++++++++ .gitignore | 2 ++ Makefile | 4 +--- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..98d2367c9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,46 @@ +# Python CircleCI 2.0 configuration file +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.6 + working_directory: ~/repo + + steps: + - checkout + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + - run: + name: build phase0 spec + command: make build/phase0 + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + # run tests! + # this example uses Django's built-in test-runner + # other common Python testing frameworks include pytest and nose + # https://pytest.org + # https://nose.readthedocs.io + - run: + name: run tests + command: | + . venv/bin/activate + pytest tests + + - store_artifacts: + path: test-reports + destination: test-reports \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5e19cd2a5..f9a966cea 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /__pycache__ /venv /.pytest_cache + +build/phase0/spec.py diff --git a/Makefile b/Makefile index 745f8f901..8be65fd44 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,8 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts BUILD_DIR = ./build -.PHONY: clean all +.PHONY: clean all $(BUILD_DIR)/phase0 $(BUILD_DIR)/phase0: - mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py - touch $(BUILD_DIR)/__init__.py $(BUILD_DIR)/phase0/__init__.py From 6715a0d4cc0c7b4842dac7cd88264de0f46ccde6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 14:14:26 -0600 Subject: [PATCH 19/32] reconfigure build a bit --- .circleci/config.yml | 7 +- .gitignore | 2 +- Makefile | 6 ++ build/phase0/spec.py | 4 +- build/phase0/state_transition.py | 4 +- build/utils/monkey_patches.py | 29 ----- scripts/phase0/build_spec.py | 4 +- tests/phase0/conftest.py | 2 +- tests/phase0/test_sanity.py | 4 +- {build => utils}/__init__.py | 0 {build/utils => utils/phase0}/__init__.py | 0 {build/utils => utils/phase0}/bls_stub.py | 0 .../utils => utils/phase0}/hash_function.py | 0 .../utils => utils/phase0}/merkle_minimal.py | 0 {build/utils => utils/phase0}/minimal_ssz.py | 0 utils/phase0/state_transition.py | 100 ++++++++++++++++++ 16 files changed, 117 insertions(+), 45 deletions(-) delete mode 100644 build/utils/monkey_patches.py rename {build => utils}/__init__.py (100%) rename {build/utils => utils/phase0}/__init__.py (100%) rename {build/utils => utils/phase0}/bls_stub.py (100%) rename {build/utils => utils/phase0}/hash_function.py (100%) rename {build/utils => utils/phase0}/merkle_minimal.py (100%) rename {build/utils => utils/phase0}/minimal_ssz.py (100%) create mode 100644 utils/phase0/state_transition.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 98d2367c9..02871530e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,11 +30,6 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - # run tests! - # this example uses Django's built-in test-runner - # other common Python testing frameworks include pytest and nose - # https://pytest.org - # https://nose.readthedocs.io - run: name: run tests command: | @@ -43,4 +38,4 @@ jobs: - store_artifacts: path: test-reports - destination: test-reports \ No newline at end of file + destination: test-reports diff --git a/.gitignore b/.gitignore index f9a966cea..dfb38d170 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ /venv /.pytest_cache -build/phase0/spec.py +build/ diff --git a/Makefile b/Makefile index 8be65fd44..593ea8bf4 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts BUILD_DIR = ./build +UTILS_DIR = ./utils .PHONY: clean all $(BUILD_DIR)/phase0 $(BUILD_DIR)/phase0: + mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py + mkdir -p $@/utils + cp $(UTILS_DIR)/phase0/* $@/utils + cp $(UTILS_DIR)/phase0/state_transition.py $@ + touch $@/__init__.py $@/utils/__init__.py diff --git a/build/phase0/spec.py b/build/phase0/spec.py index 8c05b1208..3fd82c33f 100644 --- a/build/phase0/spec.py +++ b/build/phase0/spec.py @@ -1,5 +1,5 @@ -from build.utils.minimal_ssz import * -from build.utils.bls_stub import * +from build.phase0.utils.minimal_ssz import * +from build.phase0.utils.bls_stub import * def int_to_bytes1(x): return x.to_bytes(1, 'little') def int_to_bytes2(x): return x.to_bytes(2, 'little') def int_to_bytes3(x): return x.to_bytes(3, 'little') diff --git a/build/phase0/state_transition.py b/build/phase0/state_transition.py index 2bd33f6d6..170f647ab 100644 --- a/build/phase0/state_transition.py +++ b/build/phase0/state_transition.py @@ -1,4 +1,4 @@ -import build.phase0.spec as spec +from . import spec from typing import ( @@ -9,7 +9,7 @@ from typing import ( Tuple, ) -from build.phase0.spec import ( +from .spec import ( BeaconState, BeaconBlock, ) diff --git a/build/utils/monkey_patches.py b/build/utils/monkey_patches.py deleted file mode 100644 index 8a35b8f27..000000000 --- a/build/utils/monkey_patches.py +++ /dev/null @@ -1,29 +0,0 @@ -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) - - if param_hash in shuffling_cache: - # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] - else: - # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 8b5941b62..eb4f580bd 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -5,8 +5,8 @@ import function_puller def build_spec(sourcefile, outfile): code_lines = [] - code_lines.append("from build.utils.minimal_ssz import *") - code_lines.append("from build.utils.bls_stub import *") + code_lines.append("from build.phase0.utils.minimal_ssz import *") + code_lines.append("from build.phase0.utils.bls_stub import *") for i in (1, 2, 3, 4, 8, 32, 48, 96): code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 7e2800afd..7d372f164 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -4,7 +4,7 @@ from py_ecc import bls from build.phase0 import spec -from build.utils.merkle_minimal import ( +from build.phase0.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 573c3ba21..0e04df5dd 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -6,7 +6,7 @@ from copy import deepcopy from py_ecc import bls import build.phase0.spec as spec -from build.utils.minimal_ssz import signed_root +from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( # SSZ Attestation, @@ -43,7 +43,7 @@ from build.phase0.spec import ( from build.phase0.state_transition import ( state_transition, ) -from build.utils.merkle_minimal import ( +from build.phase0.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/build/__init__.py b/utils/__init__.py similarity index 100% rename from build/__init__.py rename to utils/__init__.py diff --git a/build/utils/__init__.py b/utils/phase0/__init__.py similarity index 100% rename from build/utils/__init__.py rename to utils/phase0/__init__.py diff --git a/build/utils/bls_stub.py b/utils/phase0/bls_stub.py similarity index 100% rename from build/utils/bls_stub.py rename to utils/phase0/bls_stub.py diff --git a/build/utils/hash_function.py b/utils/phase0/hash_function.py similarity index 100% rename from build/utils/hash_function.py rename to utils/phase0/hash_function.py diff --git a/build/utils/merkle_minimal.py b/utils/phase0/merkle_minimal.py similarity index 100% rename from build/utils/merkle_minimal.py rename to utils/phase0/merkle_minimal.py diff --git a/build/utils/minimal_ssz.py b/utils/phase0/minimal_ssz.py similarity index 100% rename from build/utils/minimal_ssz.py rename to utils/phase0/minimal_ssz.py diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py new file mode 100644 index 000000000..170f647ab --- /dev/null +++ b/utils/phase0/state_transition.py @@ -0,0 +1,100 @@ +from . import spec + + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + +from .spec import ( + BeaconState, + BeaconBlock, +) + + +def process_transaction_type(state: BeaconState, + transactions: List[Any], + max_transactions: int, + tx_fn: Callable[[BeaconState, Any], None]) -> None: + assert len(transactions) <= max_transactions + for transaction in transactions: + tx_fn(state, transaction) + + +def process_transactions(state: BeaconState, block: BeaconBlock) -> None: + process_transaction_type( + state, + block.body.proposer_slashings, + spec.MAX_PROPOSER_SLASHINGS, + spec.process_proposer_slashing, + ) + process_transaction_type( + state, + block.body.attester_slashings, + spec.MAX_ATTESTER_SLASHINGS, + spec.process_attester_slashing, + ) + process_transaction_type( + state, + block.body.attestations, + spec.MAX_ATTESTATIONS, + spec.process_attestation, + ) + process_transaction_type( + state, + block.body.deposits, + spec.MAX_DEPOSITS, + spec.process_deposit, + ) + process_transaction_type( + state, + block.body.voluntary_exits, + spec.MAX_VOLUNTARY_EXITS, + spec.process_voluntary_exit, + ) + assert len(block.body.transfers) == len(set(block.body.transfers)) + process_transaction_type( + state, + block.body.transfers, + spec.MAX_TRANSFERS, + spec.process_transfer, + ) + + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + + process_transactions(state, block) + if verify_state_root: + spec.verify_block_state_root(state, block) + + +def process_epoch_transition(state: BeaconState) -> None: + spec.update_justification_and_finalization(state) + spec.process_crosslinks(state) + spec.maybe_reset_eth1_period(state) + spec.apply_rewards(state) + spec.process_ejections(state) + spec.update_registry_and_shuffling_data(state) + spec.process_slashings(state) + spec.process_exit_queue(state) + spec.finish_epoch_update(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + while state.slot < block.slot: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + spec.advance_slot(state) + if block.slot == state.slot: + process_block(state, block) From d9ac06e8edb814ba12128ae1fe0164496320a77c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 14:16:04 -0600 Subject: [PATCH 20/32] remove build from git --- build/phase0/__init__.py | 0 build/phase0/spec.py | 1620 ------------------------------ build/phase0/state_transition.py | 100 -- 3 files changed, 1720 deletions(-) delete mode 100644 build/phase0/__init__.py delete mode 100644 build/phase0/spec.py delete mode 100644 build/phase0/state_transition.py diff --git a/build/phase0/__init__.py b/build/phase0/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/phase0/spec.py b/build/phase0/spec.py deleted file mode 100644 index 3fd82c33f..000000000 --- a/build/phase0/spec.py +++ /dev/null @@ -1,1620 +0,0 @@ -from build.phase0.utils.minimal_ssz import * -from build.phase0.utils.bls_stub import * -def int_to_bytes1(x): return x.to_bytes(1, 'little') -def int_to_bytes2(x): return x.to_bytes(2, 'little') -def int_to_bytes3(x): return x.to_bytes(3, 'little') -def int_to_bytes4(x): return x.to_bytes(4, 'little') -def int_to_bytes8(x): return x.to_bytes(8, 'little') -def int_to_bytes32(x): return x.to_bytes(32, 'little') -def int_to_bytes48(x): return x.to_bytes(48, 'little') -def int_to_bytes96(x): return x.to_bytes(96, 'little') -SLOTS_PER_EPOCH = 64 -def slot_to_epoch(x): return x // SLOTS_PER_EPOCH - -from typing import ( - Any, - Callable, - List, - NewType, - Tuple, -) - - -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Any = None -Store = None - -SHARD_COUNT = 2**10 -TARGET_COMMITTEE_SIZE = 2**7 -MAX_BALANCE_CHURN_QUOTIENT = 2**5 -MAX_INDICES_PER_SLASHABLE_VOTE = 2**12 -MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 -SHUFFLE_ROUND_COUNT = 90 -DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 -DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 -MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 -MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 -FORK_CHOICE_BALANCE_INCREMENT = 2**0 * 10**9 -EJECTION_BALANCE = 2**4 * 10**9 -GENESIS_FORK_VERSION = 0 -GENESIS_SLOT = 2**32 -GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) -GENESIS_START_SHARD = 0 -FAR_FUTURE_EPOCH = 2**64 - 1 -ZERO_HASH = int_to_bytes32(0) -EMPTY_SIGNATURE = int_to_bytes96(0) -BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) -SECONDS_PER_SLOT = 6 -MIN_ATTESTATION_INCLUSION_DELAY = 2**2 -SLOTS_PER_EPOCH = 2**6 -MIN_SEED_LOOKAHEAD = 2**0 -ACTIVATION_EXIT_DELAY = 2**2 -EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 -SLOTS_PER_HISTORICAL_ROOT = 2**13 -MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 -PERSISTENT_COMMITTEE_PERIOD = 2**11 -LATEST_RANDAO_MIXES_LENGTH = 2**13 -LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 -LATEST_SLASHED_EXIT_LENGTH = 2**13 -BASE_REWARD_QUOTIENT = 2**5 -WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 -ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 -INACTIVITY_PENALTY_QUOTIENT = 2**24 -MIN_PENALTY_QUOTIENT = 2**5 -MAX_PROPOSER_SLASHINGS = 2**4 -MAX_ATTESTER_SLASHINGS = 2**0 -MAX_ATTESTATIONS = 2**7 -MAX_DEPOSITS = 2**4 -MAX_VOLUNTARY_EXITS = 2**4 -MAX_TRANSFERS = 2**4 -DOMAIN_BEACON_BLOCK = 0 -DOMAIN_RANDAO = 1 -DOMAIN_ATTESTATION = 2 -DOMAIN_DEPOSIT = 3 -DOMAIN_VOLUNTARY_EXIT = 4 -DOMAIN_TRANSFER = 5 -Fork = SSZType({ - # Previous fork version - 'previous_version': 'bytes4', - # Current fork version - 'current_version': 'bytes4', - # Fork epoch number - 'epoch': 'uint64', -}) -Crosslink = SSZType({ - # Epoch number - 'epoch': 'uint64', - # Shard data since the previous crosslink - 'crosslink_data_root': 'bytes32', -}) -Eth1Data = SSZType({ - # Root of the deposit tree - 'deposit_root': 'bytes32', - # Block hash - 'block_hash': 'bytes32', -}) -Eth1DataVote = SSZType({ - # Data being voted for - 'eth1_data': Eth1Data, - # Vote count - 'vote_count': 'uint64', -}) -AttestationData = SSZType({ - # LMD GHOST vote - 'slot': 'uint64', - 'beacon_block_root': 'bytes32', - - # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_root': 'bytes32', - - # Crosslink vote - 'shard': 'uint64', - 'previous_crosslink': Crosslink, - 'crosslink_data_root': 'bytes32', -}) -AttestationDataAndCustodyBit = SSZType({ - # Attestation data - 'data': AttestationData, - # Custody bit - 'custody_bit': 'bool', -}) -SlashableAttestation = SSZType({ - # Validator indices - 'validator_indices': ['uint64'], - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Aggregate signature - 'aggregate_signature': 'bytes96', -}) -DepositInput = SSZType({ - # BLS pubkey - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # A BLS signature of this `DepositInput` - 'proof_of_possession': 'bytes96', -}) -DepositData = SSZType({ - # Amount in Gwei - 'amount': 'uint64', - # Timestamp from deposit contract - 'timestamp': 'uint64', - # Deposit input - 'deposit_input': DepositInput, -}) -BeaconBlockHeader = SSZType({ - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'block_body_root': 'bytes32', - 'signature': 'bytes96', -}) -Validator = SSZType({ - # BLS public key - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # Epoch when validator activated - 'activation_epoch': 'uint64', - # Epoch when validator exited - 'exit_epoch': 'uint64', - # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', - # Did the validator initiate an exit - 'initiated_exit': 'bool', - # Was the validator slashed - 'slashed': 'bool', -}) -PendingAttestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Inclusion slot - 'inclusion_slot': 'uint64', -}) -HistoricalBatch = SSZType({ - # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], -}) -ProposerSlashing = SSZType({ - # Proposer index - 'proposer_index': 'uint64', - # First block header - 'header_1': BeaconBlockHeader, - # Second block header - 'header_2': BeaconBlockHeader, -}) -AttesterSlashing = SSZType({ - # First slashable attestation - 'slashable_attestation_1': SlashableAttestation, - # Second slashable attestation - 'slashable_attestation_2': SlashableAttestation, -}) -Attestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # BLS aggregate signature - 'aggregate_signature': 'bytes96', -}) -Deposit = SSZType({ - # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], - # Index in the deposit tree - 'index': 'uint64', - # Data - 'deposit_data': DepositData, -}) -VoluntaryExit = SSZType({ - # Minimum epoch for processing exit - 'epoch': 'uint64', - # Index of the exiting validator - 'validator_index': 'uint64', - # Validator signature - 'signature': 'bytes96', -}) -Transfer = SSZType({ - # Sender index - 'sender': 'uint64', - # Recipient index - 'recipient': 'uint64', - # Amount in Gwei - 'amount': 'uint64', - # Fee in Gwei for block proposer - 'fee': 'uint64', - # Inclusion slot - 'slot': 'uint64', - # Sender withdrawal pubkey - 'pubkey': 'bytes48', - # Sender signature - 'signature': 'bytes96', -}) -BeaconBlockBody = SSZType({ - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], -}) -BeaconBlock = SSZType({ - # Header - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', -}) -BeaconState = SSZType({ - # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - - # Validator registry - 'validator_registry': [Validator], - 'validator_balances': ['uint64'], - 'validator_registry_update_epoch': 'uint64', - - # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'previous_shuffling_start_shard': 'uint64', - 'current_shuffling_start_shard': 'uint64', - 'previous_shuffling_epoch': 'uint64', - 'current_shuffling_epoch': 'uint64', - 'previous_shuffling_seed': 'bytes32', - 'current_shuffling_seed': 'bytes32', - - # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - - # Recent state - 'latest_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period - 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily - 'historical_roots': ['bytes32'], - - # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1DataVote], - 'deposit_index': 'uint64' -}) -def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) -def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: - """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. - """ - return BeaconBlockHeader( - slot=block.slot, - previous_block_root=block.previous_block_root, - state_root=ZERO_HASH, - block_body_root=hash_tree_root(block.body), - signature=block.signature, - ) -def slot_to_epoch(slot: Slot) -> Epoch: - """ - Return the epoch number of the given ``slot``. - """ - return slot // SLOTS_PER_EPOCH -def get_previous_epoch(state: BeaconState) -> Epoch: - """` - Return the previous epoch of the given ``state``. - """ - return get_current_epoch(state) - 1 -def get_current_epoch(state: BeaconState) -> Epoch: - """ - Return the current epoch of the given ``state``. - """ - return slot_to_epoch(state.slot) -def get_epoch_start_slot(epoch: Epoch) -> Slot: - """ - Return the starting slot of the given ``epoch``. - """ - return epoch * SLOTS_PER_EPOCH -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch -def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: - """ - Get indices of active validators from ``validators``. - """ - return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] -def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: - """ - Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. - - Utilizes 'swap or not' shuffling found in - https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf - See the 'generalized domain' algorithm on page 3. - """ - assert index < list_size - assert list_size <= 2**40 - - for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size - flip = (pivot - index) % list_size - position = max(index, flip) - source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return index -def split(values: List[Any], split_count: int) -> List[List[Any]]: - """ - Splits ``values`` into ``split_count`` pieces. - """ - list_length = len(values) - return [ - values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] - for i in range(split_count) - ] -def get_epoch_committee_count(active_validator_count: int) -> int: - """ - Return the number of committees in one epoch. - """ - return max( - 1, - min( - SHARD_COUNT // SLOTS_PER_EPOCH, - active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - ) - ) * SLOTS_PER_EPOCH -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - """ - Shuffle active validators and split into crosslink committees. - Return a list of committees (each a list of validator indices). - """ - # Shuffle active validator indices - active_validator_indices = get_active_validator_indices(validators, epoch) - length = len(active_validator_indices) - shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] - - # Split the shuffled active validator indices - return split(shuffled_indices, get_epoch_committee_count(length)) -def get_previous_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the previous epoch of the given ``state``. - """ - previous_active_validators = get_active_validator_indices( - state.validator_registry, - state.previous_shuffling_epoch, - ) - return get_epoch_committee_count(len(previous_active_validators)) -def get_current_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the current epoch of the given ``state``. - """ - current_active_validators = get_active_validator_indices( - state.validator_registry, - state.current_shuffling_epoch, - ) - return get_epoch_committee_count(len(current_active_validators)) -def get_next_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the next epoch of the given ``state``. - """ - next_active_validators = get_active_validator_indices( - state.validator_registry, - get_current_epoch(state) + 1, - ) - return get_epoch_committee_count(len(next_active_validators)) -def get_crosslink_committees_at_slot(state: BeaconState, - slot: Slot, - registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], Shard]]: - """ - Return the list of ``(committee, shard)`` tuples for the ``slot``. - - Note: There are two possible shufflings for crosslink committees for a - ``slot`` in the next epoch -- with and without a `registry_change` - """ - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 - - assert previous_epoch <= epoch <= next_epoch - - if epoch == current_epoch: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch - shuffling_start_shard = state.current_shuffling_start_shard - elif epoch == previous_epoch: - committees_per_epoch = get_previous_epoch_committee_count(state) - seed = state.previous_shuffling_seed - shuffling_epoch = state.previous_shuffling_epoch - shuffling_start_shard = state.previous_shuffling_start_shard - elif epoch == next_epoch: - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if registry_change: - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - current_committees_per_epoch = get_current_epoch_committee_count(state) - shuffling_start_shard = (state.current_shuffling_start_shard + current_committees_per_epoch) % SHARD_COUNT - elif epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - committees_per_epoch = get_next_epoch_committee_count(state) - seed = generate_seed(state, next_epoch) - shuffling_epoch = next_epoch - shuffling_start_shard = state.current_shuffling_start_shard - else: - committees_per_epoch = get_current_epoch_committee_count(state) - seed = state.current_shuffling_seed - shuffling_epoch = state.current_shuffling_epoch - shuffling_start_shard = state.current_shuffling_start_shard - - shuffling = get_shuffling( - seed, - state.validator_registry, - shuffling_epoch, - ) - offset = slot % SLOTS_PER_EPOCH - committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT - - return [ - ( - shuffling[committees_per_slot * offset + i], - (slot_start_shard + i) % SHARD_COUNT, - ) - for i in range(committees_per_slot) - ] -def get_block_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the block root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] -def get_state_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] -def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the randao mix at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) - return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] -def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the index root at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY - return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] -def generate_seed(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Generate a seed for the given ``epoch``. - """ - return hash( - get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + - get_active_index_root(state, epoch) + - int_to_bytes32(epoch) - ) -def get_beacon_proposer_index(state: BeaconState, - slot: Slot, - registry_change: bool=False) -> ValidatorIndex: - """ - Return the beacon proposer index for the ``slot``. - """ - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 - - assert previous_epoch <= epoch <= next_epoch - - first_committee, _ = get_crosslink_committees_at_slot(state, slot, registry_change)[0] - return first_committee[epoch % len(first_committee)] -def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: - """ - Verify that the given ``leaf`` is on the merkle branch ``proof`` - starting with the given ``root``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(proof[i] + value) - else: - value = hash(value + proof[i]) - return value == root -def get_attestation_participants(state: BeaconState, - attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: - """ - Return the participant indices at for the ``attestation_data`` and ``bitfield``. - """ - # Find the committee in the list with the desired shard - crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) - - assert attestation_data.shard in [shard for _, shard in crosslink_committees] - crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] - - assert verify_bitfield(bitfield, len(crosslink_committee)) - - # Find the participating attesters in the committee - participants = [] - for i, validator_index in enumerate(crosslink_committee): - aggregation_bit = get_bitfield_bit(bitfield, i) - if aggregation_bit == 0b1: - participants.append(validator_index) - return participants -def is_power_of_two(value: int) -> bool: - """ - Check if ``value`` is a power of two integer. - """ - return (value > 0) and (value & (value - 1) == 0) -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') -def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: - """ - Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - """ - return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: - """ - Return the combined effective balance of an array of ``validators``. - """ - return sum([get_effective_balance(state, i) for i in validators]) -def get_fork_version(fork: Fork, - epoch: Epoch) -> bytes: - """ - Return the fork version of the given ``epoch``. - """ - if epoch < fork.epoch: - return fork.previous_version - else: - return fork.current_version -def get_domain(fork: Fork, - epoch: Epoch, - domain_type: int) -> int: - """ - Get the domain number that represents the fork meta and signature domain. - """ - return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: - """ - Verify validity of ``slashable_attestation`` fields. - """ - if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] - return False - - if len(slashable_attestation.validator_indices) == 0: - return False - - for i in range(len(slashable_attestation.validator_indices) - 1): - if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: - return False - - if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): - return False - - if len(slashable_attestation.validator_indices) > MAX_INDICES_PER_SLASHABLE_VOTE: - return False - - custody_bit_0_indices = [] - custody_bit_1_indices = [] - for i, validator_index in enumerate(slashable_attestation.validator_indices): - if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: - custody_bit_0_indices.append(validator_index) - else: - custody_bit_1_indices.append(validator_index) - - return bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), - ], - signature=slashable_attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), - ) -def is_double_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. - """ - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - return target_epoch_1 == target_epoch_2 -def is_surround_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` surrounds ``attestation_data_2``. - """ - source_epoch_1 = attestation_data_1.source_epoch - source_epoch_2 = attestation_data_2.source_epoch - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - - return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 -def integer_squareroot(n: int) -> int: - """ - The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. - """ - assert n >= 0 - x = n - y = (x + 1) // 2 - while y < x: - x = y - y = (x + n // x) // 2 - return x -def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - """ - return epoch + 1 + ACTIVATION_EXIT_DELAY -def process_deposit(state: BeaconState, deposit: Deposit) -> None: - """ - Process a deposit from Ethereum 1.0. - Note that this function mutates ``state``. - """ - deposit_input = deposit.deposit_data.deposit_input - - # Should equal 8 bytes for deposit_data.amount + - # 8 bytes for deposit_data.timestamp + - # 176 bytes for deposit_data.deposit_input - # It should match the deposit_data in the eth1.0 deposit contract - serialized_deposit_data = serialize(deposit.deposit_data) - # Deposits must be processed in order - assert deposit.index == state.deposit_index - - # Verify the Merkle branch - merkle_branch_is_valid = verify_merkle_branch( - leaf=hash(serialized_deposit_data), - proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit.index, - root=state.latest_eth1_data.deposit_root, - ) - assert merkle_branch_is_valid - - # Increment the next deposit index we are expecting. Note that this - # needs to be done here because while the deposit contract will never - # create an invalid Merkle branch, it may admit an invalid deposit - # object, and we need to be able to skip over it - state.deposit_index += 1 - - validator_pubkeys = [v.pubkey for v in state.validator_registry] - pubkey = deposit_input.pubkey - amount = deposit.deposit_data.amount - withdrawal_credentials = deposit_input.withdrawal_credentials - - if pubkey not in validator_pubkeys: - # Verify the proof of possession - proof_is_valid = bls_verify( - pubkey=deposit_input.pubkey, - message_hash=signed_root(deposit_input), - signature=deposit_input.proof_of_possession, - domain=get_domain( - state.fork, - get_current_epoch(state), - DOMAIN_DEPOSIT, - ) - ) - if not proof_is_valid: - return - - # Add new validator - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - initiated_exit=False, - slashed=False, - ) - - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. - state.validator_registry.append(validator) - state.validator_balances.append(amount) - else: - # Increase balance by deposit amount - state.validator_balances[validator_pubkeys.index(pubkey)] += amount -def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: - """ - Activate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - - validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) -def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: - """ - Initiate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.initiated_exit = True -def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Exit the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - delayed_activation_exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) - - # The following updates only occur if not previous exited - if validator.exit_epoch <= delayed_activation_exit_epoch: - return - else: - validator.exit_epoch = delayed_activation_exit_epoch -def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Slash the validator with index ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) # [TO BE REMOVED IN PHASE 2] - exit_validator(state, index) - state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) - - whistleblower_index = get_beacon_proposer_index(state, state.slot) - whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT - state.validator_balances[whistleblower_index] += whistleblower_reward - state.validator_balances[index] -= whistleblower_reward - validator.slashed = True - validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH -def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: - """ - Set the validator with the given ``index`` as withdrawable - ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY -def get_empty_block() -> BeaconBlock: - """ - Get an empty ``BeaconBlock``. - """ - return BeaconBlock( - slot=GENESIS_SLOT, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - body=BeaconBlockBody( - randao_reveal=EMPTY_SIGNATURE, - eth1_data=Eth1Data( - deposit_root=ZERO_HASH, - block_hash=ZERO_HASH, - ), - proposer_slashings=[], - attester_slashings=[], - attestations=[], - deposits=[], - voluntary_exits=[], - transfers=[], - ), - signature=EMPTY_SIGNATURE, - ) -def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], - genesis_time: int, - genesis_eth1_data: Eth1Data) -> BeaconState: - """ - Get the genesis ``BeaconState``. - """ - state = BeaconState( - # Misc - slot=GENESIS_SLOT, - genesis_time=genesis_time, - fork=Fork( - previous_version=int_to_bytes4(GENESIS_FORK_VERSION), - current_version=int_to_bytes4(GENESIS_FORK_VERSION), - epoch=GENESIS_EPOCH, - ), - - # Validator registry - validator_registry=[], - validator_balances=[], - validator_registry_update_epoch=GENESIS_EPOCH, - - # Randomness and committees - latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)], - previous_shuffling_start_shard=GENESIS_START_SHARD, - current_shuffling_start_shard=GENESIS_START_SHARD, - previous_shuffling_epoch=GENESIS_EPOCH, - current_shuffling_epoch=GENESIS_EPOCH, - previous_shuffling_seed=ZERO_HASH, - current_shuffling_seed=ZERO_HASH, - - # Finality - previous_epoch_attestations=[], - current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH, - current_justified_epoch=GENESIS_EPOCH, - previous_justified_root=ZERO_HASH, - current_justified_root=ZERO_HASH, - justification_bitfield=0, - finalized_epoch=GENESIS_EPOCH, - finalized_root=ZERO_HASH, - - # Recent state - latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)], - latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)], - latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)], - latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)], - latest_block_header=get_temporary_block_header(get_empty_block()), - historical_roots=[], - - # Ethereum 1.0 chain data - latest_eth1_data=genesis_eth1_data, - eth1_data_votes=[], - deposit_index=0, - ) - - # Process genesis deposits - for deposit in genesis_validator_deposits: - process_deposit(state, deposit) - - # Process genesis activations - for validator_index, _ in enumerate(state.validator_registry): - if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, validator_index, is_genesis=True) - - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) - for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root - state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH) - - return state -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validator_registry - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [ - (validator_index, get_latest_attestation_target(store, validator_index)) - for validator_index in active_validator_indices - ] - - def get_vote_count(block: BeaconBlock) -> int: - return sum( - get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) - - head = start_block - while 1: - children = get_children(store, head) - if len(children) == 0: - return head - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) -def cache_state(state: BeaconState) -> None: - previous_slot_state_root = hash_tree_root(state) - - # store the previous slot's post state transition root - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root - - # cache state root in stored latest_block_header if empty - if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_slot_state_root - - # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header) -def get_current_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) -def get_previous_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) -def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: - output = set() - for a in attestations: - output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) - return sorted(list(output)) -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: - return get_total_balance(state, get_attesting_indices(state, attestations)) -def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.current_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) - ] -def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) - ] -def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.beacon_block_root == get_block_root(state, a.data.slot) - ] -def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: - all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations - valid_attestations = [ - a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] - ] - all_roots = [a.data.crosslink_data_root for a in valid_attestations] - - # handle when no attestations for shard available - if len(all_roots) == 0: - return ZERO_HASH, [] - - def get_attestations_for(root) -> List[PendingAttestation]: - return [a for a in valid_attestations if a.data.crosslink_data_root == root] - - # Winning crosslink root is the root with the most votes for it, ties broken in favor of - # lexicographically higher hash - winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) - - return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) -def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: - return min([ - a for a in state.previous_epoch_attestations if - validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) - ], key=lambda a: a.inclusion_slot) -def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: - return earliest_attestation(state, validator_index).inclusion_slot -def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: - attestation = earliest_attestation(state, validator_index) - return attestation.inclusion_slot - attestation.data.slot -def update_justification_and_finalization(state: BeaconState) -> None: - new_justified_epoch = state.current_justified_epoch - new_finalized_epoch = state.finalized_epoch - - # Rotate the justification bitfield up one epoch to make room for the current epoch - state.justification_bitfield <<= 1 - # If the previous epoch gets justified, fill the second last bit - previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) - if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - 1 - state.justification_bitfield |= 2 - # If the current epoch gets justified, fill the last bit - current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) - if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - state.justification_bitfield |= 1 - - # Process finalizations - bitfield = state.justification_bitfield - current_epoch = get_current_epoch(state) - # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: - new_finalized_epoch = state.previous_justified_epoch - # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.previous_justified_epoch - # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.current_justified_epoch - # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: - new_finalized_epoch = state.current_justified_epoch - - # Update state jusification/finality fields - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root - if new_justified_epoch != state.current_justified_epoch: - state.current_justified_epoch = new_justified_epoch - state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) - if new_finalized_epoch != state.finalized_epoch: - state.finalized_epoch = new_finalized_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) -def process_crosslinks(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - previous_epoch = current_epoch - 1 - next_epoch = current_epoch + 1 - for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - if 3 * participating_balance >= 2 * total_balance: - state.latest_crosslinks[shard] = Crosslink( - epoch=slot_to_epoch(slot), - crosslink_data_root=winning_root - ) -def maybe_reset_eth1_period(state: BeaconState) -> None: - if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - for eth1_data_vote in state.eth1_data_votes: - # If a majority of all votes were for a particular eth1_data value, - # then set that as the new canonical value - if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: - state.latest_eth1_data = eth1_data_vote.eth1_data - state.eth1_data_votes = [] -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - if get_previous_total_balance(state) == 0: - return 0 - - adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT - return get_effective_balance(state, index) // adjusted_quotient // 5 -def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: - return ( - get_base_reward(state, index) + - get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 - ) -def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch - if epochs_since_finality <= 4: - return compute_normal_justification_and_finalization_deltas(state) - else: - return compute_inactivity_leak_deltas(state) -def compute_normal_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - # Some helper variables - boundary_attestations = get_previous_epoch_boundary_attestations(state) - boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) - total_balance = get_previous_total_balance(state) - total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - matching_head_balance = get_attesting_balance(state, matching_head_attestations) - # Process rewards or penalties for all validators - for index in get_active_validator_indices(state.validator_registry, get_previous_epoch(state)): - # Expected FFG source - if index in get_attesting_indices(state, state.previous_epoch_attestations): - deltas[0][index] += get_base_reward(state, index) * total_attesting_balance // total_balance - # Inclusion speed bonus - deltas[0][index] += ( - get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_distance(state, index) - ) - else: - deltas[1][index] += get_base_reward(state, index) - # Expected FFG target - if index in get_attesting_indices(state, boundary_attestations): - deltas[0][index] += get_base_reward(state, index) * boundary_attesting_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - # Expected head - if index in get_attesting_indices(state, matching_head_attestations): - deltas[0][index] += get_base_reward(state, index) * matching_head_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - # Proposer bonus - if index in get_attesting_indices(state, state.previous_epoch_attestations): - proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) - deltas[0][proposer_index] += get_base_reward(state, index) // ATTESTATION_INCLUSION_REWARD_QUOTIENT - return deltas -def compute_inactivity_leak_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - boundary_attestations = get_previous_epoch_boundary_attestations(state) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - active_validator_indices = get_active_validator_indices(state.validator_registry, get_previous_epoch(state)) - epochs_since_finality = get_current_epoch(state) + 1 - state.finalized_epoch - for index in active_validator_indices: - if index not in get_attesting_indices(state, state.previous_epoch_attestations): - deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) - else: - # If a validator did attest, apply a small penalty for getting attestations included late - deltas[0][index] += ( - get_base_reward(state, index) * MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_distance(state, index) - ) - deltas[1][index] += get_base_reward(state, index) - if index not in get_attesting_indices(state, boundary_attestations): - deltas[1][index] += get_inactivity_penalty(state, index, epochs_since_finality) - if index not in get_attesting_indices(state, matching_head_attestations): - deltas[1][index] += get_base_reward(state, index) - # Penalize slashed-but-inactive validators as though they were active but offline - for index in range(len(state.validator_registry)): - eligible = ( - index not in active_validator_indices and - state.validator_registry[index].slashed and - get_current_epoch(state) < state.validator_registry[index].withdrawable_epoch - ) - if eligible: - deltas[1][index] += ( - 2 * get_inactivity_penalty(state, index, epochs_since_finality) + - get_base_reward(state, index) - ) - return deltas -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - # deltas[0] for rewards - # deltas[1] for penalties - deltas = [ - [0 for index in range(len(state.validator_registry))], - [0 for index in range(len(state.validator_registry))] - ] - previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - for slot in range(previous_epoch_start_slot, current_epoch_start_slot): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - for index in crosslink_committee: - if index in participants: - deltas[0][index] += get_base_reward(state, index) * participating_balance // total_balance - else: - deltas[1][index] += get_base_reward(state, index) - return deltas -def apply_rewards(state: BeaconState) -> None: - deltas1 = get_justification_and_finalization_deltas(state) - deltas2 = get_crosslink_deltas(state) - for i in range(len(state.validator_registry)): - state.validator_balances[i] = max( - 0, - state.validator_balances[i] + deltas1[0][i] + deltas2[0][i] - deltas1[1][i] - deltas2[1][i] - ) -def process_ejections(state: BeaconState) -> None: - """ - Iterate through the validator registry - and eject active validators with balance below ``EJECTION_BALANCE``. - """ - for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): - if state.validator_balances[index] < EJECTION_BALANCE: - exit_validator(state, index) -def should_update_validator_registry(state: BeaconState) -> bool: - # Must have finalized a new block - if state.finalized_epoch <= state.validator_registry_update_epoch: - return False - # Must have processed new crosslinks on all shards of the current epoch - shards_to_check = [ - (state.current_shuffling_start_shard + i) % SHARD_COUNT - for i in range(get_current_epoch_committee_count(state)) - ] - for shard in shards_to_check: - if state.latest_crosslinks[shard].epoch <= state.validator_registry_update_epoch: - return False - return True -def update_validator_registry(state: BeaconState) -> None: - """ - Update validator registry. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - # The active validators - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - # The total effective balance of active validators - total_balance = get_total_balance(state, active_validator_indices) - - # The maximum balance churn in Gwei (for deposits and exits separately) - max_balance_churn = max( - MAX_DEPOSIT_AMOUNT, - total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) - ) - - # Activate validators within the allowable balance churn - balance_churn = 0 - for index, validator in enumerate(state.validator_registry): - if validator.activation_epoch == FAR_FUTURE_EPOCH and state.validator_balances[index] >= MAX_DEPOSIT_AMOUNT: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Activate validator - activate_validator(state, index, is_genesis=False) - - # Exit validators within the allowable balance churn - balance_churn = 0 - for index, validator in enumerate(state.validator_registry): - if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Exit validator - exit_validator(state, index) - - state.validator_registry_update_epoch = current_epoch -def update_registry_and_shuffling_data(state: BeaconState) -> None: - # First set previous shuffling data to current shuffling data - state.previous_shuffling_epoch = state.current_shuffling_epoch - state.previous_shuffling_start_shard = state.current_shuffling_start_shard - state.previous_shuffling_seed = state.current_shuffling_seed - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 - # Check if we should update, and if so, update - if should_update_validator_registry(state): - update_validator_registry(state) - # If we update the registry, update the shuffling data and shards as well - state.current_shuffling_epoch = next_epoch - state.current_shuffling_start_shard = ( - state.current_shuffling_start_shard + - get_current_epoch_committee_count(state) % SHARD_COUNT - ) - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) - else: - # If processing at least one crosslink keeps failing, then reshuffle every power of two, - # but don't update the current_shuffling_start_shard - epochs_since_last_registry_update = current_epoch - state.validator_registry_update_epoch - if epochs_since_last_registry_update > 1 and is_power_of_two(epochs_since_last_registry_update): - state.current_shuffling_epoch = next_epoch - state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch) -def process_slashings(state: BeaconState) -> None: - """ - Process the slashings. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) - - # Compute `total_penalties` - total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] - total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - total_penalties = total_at_end - total_at_start - - for index, validator in enumerate(state.validator_registry): - if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: - penalty = max( - get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, - get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT - ) - state.validator_balances[index] -= penalty -def process_exit_queue(state: BeaconState) -> None: - """ - Process the exit queue. - Note that this function mutates ``state``. - """ - def eligible(index): - validator = state.validator_registry[index] - # Filter out dequeued validators - if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: - return False - # Dequeue if the minimum amount of time has passed - else: - return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY - - eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) - # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index - sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) - for dequeues, index in enumerate(sorted_indices): - if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: - break - prepare_validator_for_withdrawal(state, index) -def finish_epoch_update(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 - # Set active index root - index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH - state.latest_active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) - ) - # Set total slashed balances - state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - ) - # Set randao mix - state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) - # Set historical root accumulator - if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch( - block_roots=state.latest_block_roots, - state_roots=state.latest_state_roots, - ) - state.historical_roots.append(hash_tree_root(historical_batch)) - # Rotate current/previous epoch attestations - state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = [] -def advance_slot(state: BeaconState) -> None: - state.slot += 1 -def process_block_header(state: BeaconState, block: BeaconBlock) -> None: - # Verify that the slots match - assert block.slot == state.slot - # Verify that the parent matches - assert block.previous_block_root == hash_tree_root(state.latest_block_header) - # Save current block as the new latest block - state.latest_block_header = get_temporary_block_header(block) - # Verify proposer signature - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(block), - signature=block.signature, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) - ) -def process_randao(state: BeaconState, block: BeaconBlock) -> None: - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - # Verify that the provided randao value is valid - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=hash_tree_root(get_current_epoch(state)), - signature=block.body.randao_reveal, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) - ) - # Mix it in - state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( - xor(get_randao_mix(state, get_current_epoch(state)), - hash(block.body.randao_reveal)) - ) -def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - for eth1_data_vote in state.eth1_data_votes: - # If someone else has already voted for the same hash, add to its counter - if eth1_data_vote.eth1_data == block.body.eth1_data: - eth1_data_vote.vote_count += 1 - return - # If we're seeing this hash for the first time, make a new counter - state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) -def process_proposer_slashing(state: BeaconState, - proposer_slashing: ProposerSlashing) -> None: - """ - Process ``ProposerSlashing`` transaction. - Note that this function mutates ``state``. - """ - proposer = state.validator_registry[proposer_slashing.proposer_index] - # Verify that the epoch is the same - assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) - # But the headers are different - assert proposer_slashing.header_1 != proposer_slashing.header_2 - # Proposer is not yet slashed - assert proposer.slashed is False - # Signatures are valid - for header in (proposer_slashing.header_1, proposer_slashing.header_2): - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(header), - signature=header.signature, - domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) - ) - slash_validator(state, proposer_slashing.proposer_index) -def process_attester_slashing(state: BeaconState, - attester_slashing: AttesterSlashing) -> None: - """ - Process ``AttesterSlashing`` transaction. - Note that this function mutates ``state``. - """ - attestation1 = attester_slashing.slashable_attestation_1 - attestation2 = attester_slashing.slashable_attestation_2 - # Check that the attestations are conflicting - assert attestation1.data != attestation2.data - assert ( - is_double_vote(attestation1.data, attestation2.data) or - is_surround_vote(attestation1.data, attestation2.data) - ) - assert verify_slashable_attestation(state, attestation1) - assert verify_slashable_attestation(state, attestation2) - slashable_indices = [ - index for index in attestation1.validator_indices - if ( - index in attestation2.validator_indices and - state.validator_registry[index].slashed is False - ) - ] - assert len(slashable_indices) >= 1 - for index in slashable_indices: - slash_validator(state, index) -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - """ - Process ``Attestation`` transaction. - Note that this function mutates ``state``. - """ - # Can't submit attestations that are too far in history (or in prehistory) - assert attestation.data.slot >= GENESIS_SLOT - assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH - # Can't submit attestations too quickly - assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot - # Verify that the justified epoch and root is correct - if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state): - # Case 1: current epoch attestations - assert attestation.data.source_epoch == state.current_justified_epoch - assert attestation.data.source_root == state.current_justified_root - else: - # Case 2: previous epoch attestations - assert attestation.data.source_epoch == state.previous_justified_epoch - assert attestation.data.source_root == state.previous_justified_root - # Check that the crosslink data is valid - acceptable_crosslink_data = { - # Case 1: Latest crosslink matches the one in the state - attestation.data.previous_crosslink, - # Case 2: State has already been updated, state's latest crosslink matches the crosslink - # the attestation is trying to create - Crosslink( - crosslink_data_root=attestation.data.crosslink_data_root, - epoch=slot_to_epoch(attestation.data.slot) - ) - } - assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data - # Attestation must be nonempty! - assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield) - # Custody must be empty (to be removed in phase 1) - assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) - # Get the committee for the specific shard that this attestation is for - crosslink_committee = [ - committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot) - if shard == attestation.data.shard - ][0] - # Custody bitfield must be a subset of the attestation bitfield - for i in range(len(crosslink_committee)): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0: - assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0 - # Verify aggregate signature - participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield) - custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants] - - assert bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)), - ], - signature=attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION), - ) - # Crosslink data root is zero (to be removed in phase 1) - assert attestation.data.crosslink_data_root == ZERO_HASH - # Apply the attestation - pending_attestation = PendingAttestation( - data=attestation.data, - aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, - inclusion_slot=state.slot - ) - if slot_to_epoch(attestation.data.slot) == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state): - state.previous_epoch_attestations.append(pending_attestation) -def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: - """ - Process ``VoluntaryExit`` transaction. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[exit.validator_index] - # Verify the validator has not yet exited - assert validator.exit_epoch == FAR_FUTURE_EPOCH - # Verify the validator has not initiated an exit - assert validator.initiated_exit is False - # Exits must specify an epoch when they become valid; they are not valid before then - assert get_current_epoch(state) >= exit.epoch - # Must have been in the validator set long enough - assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD - # Verify signature - assert bls_verify( - pubkey=validator.pubkey, - message_hash=signed_root(exit), - signature=exit.signature, - domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) - ) - # Run the exit - initiate_validator_exit(state, exit.validator_index) -def process_transfer(state: BeaconState, transfer: Transfer) -> None: - """ - Process ``Transfer`` transaction. - Note that this function mutates ``state``. - """ - # Verify the amount and fee aren't individually too big (for anti-overflow purposes) - assert state.validator_balances[transfer.sender] >= max(transfer.amount, transfer.fee) - # Verify that we have enough ETH to send, and that after the transfer the balance will be either - # exactly zero or at least MIN_DEPOSIT_AMOUNT - assert ( - state.validator_balances[transfer.sender] == transfer.amount + transfer.fee or - state.validator_balances[transfer.sender] >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT - ) - # A transfer is valid in only one slot - assert state.slot == transfer.slot - # Only withdrawn or not-yet-deposited accounts can transfer - assert ( - get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH - ) - # Verify that the pubkey is valid - assert ( - state.validator_registry[transfer.sender].withdrawal_credentials == - BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] - ) - # Verify that the signature is valid - assert bls_verify( - pubkey=transfer.pubkey, - message_hash=signed_root(transfer), - signature=transfer.signature, - domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) - ) - # Process the transfer - state.validator_balances[transfer.sender] -= transfer.amount + transfer.fee - state.validator_balances[transfer.recipient] += transfer.amount - state.validator_balances[get_beacon_proposer_index(state, state.slot)] += transfer.fee -def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: - assert block.state_root == hash_tree_root(state) - -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) - - if param_hash in shuffling_cache: - # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] - else: - # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret - \ No newline at end of file diff --git a/build/phase0/state_transition.py b/build/phase0/state_transition.py deleted file mode 100644 index 170f647ab..000000000 --- a/build/phase0/state_transition.py +++ /dev/null @@ -1,100 +0,0 @@ -from . import spec - - -from typing import ( - Any, - Callable, - List, - NewType, - Tuple, -) - -from .spec import ( - BeaconState, - BeaconBlock, -) - - -def process_transaction_type(state: BeaconState, - transactions: List[Any], - max_transactions: int, - tx_fn: Callable[[BeaconState, Any], None]) -> None: - assert len(transactions) <= max_transactions - for transaction in transactions: - tx_fn(state, transaction) - - -def process_transactions(state: BeaconState, block: BeaconBlock) -> None: - process_transaction_type( - state, - block.body.proposer_slashings, - spec.MAX_PROPOSER_SLASHINGS, - spec.process_proposer_slashing, - ) - process_transaction_type( - state, - block.body.attester_slashings, - spec.MAX_ATTESTER_SLASHINGS, - spec.process_attester_slashing, - ) - process_transaction_type( - state, - block.body.attestations, - spec.MAX_ATTESTATIONS, - spec.process_attestation, - ) - process_transaction_type( - state, - block.body.deposits, - spec.MAX_DEPOSITS, - spec.process_deposit, - ) - process_transaction_type( - state, - block.body.voluntary_exits, - spec.MAX_VOLUNTARY_EXITS, - spec.process_voluntary_exit, - ) - assert len(block.body.transfers) == len(set(block.body.transfers)) - process_transaction_type( - state, - block.body.transfers, - spec.MAX_TRANSFERS, - spec.process_transfer, - ) - - -def process_block(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> None: - spec.process_block_header(state, block) - spec.process_randao(state, block) - spec.process_eth1_data(state, block) - - process_transactions(state, block) - if verify_state_root: - spec.verify_block_state_root(state, block) - - -def process_epoch_transition(state: BeaconState) -> None: - spec.update_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.maybe_reset_eth1_period(state) - spec.apply_rewards(state) - spec.process_ejections(state) - spec.update_registry_and_shuffling_data(state) - spec.process_slashings(state) - spec.process_exit_queue(state) - spec.finish_epoch_update(state) - - -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - while state.slot < block.slot: - spec.cache_state(state) - if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) - spec.advance_slot(state) - if block.slot == state.slot: - process_block(state, block) From 55c337a35c7cb6caff0d1fe6f3179d2b3161a579 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 16:20:24 -0600 Subject: [PATCH 21/32] seperate tests 'sanity' and 'minimal-config' vs all --- Makefile | 13 ++- tests/phase0/conftest.py | 102 +++++------------ tests/phase0/helpers.py | 139 +++++++++++++++++++++++ tests/phase0/test_sanity.py | 216 +++++------------------------------- 4 files changed, 202 insertions(+), 268 deletions(-) create mode 100644 tests/phase0/helpers.py diff --git a/Makefile b/Makefile index 593ea8bf4..f0f7557e9 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,19 @@ SCRIPT_DIR = ./scripts BUILD_DIR = ./build UTILS_DIR = ./utils -.PHONY: clean all $(BUILD_DIR)/phase0 +.PHONY: clean all test + + +all: $(BUILD_DIR)/phase0 + + +clean: + rm -rf $(BUILD_DIR) + + +test: + pytest -m "sanity and minimal_config" tests/ $(BUILD_DIR)/phase0: mkdir -p $@ diff --git a/tests/phase0/conftest.py b/tests/phase0/conftest.py index 7d372f164..e92896e92 100644 --- a/tests/phase0/conftest.py +++ b/tests/phase0/conftest.py @@ -1,28 +1,27 @@ import pytest -from py_ecc import bls - from build.phase0 import spec -from build.phase0.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) -from build.phase0.spec import ( - Deposit, - DepositData, - DepositInput, - Eth1Data, - get_genesis_beacon_state, - verify_merkle_branch, - hash, +from tests.phase0.helpers import ( + privkeys_list, + pubkeys_list, + create_genesis_state, ) -privkeys_list = [i+1 for i in range(1000)] -pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} +DEFAULT_CONFIG = {} # no change + +MINIMAL_CONFIG = { + "SHARD_COUNT": 8, + "MIN_ATTESTATION_INCLUSION_DELAY": 2, + "TARGET_COMMITTEE_SIZE": 4, + "SLOTS_PER_EPOCH": 8, + "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, + "SLOTS_PER_HISTORICAL_ROOT": 64, + "LATEST_RANDAO_MIXES_LENGTH": 64, + "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, + "LATEST_SLASHED_EXIT_LENGTH": 64, +} @pytest.fixture @@ -53,19 +52,14 @@ def overwrite_spec_config(config): spec.BeaconState.fields['latest_slashed_balances'][1] = config[field] -@pytest.fixture -def config(): - return { - "SHARD_COUNT": 8, - "MIN_ATTESTATION_INCLUSION_DELAY": 2, - "TARGET_COMMITTEE_SIZE": 4, - "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, - "SLOTS_PER_HISTORICAL_ROOT": 64, - "LATEST_RANDAO_MIXES_LENGTH": 64, - "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, - "LATEST_SLASHED_EXIT_LENGTH": 64, - } +@pytest.fixture( + params=[ + pytest.param(MINIMAL_CONFIG, marks=pytest.mark.minimal_config), + DEFAULT_CONFIG, + ] +) +def config(request): + return request.param @pytest.fixture(autouse=True) @@ -73,52 +67,6 @@ def overwrite_config(config): overwrite_spec_config(config) -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): - deposit_timestamp = 0 - proof_of_possession = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys_list[i] - privkey = pubkey_to_privkey[pubkey] - deposit_data = DepositData( - amount=spec.MAX_DEPOSIT_AMOUNT, - timestamp=deposit_timestamp, - deposit_input=DepositInput( - pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=proof_of_possession, - ), - ) - item = hash(deposit_data.serialize()) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - deposit_data=deposit_data_list[i] - )) - return genesis_validator_deposits, root - - -def create_genesis_state(num_validators, deposit_data_leaves): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) - return get_genesis_beacon_state( - initial_deposits, - genesis_time=0, - genesis_eth1_data=Eth1Data( - deposit_root=deposit_root, - block_hash=spec.ZERO_HASH, - ), - ) - @pytest.fixture def num_validators(): return 100 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py new file mode 100644 index 000000000..fa0ba61b5 --- /dev/null +++ b/tests/phase0/helpers.py @@ -0,0 +1,139 @@ +from copy import deepcopy + +from py_ecc import bls + +import build.phase0.spec as spec +from build.phase0.utils.minimal_ssz import signed_root +from build.phase0.spec import ( + AttestationData, + Deposit, + DepositInput, + DepositData, + Eth1Data, + get_block_root, + get_current_epoch, + get_domain, + get_empty_block, + get_epoch_start_slot, + get_genesis_beacon_state, + verify_merkle_branch, + hash, +) +from build.phase0.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) + + +privkeys_list = [i+1 for i in range(1000)] +pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} + + +def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): + deposit_timestamp = 0 + proof_of_possession = b'\x33' * 96 + + deposit_data_list = [] + for i in range(num_validators): + pubkey = pubkeys_list[i] + privkey = pubkey_to_privkey[pubkey] + deposit_data = DepositData( + amount=spec.MAX_DEPOSIT_AMOUNT, + timestamp=deposit_timestamp, + deposit_input=DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=proof_of_possession, + ), + ) + item = hash(deposit_data.serialize()) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=i)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) + deposit_data_list.append(deposit_data) + + genesis_validator_deposits = [] + for i in range(num_validators): + genesis_validator_deposits.append(Deposit( + proof=list(get_merkle_proof(tree, item_index=i)), + index=i, + deposit_data=deposit_data_list[i] + )) + return genesis_validator_deposits, root + + +def create_genesis_state(num_validators, deposit_data_leaves): + initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves) + return get_genesis_beacon_state( + initial_deposits, + genesis_time=0, + genesis_eth1_data=Eth1Data( + deposit_root=deposit_root, + block_hash=spec.ZERO_HASH, + ), + ) + +def build_empty_block_for_next_slot(state): + empty_block = get_empty_block() + empty_block.slot = state.slot + 1 + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = previous_block_header.hash_tree_root() + return empty_block + + +def build_deposit_data(state, pubkey, privkey, amount): + deposit_input = DepositInput( + pubkey=pubkey, + withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + proof_of_possession=b'00'*96, + ) + proof_of_possession = bls.sign( + message_hash=signed_root(deposit_input), + privkey=privkey, + domain=get_domain( + state.fork, + get_current_epoch(state), + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_input.proof_of_possession = proof_of_possession + deposit_data = DepositData( + amount=amount, + timestamp=0, + deposit_input=deposit_input, + ) + return deposit_data + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + block_root = build_empty_block_for_next_slot(state).previous_block_root + + epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if epoch_start_slot == slot: + epoch_boundary_root = block_root + else: + get_block_root(state, epoch_start_slot) + + if slot < epoch_start_slot: + justified_block_root = state.previous_justified_root + else: + justified_block_root = state.current_justified_root + + return AttestationData( + slot=slot, + shard=shard, + beacon_block_root=block_root, + source_epoch=state.current_justified_epoch, + source_root=justified_block_root, + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink=deepcopy(state.latest_crosslinks[shard]), + ) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 0e04df5dd..bfbb2de94 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -1,8 +1,10 @@ import os import sys import time - from copy import deepcopy + +import pytest + from py_ecc import bls import build.phase0.spec as spec @@ -48,78 +50,15 @@ from build.phase0.utils.merkle_minimal import ( get_merkle_proof, get_merkle_root, ) -# from state_test_gen import ( - # generate_from_test, - # dump_json, - # dump_yaml, -# ) +from tests.phase0.helpers import ( + build_attestation_data, + build_deposit_data, + build_empty_block_for_next_slot, +) -def get_empty_root(): - return get_merkle_root((spec.ZERO_HASH,)) - - -def construct_empty_block_for_next_slot(state): - empty_block = get_empty_block() - empty_block.slot = state.slot + 1 - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = previous_block_header.hash_tree_root() - return empty_block - - -def create_deposit_data(state, pubkey, privkey, amount): - deposit_input = DepositInput( - pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'00'*96, - ) - proof_of_possession = bls.sign( - message_hash=signed_root(deposit_input), - privkey=privkey, - domain=get_domain( - state.fork, - get_current_epoch(state), - spec.DOMAIN_DEPOSIT, - ) - ) - deposit_input.proof_of_possession = proof_of_possession - deposit_data = DepositData( - amount=amount, - timestamp=0, - deposit_input=deposit_input, - ) - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - block_root = construct_empty_block_for_next_slot(state).previous_block_root - - epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if epoch_start_slot == slot: - epoch_boundary_root = block_root - else: - get_block_root(state, epoch_start_slot) - - if slot < epoch_start_slot: - justified_block_root = state.previous_justified_root - else: - justified_block_root = state.current_justified_root - - return AttestationData( - slot=slot, - shard=shard, - beacon_block_root=block_root, - source_epoch=state.current_justified_epoch, - source_root=justified_block_root, - target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink=deepcopy(state.latest_crosslinks[shard]), - ) - +# mark entire file as 'sanity' +pytestmark = pytest.mark.sanity def test_slot_transition(state): test_state = deepcopy(state) @@ -133,7 +72,7 @@ def test_slot_transition(state): def test_empty_block_transition(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 @@ -144,7 +83,7 @@ def test_empty_block_transition(state): def test_skipped_slots(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += 3 state_transition(test_state, block) @@ -158,7 +97,7 @@ def test_skipped_slots(state): def test_empty_epoch_transition(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH state_transition(test_state, block) @@ -172,7 +111,7 @@ def test_empty_epoch_transition(state): def test_empty_epoch_transition_not_finalizing(state): test_state = deepcopy(state) - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH * 5 state_transition(test_state, block) @@ -226,7 +165,7 @@ def test_proposer_slashing(state, pubkeys, privkeys): # # Add to state via block transition # - block = construct_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(test_state) block.body.proposer_slashings.append(proposer_slashing) state_transition(test_state, block) @@ -251,7 +190,7 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): index = len(test_deposit_data_leaves) pubkey = pubkeys[index] privkey = privkeys[index] - deposit_data = create_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) + deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT) item = hash(deposit_data.serialize()) test_deposit_data_leaves.append(item) @@ -268,7 +207,7 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys): pre_state.latest_eth1_data.deposit_root = root post_state = deepcopy(pre_state) - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.body.deposits.append(deposit) state_transition(post_state, block) @@ -287,7 +226,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): amount = spec.MAX_DEPOSIT_AMOUNT // 4 pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] - deposit_data = create_deposit_data(pre_state, pubkey, privkey, amount) + deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) merkle_index = len(test_deposit_data_leaves) item = hash(deposit_data.serialize()) @@ -304,7 +243,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves): ) pre_state.latest_eth1_data.deposit_root = root - block = construct_empty_block_for_next_slot(pre_state) + block = build_empty_block_for_next_slot(pre_state) block.body.deposits.append(deposit) pre_balance = pre_state.validator_balances[validator_index] @@ -365,7 +304,7 @@ def test_attestation(state, pubkeys, privkeys): # # Add to state via block transition # - attestation_block = construct_empty_block_for_next_slot(test_state) + attestation_block = build_empty_block_for_next_slot(test_state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) state_transition(test_state, attestation_block) @@ -377,7 +316,7 @@ def test_attestation(state, pubkeys, privkeys): # pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) - epoch_block = construct_empty_block_for_next_slot(test_state) + epoch_block = build_empty_block_for_next_slot(test_state) epoch_block.slot += spec.SLOTS_PER_EPOCH state_transition(test_state, epoch_block) @@ -417,7 +356,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): # # Add to state via block transition # - initiate_exit_block = construct_empty_block_for_next_slot(post_state) + initiate_exit_block = build_empty_block_for_next_slot(post_state) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) state_transition(post_state, initiate_exit_block) @@ -428,7 +367,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): # # Process within epoch transition # - exit_block = construct_empty_block_for_next_slot(post_state) + exit_block = build_empty_block_for_next_slot(post_state) exit_block.slot += spec.SLOTS_PER_EPOCH state_transition(post_state, exit_block) @@ -476,7 +415,7 @@ def test_transfer(state, pubkeys, privkeys): # # Add to state via block transition # - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.body.transfers.append(transfer) state_transition(post_state, block) @@ -503,7 +442,7 @@ def test_ejection(state): # # trigger epoch transition # - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) block.slot += spec.SLOTS_PER_EPOCH state_transition(post_state, block) @@ -518,7 +457,7 @@ def test_historical_batch(state): post_state = deepcopy(pre_state) - block = construct_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(post_state) state_transition(post_state, block) @@ -527,106 +466,3 @@ def test_historical_batch(state): assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1 return pre_state, [block], post_state - - -def sanity_tests(num_validators=100, config=None): - print(f"Buidling state with {num_validators} validators...") - if config: - overwrite_spec_config(config) - genesis_state = create_genesis_state(num_validators=num_validators) - print("done!") - print() - - test_cases = [] - - print("Running some sanity check tests...\n") - test_slot_transition(genesis_state) - print("Passed slot transition test\n") - test_cases.append( - generate_from_test(test_empty_block_transition, genesis_state, config=config, fields=['slot']) - ) - print("Passed empty block transition test\n") - test_cases.append( - generate_from_test(test_skipped_slots, genesis_state, config=config, fields=['slot', 'latest_block_roots']) - ) - print("Passed skipped slot test\n") - test_cases.append( - generate_from_test(test_empty_epoch_transition, genesis_state, config=config, fields=['slot', 'latest_block_roots']) - ) - print("Passed empty epoch transition test\n") - test_cases.append( - generate_from_test(test_empty_epoch_transition_not_finalizing, genesis_state, config=config, fields=['slot', 'finalized_epoch']) - ) - print("Passed non-finalizing epoch test\n") - test_cases.append( - generate_from_test(test_proposer_slashing, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed proposer slashing test\n") - test_cases.append( - generate_from_test(test_attestation, genesis_state, config=config, fields=['previous_epoch_attestations', 'current_epoch_attestations']) - ) - print("Passed attestation test\n") - test_cases.append( - generate_from_test(test_deposit_in_block, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed deposit test\n") - test_cases.append( - generate_from_test(test_deposit_top_up, genesis_state, config=config, fields=['validator_registry', 'validator_balances']) - ) - print("Passed deposit top up test\n") - test_cases.append( - generate_from_test(test_voluntary_exit, genesis_state, config=config, fields=['validator_registry']) - ) - print("Passed voluntary exit test\n") - test_cases.append( - generate_from_test(test_transfer, genesis_state, config=config, fields=['validator_balances']) - ) - print("Passed transfer test\n") - test_cases.append( - generate_from_test(test_ejection, genesis_state, config=config, fields=['validator_registry']) - ) - print("Passed ejection test\n") - test_cases.append( - generate_from_test(test_historical_batch, genesis_state, config=config, fields=['historical_roots']) - ) - print("Passed historical batch test\n") - print("done!") - - return test_cases - - -if __name__ == "__main__": - config = { - "SHARD_COUNT": 8, - "MIN_ATTESTATION_INCLUSION_DELAY": 2, - "TARGET_COMMITTEE_SIZE": 4, - "SLOTS_PER_EPOCH": 8, - "GENESIS_EPOCH": spec.GENESIS_SLOT // 8, - "SLOTS_PER_HISTORICAL_ROOT": 64, - "LATEST_RANDAO_MIXES_LENGTH": 64, - "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64, - "LATEST_SLASHED_EXIT_LENGTH": 64, - } - - test_cases = sanity_tests(32, config) - # uncomment below to run/generate against the default config - # test_cases = sanity_tests(100) - - test = {} - metadata = {} - metadata['title'] = "Sanity tests" - metadata['summary'] = "Basic sanity checks from phase 0 spec pythonization. All tests are run with `verify_signatures` as set to False." - metadata['test_suite'] = "beacon_state" - metadata['fork'] = "tchaikovsky" - metadata['version'] = "v0.5.0" - test['metadata'] = metadata - test['test_cases'] = test_cases - - if '--output-json' in sys.argv: - os.makedirs('output', exist_ok=True) - with open("output/sanity_check_tests.json", "w+") as outfile: - dump_json(test, outfile) - if '--output-yaml' in sys.argv: - os.makedirs('output', exist_ok=True) - with open("output/sanity_check_tests.yaml", "w+") as outfile: - dump_yaml(test, outfile) From 4440be4e1f92154242960eeaa12320c3b0f5e404 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 18 Mar 2019 16:30:16 -0600 Subject: [PATCH 22/32] add comment to Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index f0f7557e9..b45cec410 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,12 @@ clean: rm -rf $(BUILD_DIR) +# runs a limited set of tests against a minimal config +# run pytest with `-m` option to full suite test: pytest -m "sanity and minimal_config" tests/ + $(BUILD_DIR)/phase0: mkdir -p $@ python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py From a07219c57045a22d13304148dbae8bb9121a0181 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 11:39:19 +0800 Subject: [PATCH 23/32] Fix linter --- scripts/phase0/build_spec.py | 1 - scripts/phase0/function_puller.py | 2 +- tests/phase0/helpers.py | 5 +++-- tests/phase0/test_sanity.py | 34 ++++++++----------------------- utils/phase0/bls_stub.py | 2 +- utils/phase0/hash_function.py | 5 +++-- utils/phase0/merkle_minimal.py | 12 ++++++----- utils/phase0/minimal_ssz.py | 24 ++++++++++++++++++---- utils/phase0/state_transition.py | 2 +- 9 files changed, 45 insertions(+), 42 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index eb4f580bd..ae5a5a4f2 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -34,7 +34,6 @@ Any = None Store = None """) - code_lines += function_puller.get_lines(sourcefile) code_lines.append(""" diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 8d1c1a0cc..7d5796fc7 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -9,7 +9,7 @@ def get_lines(file_name): for linenum, line in enumerate(open(sys.argv[1]).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': - current_name = line[line[:-1].rfind('`')+1: -1] + current_name = line[line[:-1].rfind('`') + 1: -1] if line[:9] == '```python': assert pulling_from is None pulling_from = linenum + 1 diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index fa0ba61b5..f7c39ffec 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -26,7 +26,7 @@ from build.phase0.utils.merkle_minimal import ( ) -privkeys_list = [i+1 for i in range(1000)] +privkeys_list = [i + 1 for i in range(1000)] pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)} @@ -77,6 +77,7 @@ def create_genesis_state(num_validators, deposit_data_leaves): ), ) + def build_empty_block_for_next_slot(state): empty_block = get_empty_block() empty_block.slot = state.slot + 1 @@ -91,7 +92,7 @@ def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'00'*96, + proof_of_possession=b'\x00' * 96, ) proof_of_possession = bls.sign( message_hash=signed_root(deposit_input), diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index bfbb2de94..8799c1ffb 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -1,6 +1,3 @@ -import os -import sys -import time from copy import deepcopy import pytest @@ -12,32 +9,21 @@ from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( # SSZ Attestation, - AttestationData, AttestationDataAndCustodyBit, BeaconBlockHeader, Deposit, - DepositData, - DepositInput, - Eth1Data, Transfer, ProposerSlashing, - Validator, VoluntaryExit, # functions - int_to_bytes32, - int_to_bytes48, get_active_validator_indices, get_attestation_participants, get_block_root, get_crosslink_committees_at_slot, get_current_epoch, get_domain, - get_empty_block, - get_epoch_start_slot, - get_genesis_beacon_state, get_state_root, advance_slot, - slot_to_epoch, cache_state, verify_merkle_branch, hash, @@ -60,6 +46,7 @@ from tests.phase0.helpers import ( # mark entire file as 'sanity' pytestmark = pytest.mark.sanity + def test_slot_transition(state): test_state = deepcopy(state) cache_state(test_state) @@ -126,18 +113,17 @@ def test_proposer_slashing(state, pubkeys, privkeys): test_state = deepcopy(state) current_epoch = get_current_epoch(test_state) validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1] - pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] slot = spec.GENESIS_SLOT header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=b'\x00'*32, - state_root=b'\x00'*32, - block_body_root=b'\x00'*32, - signature=b'\x00'*96 + previous_block_root=b'\x00' * 32, + state_root=b'\x00' * 32, + block_body_root=b'\x00' * 32, + signature=b'\x00' * 96 ) header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02'*32 + header_2.previous_block_root = b'\x02' * 32 header_2.slot = slot + 1 domain = get_domain( @@ -273,7 +259,7 @@ def test_attestation(state, pubkeys, privkeys): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=b'\x00'*96, + aggregate_signature=b'\x00' * 96, ) participants = get_attestation_participants( test_state, @@ -283,7 +269,6 @@ def test_attestation(state, pubkeys, privkeys): assert len(participants) == 1 validator_index = participants[0] - pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] message_hash = AttestationDataAndCustodyBit( @@ -329,7 +314,6 @@ def test_attestation(state, pubkeys, privkeys): def test_voluntary_exit(state, pubkeys, privkeys): pre_state = deepcopy(state) validator_index = get_active_validator_indices(pre_state.validator_registry, get_current_epoch(pre_state))[-1] - pubkey = pubkeys[validator_index] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -341,7 +325,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=b'\x00'*96, + signature=b'\x00' * 96, ) voluntary_exit.signature = bls.sign( message_hash=signed_root(voluntary_exit), @@ -392,7 +376,7 @@ def test_transfer(state, pubkeys, privkeys): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=b'\x00'*96, + signature=b'\x00' * 96, ) transfer.signature = bls.sign( message_hash=signed_root(transfer), diff --git a/utils/phase0/bls_stub.py b/utils/phase0/bls_stub.py index 7e3a6a308..108c4ef71 100644 --- a/utils/phase0/bls_stub.py +++ b/utils/phase0/bls_stub.py @@ -9,4 +9,4 @@ def bls_verify_multiple(pubkeys, message_hashes, signature, domain): def bls_aggregate_pubkeys(pubkeys): - return b'\x42'*96 + return b'\x42' * 96 diff --git a/utils/phase0/hash_function.py b/utils/phase0/hash_function.py index da5b4d979..21e6555bf 100644 --- a/utils/phase0/hash_function.py +++ b/utils/phase0/hash_function.py @@ -1,6 +1,7 @@ -from hashlib import sha256 +# from hashlib import sha256 from eth_utils import keccak # def hash(x): return sha256(x).digest() -def hash(x): return keccak(x) +def hash(x): + return keccak(x) diff --git a/utils/phase0/merkle_minimal.py b/utils/phase0/merkle_minimal.py index a811350ce..7c5483de3 100644 --- a/utils/phase0/merkle_minimal.py +++ b/utils/phase0/merkle_minimal.py @@ -2,8 +2,9 @@ from .hash_function import hash zerohashes = [b'\x00' * 32] -for i in range(1, 32): - zerohashes.append(hash(zerohashes[i-1] + zerohashes[i-1])) +for layer in range(1, 32): + zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) + # Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree def calc_merkle_tree_from_leaves(values): @@ -12,17 +13,18 @@ def calc_merkle_tree_from_leaves(values): for h in range(32): if len(values) % 2 == 1: values.append(zerohashes[h]) - # print(values) - values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] + values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)] tree.append(values[::]) return tree + def get_merkle_root(values): return calc_merkle_tree_from_leaves(values)[-1][0] + def get_merkle_proof(tree, item_index): proof = [] for i in range(32): - subindex = (item_index//2**i)^1 + subindex = (item_index // 2**i) ^ 1 proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) return proof diff --git a/utils/phase0/minimal_ssz.py b/utils/phase0/minimal_ssz.py index 845de18c3..08bd68357 100644 --- a/utils/phase0/minimal_ssz.py +++ b/utils/phase0/minimal_ssz.py @@ -5,6 +5,7 @@ BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_PREFIX = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK + def SSZType(fields): class SSZObject(): def __init__(self, **kwargs): @@ -37,6 +38,7 @@ def SSZType(fields): SSZObject.fields = fields return SSZObject + class Vector(list): def __init__(self, x): list.__init__(self, x) @@ -47,9 +49,11 @@ class Vector(list): remove = clear = extend = pop = insert = append + def is_basic(typ): return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte') + def is_constant_sized(typ): if is_basic(typ): return True @@ -67,6 +71,7 @@ def is_constant_sized(typ): else: raise Exception("Type not recognized") + def coerce_to_bytes(x): if isinstance(x, str): o = x.encode('utf-8') @@ -77,6 +82,7 @@ def coerce_to_bytes(x): else: raise Exception("Expecting bytes") + def serialize_value(value, typ=None): if typ is None: typ = infer_type(value) @@ -110,28 +116,34 @@ def serialize_value(value, typ=None): print(value, typ) raise Exception("Type not recognized") + def chunkify(bytez): bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) - return [bytez[i:i+32] for i in range(0, len(bytez), 32)] + return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + def pack(values, subtype): return chunkify(b''.join([serialize_value(value, subtype) for value in values])) + def is_power_of_two(x): - return x > 0 and x & (x-1) == 0 + return x > 0 and x & (x - 1) == 0 + def merkleize(chunks): tree = chunks[::] while not is_power_of_two(len(tree)): tree.append(ZERO_CHUNK) tree = [ZERO_CHUNK] * len(tree) + tree - for i in range(len(tree)//2-1, 0, -1): - tree[i] = hash(tree[i*2] + tree[i*2+1]) + for i in range(len(tree) // 2 - 1, 0, -1): + tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) return tree[1] + def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) + def infer_type(value): if hasattr(value.__class__, 'fields'): return value.__class__ @@ -146,6 +158,7 @@ def infer_type(value): else: raise Exception("Failed to infer type") + def hash_tree_root(value, typ=None): if typ is None: typ = infer_type(value) @@ -170,6 +183,7 @@ def hash_tree_root(value, typ=None): else: raise Exception("Type not recognized") + def truncate(container): field_keys = list(container.fields.keys()) truncated_fields = { @@ -183,8 +197,10 @@ def truncate(container): } return truncated_class(**kwargs) + def signed_root(container): return hash_tree_root(truncate(container)) + def serialize(ssz_object): return getattr(ssz_object, 'serialize')() diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py index 170f647ab..92d67c45a 100644 --- a/utils/phase0/state_transition.py +++ b/utils/phase0/state_transition.py @@ -1,7 +1,7 @@ from . import spec -from typing import ( +from typing import ( # noqa: F401 Any, Callable, List, From 8fc1fe5f2075d290d55c16e3cbe18aa782fff6c2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 11:46:32 +0800 Subject: [PATCH 24/32] Use `EMPTY_SIGNATURE` and `ZERO_HASH` in the tests --- tests/phase0/helpers.py | 6 +++++- tests/phase0/test_sanity.py | 17 ++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index f7c39ffec..76206b00d 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -5,11 +5,15 @@ from py_ecc import bls import build.phase0.spec as spec from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( + # constants + EMPTY_SIGNATURE, + # SSZ AttestationData, Deposit, DepositInput, DepositData, Eth1Data, + # functions get_block_root, get_current_epoch, get_domain, @@ -92,7 +96,7 @@ def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), - proof_of_possession=b'\x00' * 96, + proof_of_possession=EMPTY_SIGNATURE, ) proof_of_possession = bls.sign( message_hash=signed_root(deposit_input), diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 8799c1ffb..8f04f316c 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -7,6 +7,9 @@ import build.phase0.spec as spec from build.phase0.utils.minimal_ssz import signed_root from build.phase0.spec import ( + # constants + EMPTY_SIGNATURE, + ZERO_HASH, # SSZ Attestation, AttestationDataAndCustodyBit, @@ -117,10 +120,10 @@ def test_proposer_slashing(state, pubkeys, privkeys): slot = spec.GENESIS_SLOT header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=b'\x00' * 32, - state_root=b'\x00' * 32, - block_body_root=b'\x00' * 32, - signature=b'\x00' * 96 + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + block_body_root=ZERO_HASH, + signature=EMPTY_SIGNATURE, ) header_2 = deepcopy(header_1) header_2.previous_block_root = b'\x02' * 32 @@ -259,7 +262,7 @@ def test_attestation(state, pubkeys, privkeys): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=b'\x00' * 96, + aggregate_signature=EMPTY_SIGNATURE, ) participants = get_attestation_participants( test_state, @@ -325,7 +328,7 @@ def test_voluntary_exit(state, pubkeys, privkeys): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=b'\x00' * 96, + signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signed_root(voluntary_exit), @@ -376,7 +379,7 @@ def test_transfer(state, pubkeys, privkeys): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=b'\x00' * 96, + signature=EMPTY_SIGNATURE, ) transfer.signature = bls.sign( message_hash=signed_root(transfer), From d645a9a0389ea571f39e901ce74274ee9d3e704f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Mar 2019 09:49:32 -0600 Subject: [PATCH 25/32] ensure run verify-state root with block Co-Authored-By: djrtwo --- utils/phase0/state_transition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/phase0/state_transition.py b/utils/phase0/state_transition.py index 92d67c45a..eefc3d409 100644 --- a/utils/phase0/state_transition.py +++ b/utils/phase0/state_transition.py @@ -97,4 +97,4 @@ def state_transition(state: BeaconState, process_epoch_transition(state) spec.advance_slot(state) if block.slot == state.slot: - process_block(state, block) + process_block(state, block, verify_state_root) From fbd0bb9226102ef4dbde606c0853d8dd74e0b60b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 09:56:04 -0600 Subject: [PATCH 26/32] withdrwaal_key uses pubkey --- tests/phase0/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 76206b00d..510361e9c 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -42,13 +42,13 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves): deposit_data_list = [] for i in range(num_validators): pubkey = pubkeys_list[i] - privkey = pubkey_to_privkey[pubkey] deposit_data = DepositData( amount=spec.MAX_DEPOSIT_AMOUNT, timestamp=deposit_timestamp, deposit_input=DepositInput( pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], proof_of_possession=proof_of_possession, ), ) @@ -95,7 +95,8 @@ def build_empty_block_for_next_slot(state): def build_deposit_data(state, pubkey, privkey, amount): deposit_input = DepositInput( pubkey=pubkey, - withdrawal_credentials=privkey.to_bytes(32, byteorder='big'), + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], proof_of_possession=EMPTY_SIGNATURE, ) proof_of_possession = bls.sign( From ff165c197bc608eb897031231467917114f6fbdb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 10:57:40 -0600 Subject: [PATCH 27/32] fix vector to be usable in deepcopy --- utils/phase0/minimal_ssz.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/utils/phase0/minimal_ssz.py b/utils/phase0/minimal_ssz.py index 08bd68357..c4828d08f 100644 --- a/utils/phase0/minimal_ssz.py +++ b/utils/phase0/minimal_ssz.py @@ -39,15 +39,22 @@ def SSZType(fields): return SSZObject -class Vector(list): - def __init__(self, x): - list.__init__(self, x) - self.length = len(x) +class Vector(): + def __init__(self, items): + self.items = items + self.length = len(items) - def append(*args): - raise Exception("Cannot change the length of a vector") + def __getitem__(self, key): + return self.items[key] - remove = clear = extend = pop = insert = append + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return self.length def is_basic(typ): From 009563b2c35c9c9fd352e8026d4b1ff4ff9d2e69 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:15:51 -0600 Subject: [PATCH 28/32] fix a few bugs in testing compute_committee --- scripts/phase0/build_spec.py | 23 ++++++++++++----------- specs/core/0_beacon-chain.md | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index ae5a5a4f2..6116f1ffe 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -37,22 +37,23 @@ Store = None code_lines += function_puller.get_lines(sourcefile) code_lines.append(""" -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: +# Monkey patch validator get committee code +_compute_committee = compute_committee +committee_cache = {} +def compute_committee(validator_indices: List[ValidatorIndex], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + param_hash = (hash_tree_root(validator_indices), seed, index, total_committees) - if param_hash in shuffling_cache: + if param_hash in committee_cache: # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] + return committee_cache[param_hash] else: # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret + ret = _compute_committee(validator_indices, seed, index, total_committees) + committee_cache[param_hash] = ret return ret diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9708ee591..27ae71c00 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -916,7 +916,8 @@ def get_crosslink_committees_at_slot(state: BeaconState, indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) committee_count = get_epoch_committee_count(len(indices)) - committees_per_slot = committee_count // EPOCH_LENGTH + committees_per_slot = committee_count // SLOTS_PER_EPOCH + offset = slot % SLOTS_PER_EPOCH return [ ( compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) From c8e9073414114cad7b276eb6623457ed8fb1bf86 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:24:36 -0600 Subject: [PATCH 29/32] define get_split_offset and squash a couple of bugs --- specs/core/0_beacon-chain.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 27ae71c00..b56a87ad5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -61,7 +61,7 @@ - [`is_active_validator`](#is_active_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - [`get_permuted_index`](#get_permuted_index) - - [`split`](#split) + - [`get_split_offset`](#get_split_offset) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`compute_committee`](#compute_committee) - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) @@ -773,18 +773,11 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: return index ``` -### `split` +### `get_split_offset` ```python -def split(values: List[Any], split_count: int) -> List[List[Any]]: - """ - Splits ``values`` into ``split_count`` pieces. - """ - list_length = len(values) - return [ - values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] - for i in range(split_count) - ] +def get_split_offset(list_length: int, split_count: int, index: int) -> int: + return (list_length * index) // split_count ``` ### `get_epoch_committee_count` @@ -918,9 +911,11 @@ def get_crosslink_committees_at_slot(state: BeaconState, committee_count = get_epoch_committee_count(len(indices)) committees_per_slot = committee_count // SLOTS_PER_EPOCH offset = slot % SLOTS_PER_EPOCH + slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT + return [ ( - compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) + compute_committee(indices, seed, committees_per_slot * offset + i, committee_count), (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) From f5826e7f1ce5e46358873514d3a1c9d173fe55aa Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:34:49 -0600 Subject: [PATCH 30/32] small lint --- README.md | 7 ++++++- specs/core/0_beacon-chain.md | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5c88daf9..8f561a9ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Ethereum 2.0 Specifications +Ethereum 2.0 Specifications +------------ [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -25,3 +26,7 @@ The following are the broad design goals for Ethereum 2.0: * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) + +# Executable spec + +The aim is to have the entirety of the Ethereum 2.0Current just the phase 0 spec is executable. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b56a87ad5..e21607010 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -920,7 +920,6 @@ def get_crosslink_committees_at_slot(state: BeaconState, ) for i in range(committees_per_slot) ] - ``` ### `get_block_root` From f7fab30772b6d70c4a2f84acc171a0432575394e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:40:24 -0600 Subject: [PATCH 31/32] minor adjustment to not repeat committe count calc --- specs/core/0_beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e21607010..9563e22bb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -908,14 +908,13 @@ def get_crosslink_committees_at_slot(state: BeaconState, shuffling_start_shard = state.current_shuffling_start_shard indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) - committee_count = get_epoch_committee_count(len(indices)) - committees_per_slot = committee_count // SLOTS_PER_EPOCH + committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH offset = slot % SLOTS_PER_EPOCH slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT return [ ( - compute_committee(indices, seed, committees_per_slot * offset + i, committee_count), + compute_committee(indices, seed, committees_per_slot * offset + i, committees_per_epoch), (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) From ba57d91e7a31b0dab328c7582dd1159b85fdf5d7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:41:27 -0600 Subject: [PATCH 32/32] undo readme commit --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 8f561a9ab..c5c88daf9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Ethereum 2.0 Specifications ------------- +# Ethereum 2.0 Specifications [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -26,7 +25,3 @@ The following are the broad design goals for Ethereum 2.0: * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) - -# Executable spec - -The aim is to have the entirety of the Ethereum 2.0Current just the phase 0 spec is executable.