From 7f39f79b2e72654920b2e12127cfdfe6ad0088c6 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 31 Jan 2019 07:55:27 -0800 Subject: [PATCH 01/96] Use 2*63 for the genesis slot --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d93cb21d7..952f0f9d0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,7 +199,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `0` | -| `GENESIS_SLOT` | `2**19` | +| `GENESIS_SLOT` | `2**63` | | `GENESIS_EPOCH` | `slot_to_epoch(GENESIS_SLOT)` | | `GENESIS_START_SHARD` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | From 5488e7b6a4d20e88072953e4ca2432f3c206e72c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 31 Jan 2019 10:12:43 -0600 Subject: [PATCH 02/96] SSZ list Merkle hashing change The current spec is arguably inconsistent, in that if a set of N values gets chunked into M chunks where M is not an exact power of 2, the chunks between M and next_power_of_2(M) are filled with SSZ_CHUNK_SIZE zero bytes each, but the last chunk is not padded, and could be arbitrarily short (eg. if the values are 4 bytes and there are 257 of them, then that gets serialized into eight chunks chunks where the first four are 64 values each, the fifth is 4 bytes corresponding to the last value, and the last three chunks are SSZ_CHUNK_SIZE zero bytes). This PR fills every chunk up to exactly SSZ_CHUNK_SIZE bytes for consistency. --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 13cc47299..2296a47c1 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -385,7 +385,7 @@ Return the hash of the serialization of the value. #### List/Vectors -First, we define some helpers and then the Merkle tree function. +First, we define some helpers and then the Merkle tree function. `zpad(input: bytes, length: int) -> bytes` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right. ```python # Merkle tree hash of a list of homogenous, non-empty items @@ -401,7 +401,7 @@ def merkle_hash(lst): items_per_chunk = SSZ_CHUNK_SIZE // len(lst[0]) # Build a list of chunks based on the number of items in the chunk - chunkz = [b''.join(lst[i:i+items_per_chunk]) for i in range(0, len(lst), items_per_chunk)] + chunkz = [zpad(b''.join(lst[i:i+items_per_chunk]), SSZ_CHUNK_SIZE) for i in range(0, len(lst), items_per_chunk)] else: # Leave large items alone chunkz = lst From 9271e6e31878682888bb051ced9a3c8920d1fb1c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 31 Jan 2019 17:47:09 -0600 Subject: [PATCH 03/96] Update specs/simple-serialize.md Co-Authored-By: vbuterin --- specs/simple-serialize.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 2296a47c1..2838c876c 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -401,7 +401,10 @@ def merkle_hash(lst): items_per_chunk = SSZ_CHUNK_SIZE // len(lst[0]) # Build a list of chunks based on the number of items in the chunk - chunkz = [zpad(b''.join(lst[i:i+items_per_chunk]), SSZ_CHUNK_SIZE) for i in range(0, len(lst), items_per_chunk)] + chunkz = [ + zpad(b''.join(lst[i:i + items_per_chunk]), SSZ_CHUNK_SIZE) + for i in range(0, len(lst), items_per_chunk) + ] else: # Leave large items alone chunkz = lst From acb432ef3ac77dde854f89949ca8b4e9320de490 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 31 Jan 2019 17:49:51 -0600 Subject: [PATCH 04/96] Update specs/simple-serialize.md Co-Authored-By: vbuterin --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 2838c876c..1820768b3 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -385,7 +385,7 @@ Return the hash of the serialization of the value. #### List/Vectors -First, we define some helpers and then the Merkle tree function. `zpad(input: bytes, length: int) -> bytes` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right. +First, we define some helpers and then the Merkle tree function. `def zpad(input: bytes, length: int) -> bytes: return input + b'\x00' * (length - len(input))` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right. ```python # Merkle tree hash of a list of homogenous, non-empty items From 8ffd9adcaffbb0176471ee5664e62e6fe2d4eaf7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 31 Jan 2019 18:03:23 -0600 Subject: [PATCH 05/96] Hash_tree_root -> hash_tree_root_internal Clarifies the distinction between "internal" hash roots (may be < 32 bytes for trivial objects) and "external" ones (zpadded to 32). --- specs/simple-serialize.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 13cc47299..d37441ebe 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -371,7 +371,12 @@ return typ(**values), item_index ### Tree Hash -The below `hash_tree_root` algorithm is defined recursively in the case of lists and containers, and it outputs a value equal to or less than 32 bytes in size. For the final output only (ie. not intermediate outputs), if the output is less than 32 bytes, right-zero-pad it to 32 bytes. The goal is collision resistance *within* each type, not between types. +The below `hash_tree_root_internal` algorithm is defined recursively in the case of lists and containers, and it outputs a value equal to or less than 32 bytes in size. For use as a "final output" (eg. for signing), use `hash_tree_root(x) = zpad(hash_tree_root_internal(x), 32)`, where `zpad` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right: + +``` +def zpad(input: bytes, length: int) -> bytes: + return input + b'\x00' * (length - len(input))` +``` Refer to [the helper function `hash`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#hash) of Phase 0 of the [Eth2.0 specs](https://github.com/ethereum/eth2.0-specs) for a definition of the hash function used below, `hash(x)`. @@ -416,20 +421,20 @@ def merkle_hash(lst): return hash(chunkz[0] + datalen) ``` -To `hash_tree_root` a list, we simply do: +To `hash_tree_root_internal` a list, we simply do: ```python -return merkle_hash([hash_tree_root(item) for item in value]) +return merkle_hash([hash_tree_root_internal(item) for item in value]) ``` -Where the inner `hash_tree_root` is a recursive application of the tree-hashing function (returning less than 32 bytes for short single values). +Where the inner `hash_tree_root_internal` is a recursive application of the tree-hashing function (returning less than 32 bytes for short single values). #### Container Recursively tree hash the values in the container in the same order as the fields, and return the hash of the concatenation of the results. ```python -return hash(b''.join([hash_tree_root(getattr(x, field)) for field in value.fields])) +return hash(b''.join([hash_tree_root_internal(getattr(x, field)) for field in value.fields])) ``` ## Implementations From dd197e46a6a2e147b1760a5f4a8730a4c56f0d18 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 Feb 2019 22:31:00 +0800 Subject: [PATCH 06/96] Fix `zpad` --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index d37441ebe..b609aa100 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -373,9 +373,9 @@ return typ(**values), item_index The below `hash_tree_root_internal` algorithm is defined recursively in the case of lists and containers, and it outputs a value equal to or less than 32 bytes in size. For use as a "final output" (eg. for signing), use `hash_tree_root(x) = zpad(hash_tree_root_internal(x), 32)`, where `zpad` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right: -``` +```python def zpad(input: bytes, length: int) -> bytes: - return input + b'\x00' * (length - len(input))` + return input + b'\x00' * (length - len(input)) ``` Refer to [the helper function `hash`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#hash) of Phase 0 of the [Eth2.0 specs](https://github.com/ethereum/eth2.0-specs) for a definition of the hash function used below, `hash(x)`. From e0867c030f65e18781fddf6295b879d958aecb88 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 Feb 2019 22:34:10 +0800 Subject: [PATCH 07/96] Fix typo --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d93cb21d7..1404f5190 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1660,7 +1660,7 @@ Below are the processing steps that happen at every slot. #### Block roots -* Let `previous_block_root` be the `tree_hash_root` of the previous beacon block processed in the chain. +* Let `previous_block_root` be the `hash_tree_root` of the previous beacon block processed in the chain. * Set `state.latest_block_roots[(state.slot - 1) % LATEST_BLOCK_ROOTS_LENGTH] = previous_block_root`. * If `state.slot % LATEST_BLOCK_ROOTS_LENGTH == 0` append `merkle_root(state.latest_block_roots)` to `state.batched_block_roots`. From 378e1ba9a681be900e86fd87e9729c657aceb8cf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 2 Feb 2019 02:06:53 +0800 Subject: [PATCH 08/96] Misc fixes of `get_next_epoch_crosslink_committees` --- specs/core/0_beacon-chain.md | 2 +- specs/validator/0_beacon-chain-validator.md | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d93cb21d7..dbc8db0e9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -833,7 +833,7 @@ def get_next_epoch_committee_count(state: BeaconState) -> int: ```python def get_crosslink_committees_at_slot(state: BeaconState, slot: SlotNumber, - registry_change=False: bool) -> List[Tuple[List[ValidatorIndex], ShardNumber]]: + registry_change: bool=False) -> List[Tuple[List[ValidatorIndex], ShardNumber]]: """ Return the list of ``(committee, shard)`` tuples for the ``slot``. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index b01e7a3af..3df3d758d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -345,15 +345,23 @@ Either (2) or (3) occurs if (1) fails. The choice between (2) and (3) is determi ```python def get_next_epoch_crosslink_committees(state: BeaconState, - validator_index: ValidatorIndex) -> List[Tuple[ValidatorIndex], ShardNumber]: + validator_index: ValidatorIndex) -> List[Tuple[List[ValidatorIndex], ShardNumber]]: current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 next_epoch_start_slot = get_epoch_start_slot(next_epoch) potential_committees = [] - for validator_registry in [False, True]: + for registry_change in [False, True]: for slot in range(next_epoch_start_slot, next_epoch_start_slot + EPOCH_LENGTH): - shard_committees = get_crosslink_committees_at_slot(state, slot, validator_registry) - selected_committees = [committee for committee in shard_committees if validator_index in committee[0]] + crosslink_committees = get_crosslink_committees_at_slot( + state, + slot, + registry_change=registry_change, + ) + selected_committees = [ + committee # type: Tuple[List[ValidatorIndex], ShardNumber] + for committee in crosslink_committees + if validator_index in committee[0] + ] if len(selected_committees) > 0: potential_assignments.append(selected_committees) break From 7e7e5e27956556afceb01970ac9e8393a4ee930f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 1 Feb 2019 17:06:04 -0800 Subject: [PATCH 09/96] Removes the source of the Vyper contract from the spec. This change allows for easier maintenance of the code and the spec by uncoupling them. Before any edit to either document resulted in having to synchronize the other. By adding a reference to the canonical repo for the code we avoid having to maintain a duplicate copy here. --- specs/core/0_beacon-chain.md | 85 +++--------------------------------- 1 file changed, 7 insertions(+), 78 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8a83149ff..ad904332b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1378,87 +1378,16 @@ When sufficiently many full deposits have been made the deposit contract emits t ### Vyper code -```python -## compiled with v0.1.0-beta.6 ## - -MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei -MAX_DEPOSIT_AMOUNT: constant(uint256) = 32000000000 # Gwei -GWEI_PER_ETH: constant(uint256) = 1000000000 # 10**9 -CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = 16384 # 2**14 -DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 -TWO_TO_POWER_OF_TREE_DEPTH: constant(uint256) = 4294967296 # 2**32 -SECONDS_PER_DAY: constant(uint256) = 86400 - -Deposit: event({deposit_root: bytes32, data: bytes[528], merkle_tree_index: bytes[8], branch: bytes32[32]}) -ChainStart: event({deposit_root: bytes32, time: bytes[8]}) - -zerohashes: bytes32[32] -branch: bytes32[32] -deposit_count: uint256 -full_deposit_count: uint256 -chainStarted: public(bool) - -@public -def __init__(): - for i in range(31): - self.zerohashes[i+1] = sha3(concat(self.zerohashes[i], self.zerohashes[i])) - self.branch[i+1] = self.zerohashes[i+1] - -@public -@constant -def get_deposit_root() -> bytes32: - root:bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 - size:uint256 = self.deposit_count - for h in range(32): - if size % 2 == 1: - root = sha3(concat(self.branch[h], root)) - else: - root = sha3(concat(root, self.zerohashes[h])) - size /= 2 - return root - -@payable -@public -def deposit(deposit_input: bytes[512]): - assert msg.value >= as_wei_value(MIN_DEPOSIT_AMOUNT, "gwei") - assert msg.value <= as_wei_value(MAX_DEPOSIT_AMOUNT, "gwei") - - index: uint256 = self.deposit_count - deposit_amount: bytes[8] = slice(concat("", convert(msg.value / GWEI_PER_ETH, bytes32)), start=24, len=8) - deposit_timestamp: bytes[8] = slice(concat("", convert(block.timestamp, bytes32)), start=24, len=8) - deposit_data: bytes[528] = concat(deposit_amount, deposit_timestamp, deposit_input) - merkle_tree_index: bytes[8] = slice(concat("", convert(index, bytes32)), start=24, len=8) - - # add deposit to merkle tree - i: int128 = 0 - power_of_two: uint256 = 2 - for _ in range(32): - if (index+1) % power_of_two != 0: - break - i += 1 - power_of_two *= 2 - value:bytes32 = sha3(deposit_data) - for j in range(32): - if j < i: - value = sha3(concat(self.branch[j], value)) - self.branch[i] = value - - self.deposit_count += 1 - - new_deposit_root:bytes32 = self.get_deposit_root() - log.Deposit(new_deposit_root, deposit_data, merkle_tree_index, self.branch) - - if msg.value == as_wei_value(MAX_DEPOSIT_AMOUNT, "gwei"): - self.full_deposit_count += 1 - if self.full_deposit_count == CHAIN_START_FULL_DEPOSIT_THRESHOLD: - timestamp_day_boundary: uint256 = as_unitless_number(block.timestamp) - as_unitless_number(block.timestamp) % SECONDS_PER_DAY + SECONDS_PER_DAY - chainstart_time: bytes[8] = slice(concat("", convert(timestamp_day_boundary, bytes32)), start=24, len=8) - log.ChainStart(new_deposit_root, chainstart_time) - self.chainStarted = True -``` +The source for the Vyper contract lives in a separate repository at [https://github.com/ethereum/deposit_contract](https://github.com/ethereum/deposit_contract). Note: to save ~10x on gas this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in python tested for correctness. +For convenience, we provide the interface to the contract here: + +* `__init__()`: initializes the contract +* `get_deposit_root() -> bytes32`: returns the current root of the deposit tree +* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. + ## On startup A valid block with slot `GENESIS_SLOT` (a "genesis block") has the following values. Other validity rules (e.g. requiring a signature) do not apply. From 8d82ee8ce7ea1d3d3bb166683cd497e115cc852b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 1 Feb 2019 21:02:09 -0800 Subject: [PATCH 10/96] do not mix in epoch to seed in get_shuffling. add epoch to generate_seed --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8a83149ff..1ee3596e8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -778,7 +778,6 @@ def get_shuffling(seed: Bytes32, committees_per_epoch = get_epoch_committee_count(len(active_validator_indices)) # Shuffle - seed = xor(seed, int_to_bytes32(epoch)) shuffled_active_validator_indices = shuffle(active_validator_indices, seed) # Split the shuffled list into committees_per_epoch pieces @@ -942,7 +941,8 @@ def generate_seed(state: BeaconState, """ return hash( get_randao_mix(state, epoch - SEED_LOOKAHEAD) + - get_active_index_root(state, epoch) + get_active_index_root(state, epoch) + + int_to_bytes32(epoch) ) ``` From a781eb5a4f22e2b4b74c509f44d6c210f5f6bb87 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sat, 2 Feb 2019 06:09:45 +0100 Subject: [PATCH 11/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8a83149ff..6f10ad1f9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -821,6 +821,9 @@ def get_current_epoch_committee_count(state: BeaconState) -> int: ```python def get_next_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the next epoch of the given ``state``. + """ next_active_validators = get_active_validator_indices( state.validator_registry, get_current_epoch(state) + 1, From f3d47e26fc30f87f09b008f328545c81ad403bdf Mon Sep 17 00:00:00 2001 From: Dean Eigenmann Date: Sat, 2 Feb 2019 13:49:52 +0100 Subject: [PATCH 12/96] `shard` -> `attestation.data.shard` --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d93cb21d7..fa963f8a8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1731,7 +1731,7 @@ For each `attestation` in `block.body.attestations`: * Verify that `attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY < attestation.data.slot + EPOCH_LENGTH`. * Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch if attestation.data.slot >= get_epoch_start_slot(get_current_epoch(state)) else state.previous_justified_epoch`. * Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, get_epoch_start_slot(attestation.data.justified_epoch))`. -* Verify that either `attestation.data.latest_crosslink_root` or `attestation.data.shard_block_root` equals `state.latest_crosslinks[shard].shard_block_root`. +* Verify that either `attestation.data.latest_crosslink_root` or `attestation.data.shard_block_root` equals `state.latest_crosslinks[attestation.data.shard].shard_block_root`. * Verify bitfields and aggregate signature: ```python From 650f4a20dfa25c6aa85a64dfe21aca5de75f8eeb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 2 Feb 2019 13:16:39 -0800 Subject: [PATCH 13/96] clarify eth1 that there will be at most 1 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index caaac363f..1c7573453 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1688,7 +1688,7 @@ Below are the processing steps that happen at every `block`. #### Eth1 data -* If `block.eth1_data` equals `eth1_data_vote.eth1_data` for some `eth1_data_vote` in `state.eth1_data_votes`, set `eth1_data_vote.vote_count += 1`. +* If there exists an `eth1_data_vote` in `states.eth1_data_votes` for which `eth1_data_vote.eth1_data == block.eth1_data` (there will be at most one), set `eth1_data_vote.vote_count += 1`. * Otherwise, append to `state.eth1_data_votes` a new `Eth1DataVote(eth1_data=block.eth1_data, vote_count=1)`. #### Operations From 2a32e7f6653959da1f21e2812426cb9d87baf24a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 2 Feb 2019 16:41:59 -0800 Subject: [PATCH 14/96] convert int_to_bytes to little endian --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index caaac363f..f67afd8a6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -710,8 +710,8 @@ def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: if remaining == 1: break - # Read 3-bytes of `source` as a 24-bit big-endian integer. - sample_from_source = int.from_bytes(source[position:position + rand_bytes], 'big') + # Read 3-bytes of `source` as a 24-bit little-endian integer. + sample_from_source = int.from_bytes(source[position:position + rand_bytes], 'little') # Sample values greater than or equal to `sample_max` will cause # modulo bias when mapped into the `remaining` range. @@ -1015,7 +1015,7 @@ def is_power_of_two(value: int) -> bool: ### `int_to_bytes1`, `int_to_bytes2`, ... -`int_to_bytes1(x): return x.to_bytes(1, 'big')`, `int_to_bytes2(x): return x.to_bytes(2, 'big')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96. +`int_to_bytes1(x): return x.to_bytes(1, 'little')`, `int_to_bytes2(x): return x.to_bytes(2, 'little')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96. ### `get_effective_balance` From e4c4c04e52bde69edf85d36726cbfa3b9aa9fae7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 3 Feb 2019 10:36:21 +0100 Subject: [PATCH 15/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 48075c8d4..5c1d158b5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -54,6 +54,7 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`slot_to_epoch`](#slot_to_epoch) + - [`get_previous_epoch`](#get_previous_epoch) - [`get_current_epoch`](#get_current_epoch) - [`get_epoch_start_slot`](#get_epoch_start_slot) - [`is_active_validator`](#is_active_validator) @@ -639,6 +640,18 @@ def slot_to_epoch(slot: SlotNumber) -> EpochNumber: return slot // EPOCH_LENGTH ``` +### `get_previous_epoch` + +```python +def get_previous_epoch(state: BeaconState) -> EpochNumber: + """` + Return the previous epoch of the given ``state``. + """ + if slot_to_epoch(state.slot) > GENESIS_EPOCH: + return slot_to_epoch(state.slot) - 1 + return slot_to_epoch(state.slot) +``` + ### `get_current_epoch` ```python @@ -844,7 +857,7 @@ def get_crosslink_committees_at_slot(state: BeaconState, """ epoch = slot_to_epoch(slot) current_epoch = get_current_epoch(state) - previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else current_epoch + previous_epoch = get_previous_epoch(state) next_epoch = current_epoch + 1 assert previous_epoch <= epoch <= next_epoch @@ -1827,7 +1840,7 @@ The steps below happen when `(state.slot + 1) % EPOCH_LENGTH == 0`. #### Helpers * Let `current_epoch = get_current_epoch(state)`. -* Let `previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else current_epoch`. +* Let `previous_epoch = get_previous_epoch(state)`. * Let `next_epoch = current_epoch + 1`. [Validators](#dfn-Validator) attesting during the current epoch: From 969896b0a83bed0e1f412556ff4027f85983fb01 Mon Sep 17 00:00:00 2001 From: Dean Eigenmann Date: Sun, 3 Feb 2019 11:14:02 +0100 Subject: [PATCH 16/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 48075c8d4..b6c4d51f3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -76,6 +76,7 @@ - [`is_power_of_two`](#is_power_of_two) - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) - [`get_effective_balance`](#get_effective_balance) + - [`total_balance`](#total_balance) - [`get_fork_version`](#get_fork_version) - [`get_domain`](#get_domain) - [`get_bitfield_bit`](#get_bitfield_bit) @@ -1027,6 +1028,16 @@ def get_effective_balance(state: State, index: ValidatorIndex) -> Gwei: return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) ``` +### `total_balance` + +```python +def total_balance(state: State, validators: [ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of an array of validators. + """ + sum([get_effective_balance(state, i) for i in validators]) +``` + ### `get_fork_version` ```python @@ -1832,31 +1843,31 @@ The steps below happen when `(state.slot + 1) % EPOCH_LENGTH == 0`. [Validators](#dfn-Validator) attesting during the current epoch: -* Let `current_total_balance = sum([get_effective_balance(state, i) for i in get_active_validator_indices(state.validator_registry, current_epoch)])`. +* Let `current_total_balance = total_balance(state, get_active_validator_indices(state.validator_registry, current_epoch))`. * Let `current_epoch_attestations = [a for a in state.latest_attestations if current_epoch == slot_to_epoch(a.data.slot)]`. (Note: this is the set of attestations of slots in the epoch `current_epoch`, _not_ attestations that got included in the chain during the epoch `current_epoch`.) * Validators justifying the epoch boundary block at the start of the current epoch: * Let `current_epoch_boundary_attestations = [a for a in current_epoch_attestations if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(current_epoch)) and a.data.justified_epoch == state.justified_epoch]`. * Let `current_epoch_boundary_attester_indices` be the union of the [validator](#dfn-validator) index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in current_epoch_boundary_attestations]`. - * Let `current_epoch_boundary_attesting_balance = sum([get_effective_balance(state, i) for i in current_epoch_boundary_attester_indices])`. + * Let `current_epoch_boundary_attesting_balance = total_balance(state, current_epoch_boundary_attester_indices)`. [Validators](#dfn-Validator) attesting during the previous epoch: -* Let `previous_total_balance = sum([get_effective_balance(state, i) for i in get_active_validator_indices(state.validator_registry, previous_epoch)])`. +* Let `previous_total_balance = total_balance(state, get_active_validator_indices(state.validator_registry, previous_epoch))`. * Validators that made an attestation during the previous epoch: * Let `previous_epoch_attestations = [a for a in state.latest_attestations if previous_epoch == slot_to_epoch(a.data.slot)]`. * Let `previous_epoch_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_attestations]`. * Validators targeting the previous justified slot: * Let `previous_epoch_justified_attestations = [a for a in current_epoch_attestations + previous_epoch_attestations if a.data.justified_epoch == state.previous_justified_epoch]`. * Let `previous_epoch_justified_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_justified_attestations]`. - * Let `previous_epoch_justified_attesting_balance = sum([get_effective_balance(state, i) for i in previous_epoch_justified_attester_indices])`. + * Let `previous_epoch_justified_attesting_balance = total_balance(state, previous_epoch_justified_attester_indices)`. * Validators justifying the epoch boundary block at the start of the previous epoch: * Let `previous_epoch_boundary_attestations = [a for a in previous_epoch_justified_attestations if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(previous_epoch))]`. * Let `previous_epoch_boundary_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_boundary_attestations]`. - * Let `previous_epoch_boundary_attesting_balance = sum([get_effective_balance(state, i) for i in previous_epoch_boundary_attester_indices])`. + * Let `previous_epoch_boundary_attesting_balance = total_balance(state, previous_epoch_boundary_attester_indices)`. * Validators attesting to the expected beacon chain head during the previous epoch: * Let `previous_epoch_head_attestations = [a for a in previous_epoch_attestations if a.data.beacon_block_root == get_block_root(state, a.data.slot)]`. * Let `previous_epoch_head_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_head_attestations]`. - * Let `previous_epoch_head_attesting_balance = sum([get_effective_balance(state, i) for i in previous_epoch_head_attester_indices])`. + * Let `previous_epoch_head_attesting_balance = total_balance(state, previous_epoch_head_attester_indices)`. **Note**: `previous_total_balance` and `previous_epoch_boundary_attesting_balance` balance might be marginally different than the actual balances during previous epoch transition. Due to the tight bound on validator churn each epoch and small per-epoch rewards/penalties, the potential balance difference is very low and only marginally affects consensus safety. @@ -1864,10 +1875,9 @@ For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_s * Let `shard_block_root` be `state.latest_crosslinks[shard].shard_block_root` * Let `attesting_validator_indices(crosslink_committee, shard_block_root)` be the union of the [validator](#dfn-validator) index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in current_epoch_attestations + previous_epoch_attestations if a.data.shard == shard and a.data.shard_block_root == shard_block_root]`. -* Let `winning_root(crosslink_committee)` be equal to the value of `shard_block_root` such that `sum([get_effective_balance(state, i) for i in attesting_validator_indices(crosslink_committee, shard_block_root)])` is maximized (ties broken by favoring lower `shard_block_root` values). +* Let `winning_root(crosslink_committee)` be equal to the value of `shard_block_root` such that `total_balance(state, attesting_validator_indices(crosslink_committee, shard_block_root))` is maximized (ties broken by favoring lower `shard_block_root` values). * Let `attesting_validators(crosslink_committee)` be equal to `attesting_validator_indices(crosslink_committee, winning_root(crosslink_committee))` for convenience. -* Let `total_attesting_balance(crosslink_committee) = sum([get_effective_balance(state, i) for i in attesting_validators(crosslink_committee)])`. -* Let `total_balance(crosslink_committee) = sum([get_effective_balance(state, i) for i in crosslink_committee])`. +* Let `total_attesting_balance(crosslink_committee) = total_balance(state, attesting_validators(crosslink_committee))`. Define the following helpers to process attestation inclusion rewards and inclusion distance reward/penalty. For every attestation `a` in `previous_epoch_attestations`: @@ -1997,7 +2007,7 @@ def update_validator_registry(state: BeaconState) -> None: # The active validators active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) # The total effective balance of active validators - total_balance = sum([get_effective_balance(state, i) for i in active_validator_indices]) + total_balance = total_balance(state, active_validator_indices) # The maximum balance churn in Gwei (for deposits and exits separately) max_balance_churn = max( From 8e16d122be108af8b92f31ca5a014bb9ae99fa97 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 3 Feb 2019 11:42:12 +0100 Subject: [PATCH 17/96] Update specs/core/0_beacon-chain.md Co-Authored-By: decanus --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b6c4d51f3..e838b23c1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1031,7 +1031,7 @@ def get_effective_balance(state: State, index: ValidatorIndex) -> Gwei: ### `total_balance` ```python -def total_balance(state: State, validators: [ValidatorIndex]) -> Gwei: +def total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of an array of validators. """ From 5c56751cc9cf51e37329c33a5a9c0bd4d87bbfbf Mon Sep 17 00:00:00 2001 From: Dean Eigenmann Date: Sun, 3 Feb 2019 11:43:33 +0100 Subject: [PATCH 18/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e838b23c1..71426790e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -76,7 +76,7 @@ - [`is_power_of_two`](#is_power_of_two) - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) - [`get_effective_balance`](#get_effective_balance) - - [`total_balance`](#total_balance) + - [`get_total_balance`](#get_total_balance) - [`get_fork_version`](#get_fork_version) - [`get_domain`](#get_domain) - [`get_bitfield_bit`](#get_bitfield_bit) @@ -1028,10 +1028,10 @@ def get_effective_balance(state: State, index: ValidatorIndex) -> Gwei: return min(state.validator_balances[index], MAX_DEPOSIT_AMOUNT) ``` -### `total_balance` +### `get_total_balance` ```python -def total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of an array of validators. """ @@ -1843,31 +1843,31 @@ The steps below happen when `(state.slot + 1) % EPOCH_LENGTH == 0`. [Validators](#dfn-Validator) attesting during the current epoch: -* Let `current_total_balance = total_balance(state, get_active_validator_indices(state.validator_registry, current_epoch))`. +* Let `current_total_balance = get_total_balance(state, get_active_validator_indices(state.validator_registry, current_epoch))`. * Let `current_epoch_attestations = [a for a in state.latest_attestations if current_epoch == slot_to_epoch(a.data.slot)]`. (Note: this is the set of attestations of slots in the epoch `current_epoch`, _not_ attestations that got included in the chain during the epoch `current_epoch`.) * Validators justifying the epoch boundary block at the start of the current epoch: * Let `current_epoch_boundary_attestations = [a for a in current_epoch_attestations if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(current_epoch)) and a.data.justified_epoch == state.justified_epoch]`. * Let `current_epoch_boundary_attester_indices` be the union of the [validator](#dfn-validator) index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in current_epoch_boundary_attestations]`. - * Let `current_epoch_boundary_attesting_balance = total_balance(state, current_epoch_boundary_attester_indices)`. + * Let `current_epoch_boundary_attesting_balance = get_total_balance(state, current_epoch_boundary_attester_indices)`. [Validators](#dfn-Validator) attesting during the previous epoch: -* Let `previous_total_balance = total_balance(state, get_active_validator_indices(state.validator_registry, previous_epoch))`. +* Let `previous_total_balance = get_total_balance(state, get_active_validator_indices(state.validator_registry, previous_epoch))`. * Validators that made an attestation during the previous epoch: * Let `previous_epoch_attestations = [a for a in state.latest_attestations if previous_epoch == slot_to_epoch(a.data.slot)]`. * Let `previous_epoch_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_attestations]`. * Validators targeting the previous justified slot: * Let `previous_epoch_justified_attestations = [a for a in current_epoch_attestations + previous_epoch_attestations if a.data.justified_epoch == state.previous_justified_epoch]`. * Let `previous_epoch_justified_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_justified_attestations]`. - * Let `previous_epoch_justified_attesting_balance = total_balance(state, previous_epoch_justified_attester_indices)`. + * Let `previous_epoch_justified_attesting_balance = get_total_balance(state, previous_epoch_justified_attester_indices)`. * Validators justifying the epoch boundary block at the start of the previous epoch: * Let `previous_epoch_boundary_attestations = [a for a in previous_epoch_justified_attestations if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(previous_epoch))]`. * Let `previous_epoch_boundary_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_boundary_attestations]`. - * Let `previous_epoch_boundary_attesting_balance = total_balance(state, previous_epoch_boundary_attester_indices)`. + * Let `previous_epoch_boundary_attesting_balance = get_total_balance(state, previous_epoch_boundary_attester_indices)`. * Validators attesting to the expected beacon chain head during the previous epoch: * Let `previous_epoch_head_attestations = [a for a in previous_epoch_attestations if a.data.beacon_block_root == get_block_root(state, a.data.slot)]`. * Let `previous_epoch_head_attester_indices` be the union of the validator index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in previous_epoch_head_attestations]`. - * Let `previous_epoch_head_attesting_balance = total_balance(state, previous_epoch_head_attester_indices)`. + * Let `previous_epoch_head_attesting_balance = get_total_balance(state, previous_epoch_head_attester_indices)`. **Note**: `previous_total_balance` and `previous_epoch_boundary_attesting_balance` balance might be marginally different than the actual balances during previous epoch transition. Due to the tight bound on validator churn each epoch and small per-epoch rewards/penalties, the potential balance difference is very low and only marginally affects consensus safety. @@ -1875,9 +1875,9 @@ For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_s * Let `shard_block_root` be `state.latest_crosslinks[shard].shard_block_root` * Let `attesting_validator_indices(crosslink_committee, shard_block_root)` be the union of the [validator](#dfn-validator) index sets given by `[get_attestation_participants(state, a.data, a.aggregation_bitfield) for a in current_epoch_attestations + previous_epoch_attestations if a.data.shard == shard and a.data.shard_block_root == shard_block_root]`. -* Let `winning_root(crosslink_committee)` be equal to the value of `shard_block_root` such that `total_balance(state, attesting_validator_indices(crosslink_committee, shard_block_root))` is maximized (ties broken by favoring lower `shard_block_root` values). +* Let `winning_root(crosslink_committee)` be equal to the value of `shard_block_root` such that `get_total_balance(state, attesting_validator_indices(crosslink_committee, shard_block_root))` is maximized (ties broken by favoring lower `shard_block_root` values). * Let `attesting_validators(crosslink_committee)` be equal to `attesting_validator_indices(crosslink_committee, winning_root(crosslink_committee))` for convenience. -* Let `total_attesting_balance(crosslink_committee) = total_balance(state, attesting_validators(crosslink_committee))`. +* Let `total_attesting_balance(crosslink_committee) = get_total_balance(state, attesting_validators(crosslink_committee))`. Define the following helpers to process attestation inclusion rewards and inclusion distance reward/penalty. For every attestation `a` in `previous_epoch_attestations`: @@ -1916,7 +1916,7 @@ Finally, update the following: For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch))`, let `crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)`. For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot`, compute: -* Set `state.latest_crosslinks[shard] = Crosslink(epoch=current_epoch, shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * total_balance(crosslink_committee)`. +* Set `state.latest_crosslinks[shard] = Crosslink(epoch=current_epoch, shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * get_total_balance(crosslink_committee)`. #### Rewards and penalties @@ -1964,7 +1964,7 @@ For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_s * Let `crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)`. * For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot`: - * If `index in attesting_validators(crosslink_committee)`, `state.validator_balances[index] += base_reward(state, index) * total_attesting_balance(crosslink_committee) // total_balance(crosslink_committee))`. + * If `index in attesting_validators(crosslink_committee)`, `state.validator_balances[index] += base_reward(state, index) * total_attesting_balance(crosslink_committee) // get_total_balance(state, crosslink_committee))`. * If `index not in attesting_validators(crosslink_committee)`, `state.validator_balances[index] -= base_reward(state, index)`. #### Ejections @@ -2007,7 +2007,7 @@ def update_validator_registry(state: BeaconState) -> None: # The active validators active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) # The total effective balance of active validators - total_balance = total_balance(state, active_validator_indices) + total_balance = get_total_balance(state, active_validator_indices) # The maximum balance churn in Gwei (for deposits and exits separately) max_balance_churn = max( From 3a6da9839a64a7edc951bef541b47a65554bfed8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 3 Feb 2019 14:27:09 +0100 Subject: [PATCH 19/96] Added a note for underflow. Thanks Hsiao Wei! : ) Co-Authored-By: terenc3t --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5c1d158b5..b8815388d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -646,6 +646,7 @@ def slot_to_epoch(slot: SlotNumber) -> EpochNumber: def get_previous_epoch(state: BeaconState) -> EpochNumber: """` Return the previous epoch of the given ``state``. + If the current epoch is ``GENESIS_EPOCH``, return ``GENESIS_EPOCH``. """ if slot_to_epoch(state.slot) > GENESIS_EPOCH: return slot_to_epoch(state.slot) - 1 From 197fa188936fb21289e86a7e4c88ff5b61815e93 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 3 Feb 2019 14:26:07 -0600 Subject: [PATCH 20/96] Zpad is already in #543 --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 1820768b3..6976e9cbe 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -385,7 +385,7 @@ Return the hash of the serialization of the value. #### List/Vectors -First, we define some helpers and then the Merkle tree function. `def zpad(input: bytes, length: int) -> bytes: return input + b'\x00' * (length - len(input))` is a helper that extends the given `bytes` value to the desired `length` by adding zero bytes on the right. +First, we define the Merkle tree function. ```python # Merkle tree hash of a list of homogenous, non-empty items From 8c5868cbe8603c61b54126599aa1fcdbf7243f52 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sun, 3 Feb 2019 15:20:49 -0800 Subject: [PATCH 21/96] Extend `deposit` API with note about bounds on acceptable values --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ad904332b..02804ab6f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1386,7 +1386,7 @@ For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. +* `deposit(bytes[512])`: adds a deposit instance to the deposit tree, incorporating the input argument and the value transferred in the given call. Note: the amount of value transferred *must* be within `MIN_DEPOSIT_AMOUNT` and `MAX_DEPOSIT_AMOUNT`, inclusive. Each of these constants are specified in units of Gwei. ## On startup From 8755dc34d51ac88c3cd35b76cc8ef9544abb294c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Sun, 3 Feb 2019 15:22:03 -0800 Subject: [PATCH 22/96] Add direct link to validator registration contract --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 02804ab6f..016d773e4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1378,7 +1378,7 @@ When sufficiently many full deposits have been made the deposit contract emits t ### Vyper code -The source for the Vyper contract lives in a separate repository at [https://github.com/ethereum/deposit_contract](https://github.com/ethereum/deposit_contract). +The source for the Vyper contract lives in a [separate repository](https://github.com/ethereum/deposit_contract) at [https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py](https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py). Note: to save ~10x on gas this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in python tested for correctness. From 147ee2f991f513c0962f5b610f431f86b92a0f88 Mon Sep 17 00:00:00 2001 From: Taras Bobrovytsky Date: Mon, 4 Feb 2019 01:49:16 -0600 Subject: [PATCH 23/96] Fix get_total_balance() --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 788876689..cc78c8b71 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1049,7 +1049,7 @@ def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> G """ Return the combined effective balance of an array of validators. """ - sum([get_effective_balance(state, i) for i in validators]) + return sum([get_effective_balance(state, i) for i in validators]) ``` ### `get_fork_version` From 867dea3473733946d0eb51b8e3132eb3ccacfb2c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Feb 2019 19:16:10 +0800 Subject: [PATCH 24/96] Fix `verify_bitfield` --- specs/core/0_beacon-chain.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cc78c8b71..ed5d1b3b3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1099,9 +1099,11 @@ def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: if len(bitfield) != (committee_size + 7) // 8: return False - for i in range(committee_size + 1, committee_size - committee_size % 8 + 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False + # Check if `bitfield` has padding zeros + if committee_size % 8 != 0: + for i in range(committee_size, committee_size - committee_size % 8 + 8): + if get_bitfield_bit(bitfield, i) == 0b1: + return False return True ``` From e5788f5751a9a0be34d02ed72ab26ad3dddd12f6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Feb 2019 01:11:00 +0800 Subject: [PATCH 25/96] Fix typo: it's `bls_verify_multiple` in `verify_slashable_attestation` (#574) --- specs/core/0_beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cc78c8b71..ccc9bf3af 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1137,7 +1137,7 @@ def verify_slashable_attestation(state: BeaconState, slashable_attestation: Slas else: custody_bit_1_indices.append(validator_index) - return bls_verify( + return bls_verify_multiple( pubkeys=[ bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), @@ -1147,11 +1147,7 @@ def verify_slashable_attestation(state: BeaconState, slashable_attestation: Slas hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), ], signature=slashable_attestation.aggregate_signature, - domain=get_domain( - state.fork, - slot_to_epoch(vote_data.data.slot), - DOMAIN_ATTESTATION, - ), + domain=get_domain(state.fork, slot_to_epoch(vote_data.data.slot), DOMAIN_ATTESTATION), ) ``` From 1d76ad65ec4dcde8666d57344057d9060ed30b7f Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 5 Feb 2019 11:49:52 -0800 Subject: [PATCH 26/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ed5d1b3b3..6b41e6c8a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1100,10 +1100,9 @@ def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: return False # Check if `bitfield` has padding zeros - if committee_size % 8 != 0: - for i in range(committee_size, committee_size - committee_size % 8 + 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False + for i in range(committee_size, len(bitfield) * 8): + if get_bitfield_bit(bitfield, i) == 0b1: + return False return True ``` From 180c8a0e728cf2f3c21e2ea10a0d439efa9fd792 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 5 Feb 2019 11:51:06 -0800 Subject: [PATCH 27/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6b41e6c8a..77196312b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1099,7 +1099,7 @@ def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: if len(bitfield) != (committee_size + 7) // 8: return False - # Check if `bitfield` has padding zeros + # Check `bitfield` is padded with zero bits only for i in range(committee_size, len(bitfield) * 8): if get_bitfield_bit(bitfield, i) == 0b1: return False From c58410e6ce9904c6619cd925b64fbd04c00b9a89 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 06:48:46 -0600 Subject: [PATCH 28/96] Introduce swap-or-not shuffle See #563 for discussion. --- specs/core/0_beacon-chain.md | 61 +++++++++--------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 44b88d126..3cb1b459f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -697,53 +697,22 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber ```python def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: - """ - Return the shuffled ``values`` with ``seed`` as entropy. - """ - values_count = len(values) + indices = list(range(len(values))) + for round in range(90): + hashvalues = b''.join([ + hash(seed + round.to_bytes(1, 'big') + i.to_bytes(4, 'big')) + for i in range((n + 255) // 256) + ]) + pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'big')), 'big') % n + + def permute(pos): + flip = (pivot - pos) % n + maxpos = max(pos, flip) + bit = (hashvalues[maxpos // 8] >> (maxpos % 8)) % 2 + return flip if bit else pos - # Entropy is consumed from the seed in 3-byte (24 bit) chunks. - rand_bytes = 3 - # The highest possible result of the RNG. - rand_max = 2 ** (rand_bytes * 8) - 1 - - # The range of the RNG places an upper-bound on the size of the list that - # may be shuffled. It is a logic error to supply an oversized list. - assert values_count < rand_max - - output = [x for x in values] - source = seed - index = 0 - while index < values_count - 1: - # Re-hash the `source` to obtain a new pattern of bytes. - source = hash(source) - # Iterate through the `source` bytes in 3-byte chunks. - for position in range(0, 32 - (32 % rand_bytes), rand_bytes): - # Determine the number of indices remaining in `values` and exit - # once the last index is reached. - remaining = values_count - index - if remaining == 1: - break - - # Read 3-bytes of `source` as a 24-bit big-endian integer. - sample_from_source = int.from_bytes(source[position:position + rand_bytes], 'big') - - # Sample values greater than or equal to `sample_max` will cause - # modulo bias when mapped into the `remaining` range. - sample_max = rand_max - rand_max % remaining - - # Perform a swap if the consumed entropy will not cause modulo bias. - if sample_from_source < sample_max: - # Select a replacement index for the current index. - replacement_position = (sample_from_source % remaining) + index - # Swap the current index with the replacement index. - output[index], output[replacement_position] = output[replacement_position], output[index] - index += 1 - else: - # The sample causes modulo bias. A new sample should be read. - pass - - return output + indices = [permute(v) for v in indices] + return [v[index] for index in indices] ``` ### `split` From 8f37c5c0f8804d37dde2e513e6eeaa91b8b7879e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 10:34:19 -0600 Subject: [PATCH 29/96] Update fork choice rule (#571) --- specs/core/0_beacon-chain.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 44b88d126..9015a4970 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -13,6 +13,7 @@ - [Constants](#constants) - [Misc](#misc) - [Deposit contract](#deposit-contract) + - [Gwei values](#gwei-values) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) @@ -179,7 +180,6 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be | - | - | :-: | | `SHARD_COUNT` | `2**10` (= 1,024) | shards | | `TARGET_COMMITTEE_SIZE` | `2**7` (= 128) | [validators](#dfn-validator) | -| `EJECTION_BALANCE` | `2**4 * 1e9` (= 16,000,000,000) | Gwei | | `MAX_BALANCE_CHURN_QUOTIENT` | `2**5` (= 32) | - | | `BEACON_CHAIN_SHARD_NUMBER` | `2**64 - 1` | - | | `MAX_INDICES_PER_SLASHABLE_VOTE` | `2**12` (= 4,096) | votes | @@ -189,12 +189,19 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be ### Deposit contract +| Name | Value | +| - | - | +| `DEPOSIT_CONTRACT_ADDRESS` | **TBD** | +| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | + +### Gwei values + | Name | Value | Unit | | - | - | :-: | -| `DEPOSIT_CONTRACT_ADDRESS` | **TBD** | -| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | - | | `MIN_DEPOSIT_AMOUNT` | `2**0 * 1e9` (= 1,000,000,000) | Gwei | | `MAX_DEPOSIT_AMOUNT` | `2**5 * 1e9` (= 32,000,000,000) | Gwei | +| `FORK_CHOICE_BALANCE_INCREMENT` | `2**0 * 1e9` (= 1,000,000,000) | Gwei | +| `EJECTION_BALANCE` | `2**4 * 1e9` (= 16,000,000,000) | Gwei | ### Initial values @@ -1555,8 +1562,8 @@ def get_ancestor(store: Store, block: BeaconBlock, slot: SlotNumber) -> BeaconBl return get_ancestor(store, store.get_parent(block), slot) ``` -* Let `get_latest_attestation(store: Store, validator: Validator) -> Attestation` be the attestation with the highest slot number in `store` from `validator`. If several such attestations exist, use the one the [validator](#dfn-validator) `v` observed first. -* Let `get_latest_attestation_target(store: Store, validator: Validator) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, validator)`. +* Let `get_latest_attestation(store: Store, validator_index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `validator_index`. If several such attestations exist, use the one the [validator](#dfn-validator) `v` observed first. +* Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, validator_index)`. * Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` returns the child blocks of the given `block`. * Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. * The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. @@ -1567,21 +1574,18 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. """ validators = start_state.validator_registry - active_validators = [ - validators[i] - for i in get_active_validator_indices(validators, start_state.slot) - ] + active_validator_indices = get_active_validator_indices(validators, start_state.slot) attestation_targets = [ - get_latest_attestation_target(store, validator) - for validator in active_validators + (validator_index, get_latest_attestation_target(store, validator_index)) + for validator_index in active_validator_indices ] def get_vote_count(block: BeaconBlock) -> int: - return len([ - target - for target in attestation_targets + return sum( + get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT + for validator_index, target in attestation_targets if get_ancestor(store, target, block.slot) == block - ]) + ) head = start_block while 1: From 37b41a2ce645b644a56c473cba2ded3a49689139 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Feb 2019 18:33:11 -0600 Subject: [PATCH 30/96] Update specs/core/0_beacon-chain.md Co-Authored-By: vbuterin --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3cb1b459f..5471d76a1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -704,7 +704,6 @@ def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: for i in range((n + 255) // 256) ]) pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'big')), 'big') % n - def permute(pos): flip = (pivot - pos) % n maxpos = max(pos, flip) From 4ec721f3b7d4459c3db601d9a3bd53511a1dfd31 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 6 Feb 2019 18:33:22 -0600 Subject: [PATCH 31/96] Update specs/core/0_beacon-chain.md Co-Authored-By: vbuterin --- specs/core/0_beacon-chain.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5471d76a1..2e453b061 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -697,6 +697,13 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber ```python def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: + """ + Return the shuffled ``values`` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ indices = list(range(len(values))) for round in range(90): hashvalues = b''.join([ From 6a5b7540da501c976d1e0ec76b44725bb96f7f37 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Feb 2019 18:33:29 -0600 Subject: [PATCH 32/96] Update specs/core/0_beacon-chain.md Co-Authored-By: vbuterin --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2e453b061..1426bb4e8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -718,7 +718,7 @@ def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: return flip if bit else pos indices = [permute(v) for v in indices] - return [v[index] for index in indices] + return [values[index] for index in indices] ``` ### `split` From 47b00f38dda4336ada69b166f8f288b840c628d3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 18:34:05 -0600 Subject: [PATCH 33/96] n -> len(values) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1426bb4e8..60bacba63 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -708,7 +708,7 @@ def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: for round in range(90): hashvalues = b''.join([ hash(seed + round.to_bytes(1, 'big') + i.to_bytes(4, 'big')) - for i in range((n + 255) // 256) + for i in range((len(values) + 255) // 256) ]) pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'big')), 'big') % n def permute(pos): From 8597cbef1fb28f6f54eedfded5b5f633b4928694 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 20:32:56 -0600 Subject: [PATCH 34/96] Updated phase 1: shard block structure --- specs/core/1_shard-data-chains.md | 58 +++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index cabe2934e..f3324bbbd 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -16,10 +16,11 @@ Ethereum 2.0 consists of a central beacon chain along with `SHARD_COUNT` shard c Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md#constants) in addition to the following: -| Constant | Value | Unit | Approximation | -|------------------------|-----------------|-------|---------------| -| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | -| `SHARD_BLOCK_SIZE` | 2**14 (= 16384) | bytes | | +| Constant | Value | Unit | Approximation | +|-----------------------------|-----------------|--------|---------------| +| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | +| `SHARD_BLOCK_SIZE` | 2**14 (= 16384) | bytes | | +| `PROPOSAL_RESHUFFLE_PERIOD` | 2**11 (= 2048) | epochs | 9 days | ### Flags, domains, etc. @@ -40,12 +41,10 @@ A `ShardBlock` object has the following fields: 'slot': 'uint64', # What shard is it on 'shard_id': 'uint64', - # Parent block's hash of root + # Parent block's root 'parent_root': 'hash32', # Beacon chain block 'beacon_chain_ref': 'hash32', - # Depth of the Merkle tree - 'data_tree_depth': 'uint8', # Merkle root of data 'data_root': 'hash32' # State root (placeholder for now) @@ -67,22 +66,43 @@ For a block on a shard to be processed by a node, the following conditions must To validate a block header on shard `shard_id`, compute as follows: -* Verify that `beacon_chain_ref` is the hash of a block in the beacon chain with slot less than or equal to `slot`. Verify that `beacon_chain_ref` is equal to or a descendant of the `beacon_chain_ref` specified in the `ShardBlock` pointed to by `parent_root`. -* Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. Let `validators` be `[validators[i] for i in state.current_persistent_committees[shard_id]]`. -* Assert `len(participation_bitfield) == ceil_div8(len(validators))` +* Verify that `beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. +* Verify that `beacon_chain_ref` is equal to or a descendant of the `beacon_chain_ref` specified in the `ShardBlock` pointed to by `parent_root`. +* Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, slot, shard_id)`. +* Assert `verify_bitfield(participation_bitfield, len(persistent_committee))` * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_id) + int_to_bytes8(slot)) % len(validators)`. Let `msg` be the block but with the `block.signature` set to `[0, 0]`. Verify that `BLSVerify(pub=validators[proposer_index].pubkey, msg=hash(msg), sig=block.signature, domain=get_domain(state, slot, SHARD_PROPOSER_DOMAIN))` passes. -* Generate the `group_public_key` by adding the public keys of all the validators for whom the corresponding position in the bitfield is set to 1. Verify that `BLSVerify(pub=group_public_key, msg=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. +* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, msg=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. -### Block Merklization helper +We define the helper `get_proposal_committee` as follows: ```python -def merkle_root(block_body): - assert len(block_body) == SHARD_BLOCK_SIZE - chunks = SHARD_BLOCK_SIZE // SHARD_CHUNK_SIZE - o = [0] * chunks + [block_body[i * SHARD_CHUNK_SIZE: (i+1) * SHARD_CHUNK_SIZE] for i in range(chunks)] - for i in range(chunks-1, 0, -1): - o[i] = hash(o[i*2] + o[i*2+1]) - return o[1] +def get_proposal_committee(seed: Bytes32, + validators: List[Validator], + shard: int, + epoch: EpochNumber) -> List[ValidatorIndex]: + + earlier_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD * 2 + earlier_committee = split(shuffle( + get_active_validator_indices(validators, earlier_committee_start), + generate_seed(state, earlier_committee_start) + ), SHARD_COUNT)[shard] + + later_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD + later_committee = split(shuffle( + get_active_validator_indices(validators, earlier_committee_start), + generate_seed(state, earlier_committee_start) + ), SHARD_COUNT)[shard] + + def get_switchover_epoch(index): + return ( + int.from_bytes(hash(generate_seed(state, earlier_committee_start) + bytes3(index)), 'little') % + PROPOSAL_RESHUFFLE_PERIOD + ) + + return ( + [i for i in earlier_committee if epoch % PROPOSAL_RESHUFFLE_PERIOD < get_switchover_epoch(i)] + + [i for i in later_committee if epoch % PROPOSAL_RESHUFFLE_PERIOD >= get_switchover_epoch(i)] + ) ``` ### Verifying shard block data From b3db7b03942d2c666952fb94fe01774d15089105 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 6 Feb 2019 20:32:05 -0800 Subject: [PATCH 35/96] big to little in shuffle --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60bacba63..641ad02ca 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -707,10 +707,10 @@ def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: indices = list(range(len(values))) for round in range(90): hashvalues = b''.join([ - hash(seed + round.to_bytes(1, 'big') + i.to_bytes(4, 'big')) + hash(seed + round.to_bytes(1, 'little') + i.to_bytes(4, 'little')) for i in range((len(values) + 255) // 256) ]) - pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'big')), 'big') % n + pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'little')), 'little') % n def permute(pos): flip = (pivot - pos) % n maxpos = max(pos, flip) From 65255e53c46ac8d2ee520f2845e9869193ae783c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 23:29:24 -0600 Subject: [PATCH 36/96] shuffle -> get_permuted_index --- specs/core/0_beacon-chain.md | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 641ad02ca..dd58482b3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -59,7 +59,7 @@ - [`get_epoch_start_slot`](#get_epoch_start_slot) - [`is_active_validator`](#is_active_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`shuffle`](#shuffle) + - [`get_permuted_index`](#get_permuted_index) - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`get_shuffling`](#get_shuffling) @@ -693,32 +693,27 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] ``` -### `shuffle` +### `get_permuted_index` ```python -def shuffle(values: List[Any], seed: Bytes32) -> List[Any]: +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: """ - Return the shuffled ``values`` with ``seed`` as entropy. + Return a pseudorandom permutation of `0...list_size-1` with ``seed`` as entropy. Utilizes 'swap or not' shuffling found in https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf See the 'generalized domain' algorithm on page 3. """ - indices = list(range(len(values))) for round in range(90): - hashvalues = b''.join([ - hash(seed + round.to_bytes(1, 'little') + i.to_bytes(4, 'little')) - for i in range((len(values) + 255) // 256) - ]) - pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'little')), 'little') % n - def permute(pos): - flip = (pivot - pos) % n - maxpos = max(pos, flip) - bit = (hashvalues[maxpos // 8] >> (maxpos % 8)) % 2 - return flip if bit else pos - - indices = [permute(v) for v in indices] - return [values[index] for index in indices] + pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'little')), 'little') % list_size + + flip = (pivot - index) % list_size + hash_pos = max(index, flip) + h = hash(seed + round.to_bytes(1, 'little') + (hash_pos // 256).to_bytes(4, 'little')) + byte = h[(hash_pos % 256) // 8] + bit = (byte >> (hash_pos % 8)) % 2 + index = flip if bit else index + return index ``` ### `split` @@ -768,7 +763,10 @@ def get_shuffling(seed: Bytes32, committees_per_epoch = get_epoch_committee_count(len(active_validator_indices)) # Shuffle - shuffled_active_validator_indices = shuffle(active_validator_indices, seed) + shuffled_active_validator_indices = [ + active_validator_indices[get_permuted_index(i, len(active_validator_indices), seed)] + for i in active_validator_indices + ] # Split the shuffled list into committees_per_epoch pieces return split(shuffled_active_validator_indices, committees_per_epoch) From 92514716fb77de05637a560ec3f62a0ed5209dc6 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 6 Feb 2019 23:32:20 -0600 Subject: [PATCH 37/96] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dd58482b3..2c0fce8c5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -698,7 +698,7 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber ```python def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: """ - Return a pseudorandom permutation of `0...list_size-1` with ``seed`` as entropy. + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. Utilizes 'swap or not' shuffling found in https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf From 70e482be28a10ca0d37de34fb3a0b5cd94108ad3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 7 Feb 2019 19:14:58 +0800 Subject: [PATCH 38/96] Add vbuterin's optimization and some formatting --- specs/core/0_beacon-chain.md | 44 +++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2c0fce8c5..3da520300 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -59,7 +59,7 @@ - [`get_epoch_start_slot`](#get_epoch_start_slot) - [`is_active_validator`](#is_active_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`get_permuted_index`](#get_permuted_index) + - [`shuffle`](#shuffle) - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`get_shuffling`](#get_shuffling) @@ -693,27 +693,38 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] ``` -### `get_permuted_index` +### `shuffle` ```python -def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: +def shuffle(list_size: int, seed: Bytes32) -> List[int]: """ - Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. - + Return shuffled indices in a pseudorandom permutation `0...list_size-1` with ``seed`` as entropy. + Utilizes 'swap or not' shuffling found in https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf See the 'generalized domain' algorithm on page 3. """ + indices = list(range(list_size)) + power_of_two_numbers = [1, 2, 4, 8, 16, 32, 64, 128] for round in range(90): - pivot = int.from_bytes(hash(seed + round.to_bytes(1, 'little')), 'little') % list_size - - flip = (pivot - index) % list_size - hash_pos = max(index, flip) - h = hash(seed + round.to_bytes(1, 'little') + (hash_pos // 256).to_bytes(4, 'little')) - byte = h[(hash_pos % 256) // 8] - bit = (byte >> (hash_pos % 8)) % 2 - index = flip if bit else index - return index + hash_bytes = b''.join([ + hash(seed + int_to_bytes1(round) + int_to_bytes4(i)) + for i in range((list_size + 255) // 256) + ]) + + pivot = int.from_bytes(hash(seed + int_to_bytes1(round)), 'little') % list_size + for i in range(list_size): + flip = (pivot - indices[i]) % list_size + hash_position = indices[i] if indices[i] > flip else flip + byte = hash_bytes[hash_position // 8] + mask = power_of_two_numbers[hash_position % 8] + if byte & mask: + indices[i] = flip + else: + # not swap + pass + + return indices ``` ### `split` @@ -763,9 +774,10 @@ def get_shuffling(seed: Bytes32, committees_per_epoch = get_epoch_committee_count(len(active_validator_indices)) # Shuffle + shuffled_indices = shuffle(len(active_validator_indices), seed) shuffled_active_validator_indices = [ - active_validator_indices[get_permuted_index(i, len(active_validator_indices), seed)] - for i in active_validator_indices + active_validator_indices[i] + for i in shuffle(len(active_validator_indices), seed) ] # Split the shuffled list into committees_per_epoch pieces From aa9f9fc9be337ad072e855e5a4132735eea7636b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 7 Feb 2019 19:18:39 +0800 Subject: [PATCH 39/96] amend --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3da520300..6a4ac6ce5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -774,7 +774,6 @@ def get_shuffling(seed: Bytes32, committees_per_epoch = get_epoch_committee_count(len(active_validator_indices)) # Shuffle - shuffled_indices = shuffle(len(active_validator_indices), seed) shuffled_active_validator_indices = [ active_validator_indices[i] for i in shuffle(len(active_validator_indices), seed) From 3f3472087af00ddb54210feb7f82cc7f6e22d966 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 Feb 2019 10:37:01 -0800 Subject: [PATCH 40/96] change message to message_hash in bls spec addresses #572 --- specs/bls_signature.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index fd9bae58e..b0490b7ae 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -69,10 +69,10 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message: bytes32, domain: uint64) -> [uint384]: +def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: # Initial candidate x coordinate - x_re = int.from_bytes(hash(message + bytes8(domain) + b'\x01'), 'big') - x_im = int.from_bytes(hash(message + bytes8(domain) + b'\x02'), 'big') + x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') + x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big') x_coordinate = Fq2([x_re, x_im]) # x = x_re + i * x_im # Test candidate y coordinates until a one is found @@ -128,17 +128,17 @@ g = Fq2([g_x, g_y]) ### `bls_verify` -Let `bls_verify(pubkey: Bytes48, message: Bytes32, signature: Bytes96, domain: uint64) -> bool`: +Let `bls_verify(pubkey: Bytes48, message_hash: Bytes32, signature: Bytes96, domain: uint64) -> bool`: * Verify that `pubkey` is a valid G1 point. * Verify that `signature` is a valid G2 point. -* Verify that `e(pubkey, hash_to_G2(message, domain)) == e(g, signature)`. +* Verify that `e(pubkey, hash_to_G2(message_hash, domain)) == e(g, signature)`. ### `bls_verify_multiple` -Let `bls_verify_multiple(pubkeys: List[Bytes48], messages: List[Bytes32], signature: Bytes96, domain: uint64) -> bool`: +Let `bls_verify_multiple(pubkeys: List[Bytes48], message_hashes: List[Bytes32], signature: Bytes96, domain: uint64) -> bool`: * Verify that each `pubkey` in `pubkeys` is a valid G1 point. * Verify that `signature` is a valid G2 point. -* Verify that `len(pubkeys)` equals `len(messages)` and denote the length `L`. -* Verify that `e(pubkeys[0], hash_to_G2(messages[0], domain)) * ... * e(pubkeys[L-1], hash_to_G2(messages[L-1], domain)) == e(g, signature)`. +* Verify that `len(pubkeys)` equals `len(message_hashes)` and denote the length `L`. +* Verify that `e(pubkeys[0], hash_to_G2(message_hashes[0], domain)) * ... * e(pubkeys[L-1], hash_to_G2(message_hashes[L-1], domain)) == e(g, signature)`. From d4901be1984e44a61d410bda9d4592caa25cf470 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 Feb 2019 11:48:36 -0700 Subject: [PATCH 41/96] get_next_epoch_committee_assignments returns slot and is_proposer --- specs/validator/0_beacon-chain-validator.md | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 3df3d758d..093f29f5b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -344,12 +344,22 @@ Either (2) or (3) occurs if (1) fails. The choice between (2) and (3) is determi `get_crosslink_committees_at_slot` is designed to be able to query slots in the next epoch. When querying slots in the next epoch there are two options -- with and without a `registry_change` -- which is the optional third parameter of the function. The following helper can be used to get the potential crosslink committees in the next epoch for a given `validator_index`. This function returns a list of 2 shard committee tuples. ```python -def get_next_epoch_crosslink_committees(state: BeaconState, - validator_index: ValidatorIndex) -> List[Tuple[List[ValidatorIndex], ShardNumber]]: +def get_next_epoch_committee_assignments( + state: BeaconState, + validator_index: ValidatorIndex) -> List[Tuple[List[ValidatorIndex], ShardNumber, SlotNumber, bool]]: + """ + Return a list of the two possible committee assignments for ``validator_index`` at the next epoch. + Possible committee ``assignment`` is of the form (List[ValidatorIndex], ShardNumber, SlotNumber, bool). + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the shard to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + * ``assignment[3]`` is a bool signalling if the validator is expected to propose + a beacon block at the assigned slot. + """ current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 next_epoch_start_slot = get_epoch_start_slot(next_epoch) - potential_committees = [] + potential_assignments = [] for registry_change in [False, True]: for slot in range(next_epoch_start_slot, next_epoch_start_slot + EPOCH_LENGTH): crosslink_committees = get_crosslink_committees_at_slot( @@ -363,13 +373,19 @@ def get_next_epoch_crosslink_committees(state: BeaconState, if validator_index in committee[0] ] if len(selected_committees) > 0: - potential_assignments.append(selected_committees) + assignment = selected_committees[0] + assignment += (slot,) + first_committee_at_slot = crosslink_committees[0] + is_proposer = first_committee_at_slot[slot % len(first_committee_at_slot)] == validator_index + assignment += (is_proposer,) + + potential_assignments.append(assignment) break return potential_assignments ``` -`get_next_epoch_crosslink_committees` should be called at the beginning of each epoch to plan for the next epoch. A validator should always plan for both values of `registry_change` as a possibility unless the validator can concretely eliminate one of the options. Planning for a future shuffling involves noting at which slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). +`get_next_epoch_committee_assignments` should be called at the beginning of each epoch to plan for the next epoch. A validator should always plan for both values of `registry_change` as a possibility unless the validator can concretely eliminate one of the options. Planning for a future shuffling involves noting at which slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). ## How to avoid slashing From e4f5efadb79dcfe6f83b71a3bf339fd11b224f0a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 8 Feb 2019 04:03:13 +0800 Subject: [PATCH 42/96] Fix `first_committee_at_slot` --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 093f29f5b..4d8d5eadf 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -368,14 +368,14 @@ def get_next_epoch_committee_assignments( registry_change=registry_change, ) selected_committees = [ - committee # type: Tuple[List[ValidatorIndex], ShardNumber] + committee # Tuple[List[ValidatorIndex], ShardNumber] for committee in crosslink_committees if validator_index in committee[0] ] if len(selected_committees) > 0: assignment = selected_committees[0] assignment += (slot,) - first_committee_at_slot = crosslink_committees[0] + first_committee_at_slot = crosslink_committees[0][0] # List[ValidatorIndex] is_proposer = first_committee_at_slot[slot % len(first_committee_at_slot)] == validator_index assignment += (is_proposer,) From 859bf6248467eb8f5a23c28ac721f605185ccfaf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 8 Feb 2019 05:08:25 +0800 Subject: [PATCH 43/96] Revert and refactor --- specs/core/0_beacon-chain.md | 41 ++++++++++++++---------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6a4ac6ce5..38d233b23 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -59,7 +59,7 @@ - [`get_epoch_start_slot`](#get_epoch_start_slot) - [`is_active_validator`](#is_active_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - - [`shuffle`](#shuffle) + - [`get_permuted_index`](#get_permuted_index) - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`get_shuffling`](#get_shuffling) @@ -693,38 +693,29 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] ``` -### `shuffle` +### `get_permuted_index` ```python -def shuffle(list_size: int, seed: Bytes32) -> List[int]: +def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: int=90) -> List[int]: """ - Return shuffled indices in a pseudorandom permutation `0...list_size-1` with ``seed`` as entropy. + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. + + Note that ``round_count`` is ``90`` in protocol and parameterized for the shuffling tests. Utilizes 'swap or not' shuffling found in https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf See the 'generalized domain' algorithm on page 3. """ - indices = list(range(list_size)) - power_of_two_numbers = [1, 2, 4, 8, 16, 32, 64, 128] - for round in range(90): - hash_bytes = b''.join([ - hash(seed + int_to_bytes1(round) + int_to_bytes4(i)) - for i in range((list_size + 255) // 256) - ]) - + for round in range(round_count): pivot = int.from_bytes(hash(seed + int_to_bytes1(round)), 'little') % list_size - for i in range(list_size): - flip = (pivot - indices[i]) % list_size - hash_position = indices[i] if indices[i] > flip else flip - byte = hash_bytes[hash_position // 8] - mask = power_of_two_numbers[hash_position % 8] - if byte & mask: - indices[i] = flip - else: - # not swap - pass + flip = (pivot - index) % list_size + position = max(index, flip) + source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index - return indices + return index ``` ### `split` @@ -775,8 +766,8 @@ def get_shuffling(seed: Bytes32, # Shuffle shuffled_active_validator_indices = [ - active_validator_indices[i] - for i in shuffle(len(active_validator_indices), seed) + active_validator_indices[get_permuted_index(i, len(active_validator_indices), seed)] + for i in active_validator_indices ] # Split the shuffled list into committees_per_epoch pieces From cf7ebe9ad31d1b4831f8a26353ac70c1fd60d27c Mon Sep 17 00:00:00 2001 From: mratsim Date: Thu, 7 Feb 2019 22:09:41 +0100 Subject: [PATCH 44/96] `message` to `message_hash` in the rest of the spec (followup https://github.com/ethereum/eth2.0-specs/pull/580) --- specs/core/0_beacon-chain.md | 280 ++++++++++---------- specs/validator/0_beacon-chain-validator.md | 102 +++---- 2 files changed, 191 insertions(+), 191 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b0433ce04..e6cf019f4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -6,139 +6,139 @@ - [Ethereum 2.0 Phase 0 -- The Beacon Chain](#ethereum-20-phase-0----the-beacon-chain) - - [Table of contents](#table-of-contents) - - [Introduction](#introduction) - - [Notation](#notation) - - [Terminology](#terminology) - - [Constants](#constants) - - [Misc](#misc) - - [Deposit contract](#deposit-contract) - - [Gwei values](#gwei-values) - - [Initial values](#initial-values) - - [Time parameters](#time-parameters) - - [State list lengths](#state-list-lengths) - - [Reward and penalty quotients](#reward-and-penalty-quotients) - - [Status flags](#status-flags) - - [Max operations per block](#max-operations-per-block) - - [Signature domains](#signature-domains) - - [Data structures](#data-structures) - - [Beacon chain operations](#beacon-chain-operations) - - [Proposer slashings](#proposer-slashings) - - [`ProposerSlashing`](#proposerslashing) - - [Attester slashings](#attester-slashings) - - [`AttesterSlashing`](#attesterslashing) - - [`SlashableAttestation`](#slashableattestation) - - [Attestations](#attestations) - - [`Attestation`](#attestation) - - [`AttestationData`](#attestationdata) - - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - - [Deposits](#deposits) - - [`Deposit`](#deposit) - - [`DepositData`](#depositdata) - - [`DepositInput`](#depositinput) - - [Exits](#exits) - - [`Exit`](#exit) - - [Beacon chain blocks](#beacon-chain-blocks) - - [`BeaconBlock`](#beaconblock) - - [`BeaconBlockBody`](#beaconblockbody) - - [`ProposalSignedData`](#proposalsigneddata) - - [Beacon chain state](#beacon-chain-state) - - [`BeaconState`](#beaconstate) - - [`Validator`](#validator) - - [`Crosslink`](#crosslink) - - [`PendingAttestation`](#pendingattestation) - - [`Fork`](#fork) - - [`Eth1Data`](#eth1data) - - [`Eth1DataVote`](#eth1datavote) - - [Custom Types](#custom-types) - - [Helper functions](#helper-functions) - - [`hash`](#hash) - - [`hash_tree_root`](#hash_tree_root) - - [`slot_to_epoch`](#slot_to_epoch) - - [`get_previous_epoch`](#get_previous_epoch) - - [`get_current_epoch`](#get_current_epoch) - - [`get_epoch_start_slot`](#get_epoch_start_slot) - - [`is_active_validator`](#is_active_validator) - - [`get_active_validator_indices`](#get_active_validator_indices) - - [`shuffle`](#shuffle) - - [`split`](#split) - - [`get_epoch_committee_count`](#get_epoch_committee_count) - - [`get_shuffling`](#get_shuffling) - - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) - - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count) - - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count) - - [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot) - - [`get_block_root`](#get_block_root) - - [`get_randao_mix`](#get_randao_mix) - - [`get_active_index_root`](#get_active_index_root) - - [`generate_seed`](#generate_seed) - - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - - [`merkle_root`](#merkle_root) - - [`get_attestation_participants`](#get_attestation_participants) - - [`is_power_of_two`](#is_power_of_two) - - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) - - [`get_effective_balance`](#get_effective_balance) - - [`get_total_balance`](#get_total_balance) - - [`get_fork_version`](#get_fork_version) - - [`get_domain`](#get_domain) - - [`get_bitfield_bit`](#get_bitfield_bit) - - [`verify_bitfield`](#verify_bitfield) - - [`verify_slashable_attestation`](#verify_slashable_attestation) - - [`is_double_vote`](#is_double_vote) - - [`is_surround_vote`](#is_surround_vote) - - [`integer_squareroot`](#integer_squareroot) - - [`get_entry_exit_effect_epoch`](#get_entry_exit_effect_epoch) - - [`bls_verify`](#bls_verify) - - [`bls_verify_multiple`](#bls_verify_multiple) - - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) - - [`validate_proof_of_possession`](#validate_proof_of_possession) - - [`process_deposit`](#process_deposit) - - [Routines for updating validator status](#routines-for-updating-validator-status) - - [`activate_validator`](#activate_validator) - - [`initiate_validator_exit`](#initiate_validator_exit) - - [`exit_validator`](#exit_validator) - - [`penalize_validator`](#penalize_validator) - - [`prepare_validator_for_withdrawal`](#prepare_validator_for_withdrawal) - - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) - - [Deposit arguments](#deposit-arguments) - - [Withdrawal credentials](#withdrawal-credentials) - - [`Deposit` logs](#deposit-logs) - - [`ChainStart` log](#chainstart-log) - - [Vyper code](#vyper-code) - - [On startup](#on-startup) - - [Beacon chain processing](#beacon-chain-processing) - - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) - - [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [Per-slot processing](#per-slot-processing) - - [Slot](#slot) - - [Block roots](#block-roots) - - [Per-block processing](#per-block-processing) - - [Slot](#slot-1) - - [Proposer signature](#proposer-signature) - - [RANDAO](#randao) - - [Eth1 data](#eth1-data) - - [Operations](#operations) - - [Proposer slashings](#proposer-slashings-1) - - [Attester slashings](#attester-slashings-1) - - [Attestations](#attestations-1) - - [Deposits](#deposits-1) - - [Exits](#exits-1) - - [Per-epoch processing](#per-epoch-processing) - - [Helpers](#helpers) - - [Eth1 data](#eth1-data-1) - - [Justification](#justification) - - [Crosslinks](#crosslinks) - - [Rewards and penalties](#rewards-and-penalties) - - [Justification and finalization](#justification-and-finalization) - - [Attestation inclusion](#attestation-inclusion) - - [Crosslinks](#crosslinks-1) - - [Ejections](#ejections) - - [Validator registry and shuffling seed data](#validator-registry-and-shuffling-seed-data) - - [Final updates](#final-updates) - - [State root verification](#state-root-verification) + - [Table of contents](#table-of-contents) + - [Introduction](#introduction) + - [Notation](#notation) + - [Terminology](#terminology) + - [Constants](#constants) + - [Misc](#misc) + - [Deposit contract](#deposit-contract) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [State list lengths](#state-list-lengths) + - [Reward and penalty quotients](#reward-and-penalty-quotients) + - [Status flags](#status-flags) + - [Max operations per block](#max-operations-per-block) + - [Signature domains](#signature-domains) + - [Data structures](#data-structures) + - [Beacon chain operations](#beacon-chain-operations) + - [Proposer slashings](#proposer-slashings) + - [`ProposerSlashing`](#proposerslashing) + - [Attester slashings](#attester-slashings) + - [`AttesterSlashing`](#attesterslashing) + - [`SlashableAttestation`](#slashableattestation) + - [Attestations](#attestations) + - [`Attestation`](#attestation) + - [`AttestationData`](#attestationdata) + - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) + - [Deposits](#deposits) + - [`Deposit`](#deposit) + - [`DepositData`](#depositdata) + - [`DepositInput`](#depositinput) + - [Exits](#exits) + - [`Exit`](#exit) + - [Beacon chain blocks](#beacon-chain-blocks) + - [`BeaconBlock`](#beaconblock) + - [`BeaconBlockBody`](#beaconblockbody) + - [`ProposalSignedData`](#proposalsigneddata) + - [Beacon chain state](#beacon-chain-state) + - [`BeaconState`](#beaconstate) + - [`Validator`](#validator) + - [`Crosslink`](#crosslink) + - [`PendingAttestation`](#pendingattestation) + - [`Fork`](#fork) + - [`Eth1Data`](#eth1data) + - [`Eth1DataVote`](#eth1datavote) + - [Custom Types](#custom-types) + - [Helper functions](#helper-functions) + - [`hash`](#hash) + - [`hash_tree_root`](#hashtreeroot) + - [`slot_to_epoch`](#slottoepoch) + - [`get_previous_epoch`](#getpreviousepoch) + - [`get_current_epoch`](#getcurrentepoch) + - [`get_epoch_start_slot`](#getepochstartslot) + - [`is_active_validator`](#isactivevalidator) + - [`get_active_validator_indices`](#getactivevalidatorindices) + - [`shuffle`](#shuffle) + - [`split`](#split) + - [`get_epoch_committee_count`](#getepochcommitteecount) + - [`get_shuffling`](#getshuffling) + - [`get_previous_epoch_committee_count`](#getpreviousepochcommitteecount) + - [`get_current_epoch_committee_count`](#getcurrentepochcommitteecount) + - [`get_next_epoch_committee_count`](#getnextepochcommitteecount) + - [`get_crosslink_committees_at_slot`](#getcrosslinkcommitteesatslot) + - [`get_block_root`](#getblockroot) + - [`get_randao_mix`](#getrandaomix) + - [`get_active_index_root`](#getactiveindexroot) + - [`generate_seed`](#generateseed) + - [`get_beacon_proposer_index`](#getbeaconproposerindex) + - [`merkle_root`](#merkleroot) + - [`get_attestation_participants`](#getattestationparticipants) + - [`is_power_of_two`](#ispoweroftwo) + - [`int_to_bytes1`, `int_to_bytes2`, ...](#inttobytes1-inttobytes2) + - [`get_effective_balance`](#geteffectivebalance) + - [`get_total_balance`](#gettotalbalance) + - [`get_fork_version`](#getforkversion) + - [`get_domain`](#getdomain) + - [`get_bitfield_bit`](#getbitfieldbit) + - [`verify_bitfield`](#verifybitfield) + - [`verify_slashable_attestation`](#verifyslashableattestation) + - [`is_double_vote`](#isdoublevote) + - [`is_surround_vote`](#issurroundvote) + - [`integer_squareroot`](#integersquareroot) + - [`get_entry_exit_effect_epoch`](#getentryexiteffectepoch) + - [`bls_verify`](#blsverify) + - [`bls_verify_multiple`](#blsverifymultiple) + - [`bls_aggregate_pubkeys`](#blsaggregatepubkeys) + - [`validate_proof_of_possession`](#validateproofofpossession) + - [`process_deposit`](#processdeposit) + - [Routines for updating validator status](#routines-for-updating-validator-status) + - [`activate_validator`](#activatevalidator) + - [`initiate_validator_exit`](#initiatevalidatorexit) + - [`exit_validator`](#exitvalidator) + - [`penalize_validator`](#penalizevalidator) + - [`prepare_validator_for_withdrawal`](#preparevalidatorforwithdrawal) + - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) + - [Deposit arguments](#deposit-arguments) + - [Withdrawal credentials](#withdrawal-credentials) + - [`Deposit` logs](#deposit-logs) + - [`ChainStart` log](#chainstart-log) + - [Vyper code](#vyper-code) + - [On startup](#on-startup) + - [Beacon chain processing](#beacon-chain-processing) + - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) + - [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Per-slot processing](#per-slot-processing) + - [Slot](#slot) + - [Block roots](#block-roots) + - [Per-block processing](#per-block-processing) + - [Slot](#slot-1) + - [Proposer signature](#proposer-signature) + - [RANDAO](#randao) + - [Eth1 data](#eth1-data) + - [Operations](#operations) + - [Proposer slashings](#proposer-slashings-1) + - [Attester slashings](#attester-slashings-1) + - [Attestations](#attestations-1) + - [Deposits](#deposits-1) + - [Exits](#exits-1) + - [Per-epoch processing](#per-epoch-processing) + - [Helpers](#helpers) + - [Eth1 data](#eth1-data-1) + - [Justification](#justification) + - [Crosslinks](#crosslinks) + - [Rewards and penalties](#rewards-and-penalties) + - [Justification and finalization](#justification-and-finalization) + - [Attestation inclusion](#attestation-inclusion) + - [Crosslinks](#crosslinks-1) + - [Ejections](#ejections) + - [Validator registry and shuffling seed data](#validator-registry-and-shuffling-seed-data) + - [Final updates](#final-updates) + - [State root verification](#state-root-verification) - [References](#references) - - [Normative](#normative) - - [Informative](#informative) + - [Normative](#normative) + - [Informative](#informative) - [Copyright](#copyright) @@ -1052,7 +1052,7 @@ def get_effective_balance(state: State, index: ValidatorIndex) -> Gwei: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of an array of validators. """ @@ -1150,7 +1150,7 @@ def verify_slashable_attestation(state: BeaconState, slashable_attestation: Slas bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), ], - messages=[ + message_hashes=[ hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), ], @@ -1245,7 +1245,7 @@ def validate_proof_of_possession(state: BeaconState, return bls_verify( pubkey=pubkey, - message=hash_tree_root(proof_of_possession_data), + message_hash=hash_tree_root(proof_of_possession_data), signature=proof_of_possession, domain=get_domain( state.fork, @@ -1633,12 +1633,12 @@ Below are the processing steps that happen at every `block`. * Let `block_without_signature_root` be the `hash_tree_root` of `block` where `block.signature` is set to `EMPTY_SIGNATURE`. * Let `proposal_root = hash_tree_root(ProposalSignedData(state.slot, BEACON_CHAIN_SHARD_NUMBER, block_without_signature_root))`. -* Verify that `bls_verify(pubkey=state.validator_registry[get_beacon_proposer_index(state, state.slot)].pubkey, message=proposal_root, signature=block.signature, domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_PROPOSAL))`. +* Verify that `bls_verify(pubkey=state.validator_registry[get_beacon_proposer_index(state, state.slot)].pubkey, message_hash=proposal_root, signature=block.signature, domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_PROPOSAL))`. #### RANDAO * Let `proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]`. -* Verify that `bls_verify(pubkey=proposer.pubkey, message=int_to_bytes32(get_current_epoch(state)), signature=block.randao_reveal, domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. +* Verify that `bls_verify(pubkey=proposer.pubkey, message_hash=int_to_bytes32(get_current_epoch(state)), signature=block.randao_reveal, domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO))`. * Set `state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = xor(get_randao_mix(state, get_current_epoch(state)), hash(block.randao_reveal))`. #### Eth1 data @@ -1659,8 +1659,8 @@ For each `proposer_slashing` in `block.body.proposer_slashings`: * Verify that `proposer_slashing.proposal_data_1.shard == proposer_slashing.proposal_data_2.shard`. * Verify that `proposer_slashing.proposal_data_1.block_root != proposer_slashing.proposal_data_2.block_root`. * Verify that `proposer.penalized_epoch > get_current_epoch(state)`. -* Verify that `bls_verify(pubkey=proposer.pubkey, message=hash_tree_root(proposer_slashing.proposal_data_1), signature=proposer_slashing.proposal_signature_1, domain=get_domain(state.fork, slot_to_epoch(proposer_slashing.proposal_data_1.slot), DOMAIN_PROPOSAL))`. -* Verify that `bls_verify(pubkey=proposer.pubkey, message=hash_tree_root(proposer_slashing.proposal_data_2), signature=proposer_slashing.proposal_signature_2, domain=get_domain(state.fork, slot_to_epoch(proposer_slashing.proposal_data_2.slot), DOMAIN_PROPOSAL))`. +* Verify that `bls_verify(pubkey=proposer.pubkey, message_hash=hash_tree_root(proposer_slashing.proposal_data_1), signature=proposer_slashing.proposal_signature_1, domain=get_domain(state.fork, slot_to_epoch(proposer_slashing.proposal_data_1.slot), DOMAIN_PROPOSAL))`. +* Verify that `bls_verify(pubkey=proposer.pubkey, message_hash=hash_tree_root(proposer_slashing.proposal_data_2), signature=proposer_slashing.proposal_signature_2, domain=get_domain(state.fork, slot_to_epoch(proposer_slashing.proposal_data_2.slot), DOMAIN_PROPOSAL))`. * Run `penalize_validator(state, proposer_slashing.proposer_index)`. ##### Attester slashings @@ -1772,7 +1772,7 @@ For each `exit` in `block.body.exits`: * Verify that `validator.exit_epoch > get_entry_exit_effect_epoch(get_current_epoch(state))`. * Verify that `get_current_epoch(state) >= exit.epoch`. * Let `exit_message = hash_tree_root(Exit(epoch=exit.epoch, validator_index=exit.validator_index, signature=EMPTY_SIGNATURE))`. -* Verify that `bls_verify(pubkey=validator.pubkey, message=exit_message, signature=exit.signature, domain=get_domain(state.fork, exit.epoch, DOMAIN_EXIT))`. +* Verify that `bls_verify(pubkey=validator.pubkey, message_hash=exit_message, signature=exit.signature, domain=get_domain(state.fork, exit.epoch, DOMAIN_EXIT))`. * Run `initiate_validator_exit(state, exit.validator_index)`. ### Per-epoch processing diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 4d8d5eadf..b8db2117c 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -7,53 +7,53 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers - [Ethereum 2.0 Phase 0 -- Honest Validator](#ethereum-20-phase-0----honest-validator) - - [Table of Contents](#table-of-contents) - - [Introduction](#introduction) - - [Prerequisites](#prerequisites) - - [Constants](#constants) - - [Misc](#misc) - - [Becoming a validator](#becoming-a-validator) - - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [BLS withdrawal key](#bls-withdrawal-key) - - [Submit deposit](#submit-deposit) - - [Process deposit](#process-deposit) - - [Validator index](#validator-index) - - [Activation](#activation) - - [Beacon chain responsibilities](#beacon-chain-responsibilities) - - [Block proposal](#block-proposal) - - [Block header](#block-header) - - [Slot](#slot) - - [Parent root](#parent-root) - - [State root](#state-root) - - [Randao reveal](#randao-reveal) - - [Eth1 Data](#eth1-data) - - [Signature](#signature) - - [Block body](#block-body) - - [Proposer slashings](#proposer-slashings) - - [Attester slashings](#attester-slashings) - - [Attestations](#attestations) - - [Deposits](#deposits) - - [Exits](#exits) - - [Attestations](#attestations-1) - - [Attestation data](#attestation-data) - - [Slot](#slot-1) - - [Shard](#shard) - - [Beacon block root](#beacon-block-root) - - [Epoch boundary root](#epoch-boundary-root) - - [Shard block root](#shard-block-root) - - [Latest crosslink root](#latest-crosslink-root) - - [Justified epoch](#justified-epoch) - - [Justified block root](#justified-block-root) - - [Construct attestation](#construct-attestation) - - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) - - [Aggregate signature](#aggregate-signature) - - [Responsibility lookahead](#responsibility-lookahead) - - [How to avoid slashing](#how-to-avoid-slashing) - - [Proposer slashing](#proposer-slashing) - - [Attester slashing](#attester-slashing) + - [Table of Contents](#table-of-contents) + - [Introduction](#introduction) + - [Prerequisites](#prerequisites) + - [Constants](#constants) + - [Misc](#misc) + - [Becoming a validator](#becoming-a-validator) + - [Initialization](#initialization) + - [BLS public key](#bls-public-key) + - [BLS withdrawal key](#bls-withdrawal-key) + - [Submit deposit](#submit-deposit) + - [Process deposit](#process-deposit) + - [Validator index](#validator-index) + - [Activation](#activation) + - [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Block header](#block-header) + - [Slot](#slot) + - [Parent root](#parent-root) + - [State root](#state-root) + - [Randao reveal](#randao-reveal) + - [Eth1 Data](#eth1-data) + - [Signature](#signature) + - [Block body](#block-body) + - [Proposer slashings](#proposer-slashings) + - [Attester slashings](#attester-slashings) + - [Attestations](#attestations) + - [Deposits](#deposits) + - [Exits](#exits) + - [Attestations](#attestations-1) + - [Attestation data](#attestation-data) + - [Slot](#slot-1) + - [Shard](#shard) + - [Beacon block root](#beacon-block-root) + - [Epoch boundary root](#epoch-boundary-root) + - [Shard block root](#shard-block-root) + - [Latest crosslink root](#latest-crosslink-root) + - [Justified epoch](#justified-epoch) + - [Justified block root](#justified-block-root) + - [Construct attestation](#construct-attestation) + - [Data](#data) + - [Aggregation bitfield](#aggregation-bitfield) + - [Custody bitfield](#custody-bitfield) + - [Aggregate signature](#aggregate-signature) + - [Responsibility lookahead](#responsibility-lookahead) + - [How to avoid slashing](#how-to-avoid-slashing) + - [Proposer slashing](#proposer-slashing) + - [Attester slashing](#attester-slashing) @@ -95,7 +95,7 @@ The validator constructs their `withdrawal_credentials` via the following: ### Submit deposit -In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#ethereum-10-deposit-contract) located at `DEPOSIT_CONTRACT_ADDRESS`. +In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#ethereum-10-deposit-contract) located at `DEPOSIT_CONTRACT_ADDRESS`. To submit a deposit: @@ -166,7 +166,7 @@ Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined a ```python epoch_signature = bls_sign( privkey=validator.privkey, # privkey store locally, not in state - message=int_to_bytes32(slot_to_epoch(block.slot)), + message_hash=int_to_bytes32(slot_to_epoch(block.slot)), domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` epoch=slot_to_epoch(block.slot), @@ -205,7 +205,7 @@ proposal_root = hash_tree_root(proposal_data) signed_proposal_data = bls_sign( privkey=validator.privkey, # privkey store locally, not in state - message=proposal_root, + message_hash=proposal_root, domain=get_domain( fork=fork, # `fork` is the fork object at the slot `block.slot` epoch=slot_to_epoch(block.slot), @@ -321,7 +321,7 @@ attestation_message_to_sign = hash_tree_root(attestation_data_and_custody_bit) signed_attestation_data = bls_sign( privkey=validator.privkey, # privkey store locally, not in state - message=attestation_message_to_sign, + message_hash=attestation_message_to_sign, domain=get_domain( fork=fork, # `fork` is the fork object at the slot, `attestation_data.slot` epoch=slot_to_epoch(attestation_data.slot), From 911e4f104bde97f4361e78c7e14c9d09d6c6b5da Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 8 Feb 2019 05:12:58 +0800 Subject: [PATCH 45/96] Add `bytes_to_int` --- specs/core/0_beacon-chain.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 38d233b23..c7103b79e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -76,6 +76,7 @@ - [`get_attestation_participants`](#get_attestation_participants) - [`is_power_of_two`](#is_power_of_two) - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) + - [`bytes_to_int`](#bytes_to_int) - [`get_effective_balance`](#get_effective_balance) - [`get_total_balance`](#get_total_balance) - [`get_fork_version`](#get_fork_version) @@ -707,7 +708,7 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: i See the 'generalized domain' algorithm on page 3. """ for round in range(round_count): - pivot = int.from_bytes(hash(seed + int_to_bytes1(round)), 'little') % list_size + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))) % list_size flip = (pivot - index) % list_size position = max(index, flip) source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) @@ -1007,6 +1008,13 @@ def is_power_of_two(value: int) -> bool: `int_to_bytes1(x): return x.to_bytes(1, 'big')`, `int_to_bytes2(x): return x.to_bytes(2, 'big')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96. +### `bytes_to_int` + +```python +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') +``` + ### `get_effective_balance` ```python From 89b9894328b6a2060c3fbe61d2079ac400ca1208 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 8 Feb 2019 05:15:42 +0800 Subject: [PATCH 46/96] Fix type hinting --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c7103b79e..7167b93a6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -697,7 +697,7 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber ### `get_permuted_index` ```python -def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: int=90) -> List[int]: +def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: int=90) -> int: """ Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. From fd3d4a5105db74016483c1d1abbf81c932889b7f Mon Sep 17 00:00:00 2001 From: mratsim Date: Thu, 7 Feb 2019 22:15:55 +0100 Subject: [PATCH 47/96] Don't change TOC --- specs/core/0_beacon-chain.md | 264 +++++++++++++++++------------------ 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e6cf019f4..d4e90a57b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -6,139 +6,139 @@ - [Ethereum 2.0 Phase 0 -- The Beacon Chain](#ethereum-20-phase-0----the-beacon-chain) - - [Table of contents](#table-of-contents) - - [Introduction](#introduction) - - [Notation](#notation) - - [Terminology](#terminology) - - [Constants](#constants) - - [Misc](#misc) - - [Deposit contract](#deposit-contract) - - [Gwei values](#gwei-values) - - [Initial values](#initial-values) - - [Time parameters](#time-parameters) - - [State list lengths](#state-list-lengths) - - [Reward and penalty quotients](#reward-and-penalty-quotients) - - [Status flags](#status-flags) - - [Max operations per block](#max-operations-per-block) - - [Signature domains](#signature-domains) - - [Data structures](#data-structures) - - [Beacon chain operations](#beacon-chain-operations) - - [Proposer slashings](#proposer-slashings) - - [`ProposerSlashing`](#proposerslashing) - - [Attester slashings](#attester-slashings) - - [`AttesterSlashing`](#attesterslashing) - - [`SlashableAttestation`](#slashableattestation) - - [Attestations](#attestations) - - [`Attestation`](#attestation) - - [`AttestationData`](#attestationdata) - - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - - [Deposits](#deposits) - - [`Deposit`](#deposit) - - [`DepositData`](#depositdata) - - [`DepositInput`](#depositinput) - - [Exits](#exits) - - [`Exit`](#exit) - - [Beacon chain blocks](#beacon-chain-blocks) - - [`BeaconBlock`](#beaconblock) - - [`BeaconBlockBody`](#beaconblockbody) - - [`ProposalSignedData`](#proposalsigneddata) - - [Beacon chain state](#beacon-chain-state) - - [`BeaconState`](#beaconstate) - - [`Validator`](#validator) - - [`Crosslink`](#crosslink) - - [`PendingAttestation`](#pendingattestation) - - [`Fork`](#fork) - - [`Eth1Data`](#eth1data) - - [`Eth1DataVote`](#eth1datavote) - - [Custom Types](#custom-types) - - [Helper functions](#helper-functions) - - [`hash`](#hash) - - [`hash_tree_root`](#hashtreeroot) - - [`slot_to_epoch`](#slottoepoch) - - [`get_previous_epoch`](#getpreviousepoch) - - [`get_current_epoch`](#getcurrentepoch) - - [`get_epoch_start_slot`](#getepochstartslot) - - [`is_active_validator`](#isactivevalidator) - - [`get_active_validator_indices`](#getactivevalidatorindices) - - [`shuffle`](#shuffle) - - [`split`](#split) - - [`get_epoch_committee_count`](#getepochcommitteecount) - - [`get_shuffling`](#getshuffling) - - [`get_previous_epoch_committee_count`](#getpreviousepochcommitteecount) - - [`get_current_epoch_committee_count`](#getcurrentepochcommitteecount) - - [`get_next_epoch_committee_count`](#getnextepochcommitteecount) - - [`get_crosslink_committees_at_slot`](#getcrosslinkcommitteesatslot) - - [`get_block_root`](#getblockroot) - - [`get_randao_mix`](#getrandaomix) - - [`get_active_index_root`](#getactiveindexroot) - - [`generate_seed`](#generateseed) - - [`get_beacon_proposer_index`](#getbeaconproposerindex) - - [`merkle_root`](#merkleroot) - - [`get_attestation_participants`](#getattestationparticipants) - - [`is_power_of_two`](#ispoweroftwo) - - [`int_to_bytes1`, `int_to_bytes2`, ...](#inttobytes1-inttobytes2) - - [`get_effective_balance`](#geteffectivebalance) - - [`get_total_balance`](#gettotalbalance) - - [`get_fork_version`](#getforkversion) - - [`get_domain`](#getdomain) - - [`get_bitfield_bit`](#getbitfieldbit) - - [`verify_bitfield`](#verifybitfield) - - [`verify_slashable_attestation`](#verifyslashableattestation) - - [`is_double_vote`](#isdoublevote) - - [`is_surround_vote`](#issurroundvote) - - [`integer_squareroot`](#integersquareroot) - - [`get_entry_exit_effect_epoch`](#getentryexiteffectepoch) - - [`bls_verify`](#blsverify) - - [`bls_verify_multiple`](#blsverifymultiple) - - [`bls_aggregate_pubkeys`](#blsaggregatepubkeys) - - [`validate_proof_of_possession`](#validateproofofpossession) - - [`process_deposit`](#processdeposit) - - [Routines for updating validator status](#routines-for-updating-validator-status) - - [`activate_validator`](#activatevalidator) - - [`initiate_validator_exit`](#initiatevalidatorexit) - - [`exit_validator`](#exitvalidator) - - [`penalize_validator`](#penalizevalidator) - - [`prepare_validator_for_withdrawal`](#preparevalidatorforwithdrawal) - - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) - - [Deposit arguments](#deposit-arguments) - - [Withdrawal credentials](#withdrawal-credentials) - - [`Deposit` logs](#deposit-logs) - - [`ChainStart` log](#chainstart-log) - - [Vyper code](#vyper-code) - - [On startup](#on-startup) - - [Beacon chain processing](#beacon-chain-processing) - - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) - - [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [Per-slot processing](#per-slot-processing) - - [Slot](#slot) - - [Block roots](#block-roots) - - [Per-block processing](#per-block-processing) - - [Slot](#slot-1) - - [Proposer signature](#proposer-signature) - - [RANDAO](#randao) - - [Eth1 data](#eth1-data) - - [Operations](#operations) - - [Proposer slashings](#proposer-slashings-1) - - [Attester slashings](#attester-slashings-1) - - [Attestations](#attestations-1) - - [Deposits](#deposits-1) - - [Exits](#exits-1) - - [Per-epoch processing](#per-epoch-processing) - - [Helpers](#helpers) - - [Eth1 data](#eth1-data-1) - - [Justification](#justification) - - [Crosslinks](#crosslinks) - - [Rewards and penalties](#rewards-and-penalties) - - [Justification and finalization](#justification-and-finalization) - - [Attestation inclusion](#attestation-inclusion) - - [Crosslinks](#crosslinks-1) - - [Ejections](#ejections) - - [Validator registry and shuffling seed data](#validator-registry-and-shuffling-seed-data) - - [Final updates](#final-updates) - - [State root verification](#state-root-verification) + - [Table of contents](#table-of-contents) + - [Introduction](#introduction) + - [Notation](#notation) + - [Terminology](#terminology) + - [Constants](#constants) + - [Misc](#misc) + - [Deposit contract](#deposit-contract) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [State list lengths](#state-list-lengths) + - [Reward and penalty quotients](#reward-and-penalty-quotients) + - [Status flags](#status-flags) + - [Max operations per block](#max-operations-per-block) + - [Signature domains](#signature-domains) + - [Data structures](#data-structures) + - [Beacon chain operations](#beacon-chain-operations) + - [Proposer slashings](#proposer-slashings) + - [`ProposerSlashing`](#proposerslashing) + - [Attester slashings](#attester-slashings) + - [`AttesterSlashing`](#attesterslashing) + - [`SlashableAttestation`](#slashableattestation) + - [Attestations](#attestations) + - [`Attestation`](#attestation) + - [`AttestationData`](#attestationdata) + - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) + - [Deposits](#deposits) + - [`Deposit`](#deposit) + - [`DepositData`](#depositdata) + - [`DepositInput`](#depositinput) + - [Exits](#exits) + - [`Exit`](#exit) + - [Beacon chain blocks](#beacon-chain-blocks) + - [`BeaconBlock`](#beaconblock) + - [`BeaconBlockBody`](#beaconblockbody) + - [`ProposalSignedData`](#proposalsigneddata) + - [Beacon chain state](#beacon-chain-state) + - [`BeaconState`](#beaconstate) + - [`Validator`](#validator) + - [`Crosslink`](#crosslink) + - [`PendingAttestation`](#pendingattestation) + - [`Fork`](#fork) + - [`Eth1Data`](#eth1data) + - [`Eth1DataVote`](#eth1datavote) + - [Custom Types](#custom-types) + - [Helper functions](#helper-functions) + - [`hash`](#hash) + - [`hash_tree_root`](#hash_tree_root) + - [`slot_to_epoch`](#slot_to_epoch) + - [`get_previous_epoch`](#get_previous_epoch) + - [`get_current_epoch`](#get_current_epoch) + - [`get_epoch_start_slot`](#get_epoch_start_slot) + - [`is_active_validator`](#is_active_validator) + - [`get_active_validator_indices`](#get_active_validator_indices) + - [`shuffle`](#shuffle) + - [`split`](#split) + - [`get_epoch_committee_count`](#get_epoch_committee_count) + - [`get_shuffling`](#get_shuffling) + - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) + - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count) + - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count) + - [`get_crosslink_committees_at_slot`](#get_crosslink_committees_at_slot) + - [`get_block_root`](#get_block_root) + - [`get_randao_mix`](#get_randao_mix) + - [`get_active_index_root`](#get_active_index_root) + - [`generate_seed`](#generate_seed) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) + - [`merkle_root`](#merkle_root) + - [`get_attestation_participants`](#get_attestation_participants) + - [`is_power_of_two`](#is_power_of_two) + - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) + - [`get_effective_balance`](#get_effective_balance) + - [`get_total_balance`](#get_total_balance) + - [`get_fork_version`](#get_fork_version) + - [`get_domain`](#get_domain) + - [`get_bitfield_bit`](#get_bitfield_bit) + - [`verify_bitfield`](#verify_bitfield) + - [`verify_slashable_attestation`](#verify_slashable_attestation) + - [`is_double_vote`](#is_double_vote) + - [`is_surround_vote`](#is_surround_vote) + - [`integer_squareroot`](#integer_squareroot) + - [`get_entry_exit_effect_epoch`](#get_entry_exit_effect_epoch) + - [`bls_verify`](#bls_verify) + - [`bls_verify_multiple`](#bls_verify_multiple) + - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) + - [`validate_proof_of_possession`](#validate_proof_of_possession) + - [`process_deposit`](#process_deposit) + - [Routines for updating validator status](#routines-for-updating-validator-status) + - [`activate_validator`](#activate_validator) + - [`initiate_validator_exit`](#initiate_validator_exit) + - [`exit_validator`](#exit_validator) + - [`penalize_validator`](#penalize_validator) + - [`prepare_validator_for_withdrawal`](#prepare_validator_for_withdrawal) + - [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) + - [Deposit arguments](#deposit-arguments) + - [Withdrawal credentials](#withdrawal-credentials) + - [`Deposit` logs](#deposit-logs) + - [`ChainStart` log](#chainstart-log) + - [Vyper code](#vyper-code) + - [On startup](#on-startup) + - [Beacon chain processing](#beacon-chain-processing) + - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) + - [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Per-slot processing](#per-slot-processing) + - [Slot](#slot) + - [Block roots](#block-roots) + - [Per-block processing](#per-block-processing) + - [Slot](#slot-1) + - [Proposer signature](#proposer-signature) + - [RANDAO](#randao) + - [Eth1 data](#eth1-data) + - [Operations](#operations) + - [Proposer slashings](#proposer-slashings-1) + - [Attester slashings](#attester-slashings-1) + - [Attestations](#attestations-1) + - [Deposits](#deposits-1) + - [Exits](#exits-1) + - [Per-epoch processing](#per-epoch-processing) + - [Helpers](#helpers) + - [Eth1 data](#eth1-data-1) + - [Justification](#justification) + - [Crosslinks](#crosslinks) + - [Rewards and penalties](#rewards-and-penalties) + - [Justification and finalization](#justification-and-finalization) + - [Attestation inclusion](#attestation-inclusion) + - [Crosslinks](#crosslinks-1) + - [Ejections](#ejections) + - [Validator registry and shuffling seed data](#validator-registry-and-shuffling-seed-data) + - [Final updates](#final-updates) + - [State root verification](#state-root-verification) - [References](#references) - - [Normative](#normative) - - [Informative](#informative) + - [Normative](#normative) + - [Informative](#informative) - [Copyright](#copyright) From ca098f8cfa5c0ab0daa38c784ee4e563fd1c9fdf Mon Sep 17 00:00:00 2001 From: mratsim Date: Thu, 7 Feb 2019 22:19:04 +0100 Subject: [PATCH 48/96] Prevent changing another TOC --- specs/validator/0_beacon-chain-validator.md | 94 ++++++++++----------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index b8db2117c..744df690f 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -7,53 +7,53 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers - [Ethereum 2.0 Phase 0 -- Honest Validator](#ethereum-20-phase-0----honest-validator) - - [Table of Contents](#table-of-contents) - - [Introduction](#introduction) - - [Prerequisites](#prerequisites) - - [Constants](#constants) - - [Misc](#misc) - - [Becoming a validator](#becoming-a-validator) - - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [BLS withdrawal key](#bls-withdrawal-key) - - [Submit deposit](#submit-deposit) - - [Process deposit](#process-deposit) - - [Validator index](#validator-index) - - [Activation](#activation) - - [Beacon chain responsibilities](#beacon-chain-responsibilities) - - [Block proposal](#block-proposal) - - [Block header](#block-header) - - [Slot](#slot) - - [Parent root](#parent-root) - - [State root](#state-root) - - [Randao reveal](#randao-reveal) - - [Eth1 Data](#eth1-data) - - [Signature](#signature) - - [Block body](#block-body) - - [Proposer slashings](#proposer-slashings) - - [Attester slashings](#attester-slashings) - - [Attestations](#attestations) - - [Deposits](#deposits) - - [Exits](#exits) - - [Attestations](#attestations-1) - - [Attestation data](#attestation-data) - - [Slot](#slot-1) - - [Shard](#shard) - - [Beacon block root](#beacon-block-root) - - [Epoch boundary root](#epoch-boundary-root) - - [Shard block root](#shard-block-root) - - [Latest crosslink root](#latest-crosslink-root) - - [Justified epoch](#justified-epoch) - - [Justified block root](#justified-block-root) - - [Construct attestation](#construct-attestation) - - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) - - [Aggregate signature](#aggregate-signature) - - [Responsibility lookahead](#responsibility-lookahead) - - [How to avoid slashing](#how-to-avoid-slashing) - - [Proposer slashing](#proposer-slashing) - - [Attester slashing](#attester-slashing) + - [Table of Contents](#table-of-contents) + - [Introduction](#introduction) + - [Prerequisites](#prerequisites) + - [Constants](#constants) + - [Misc](#misc) + - [Becoming a validator](#becoming-a-validator) + - [Initialization](#initialization) + - [BLS public key](#bls-public-key) + - [BLS withdrawal key](#bls-withdrawal-key) + - [Submit deposit](#submit-deposit) + - [Process deposit](#process-deposit) + - [Validator index](#validator-index) + - [Activation](#activation) + - [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Block header](#block-header) + - [Slot](#slot) + - [Parent root](#parent-root) + - [State root](#state-root) + - [Randao reveal](#randao-reveal) + - [Eth1 Data](#eth1-data) + - [Signature](#signature) + - [Block body](#block-body) + - [Proposer slashings](#proposer-slashings) + - [Attester slashings](#attester-slashings) + - [Attestations](#attestations) + - [Deposits](#deposits) + - [Exits](#exits) + - [Attestations](#attestations-1) + - [Attestation data](#attestation-data) + - [Slot](#slot-1) + - [Shard](#shard) + - [Beacon block root](#beacon-block-root) + - [Epoch boundary root](#epoch-boundary-root) + - [Shard block root](#shard-block-root) + - [Latest crosslink root](#latest-crosslink-root) + - [Justified epoch](#justified-epoch) + - [Justified block root](#justified-block-root) + - [Construct attestation](#construct-attestation) + - [Data](#data) + - [Aggregation bitfield](#aggregation-bitfield) + - [Custody bitfield](#custody-bitfield) + - [Aggregate signature](#aggregate-signature) + - [Responsibility lookahead](#responsibility-lookahead) + - [How to avoid slashing](#how-to-avoid-slashing) + - [Proposer slashing](#proposer-slashing) + - [Attester slashing](#attester-slashing) From 086df84bdc6805d1c0ad1e1c4ff953fa1bd3802b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 7 Feb 2019 19:34:54 -0600 Subject: [PATCH 49/96] Attestation data contains latest crosslink, not just latest crosslink data The reason to do this is that it makes it calculable from inside an attestation how many epochs the attestation spans over, which is needed for proof of custody reasons. It's a relatively small change and so arguably easier to do now than to do as a patch in phase 1. Note that this changes the meaning of latest_crosslink.epoch, from the epoch when the latest crosslink was included to the epoch that the latest crosslink was for. This affects the line: * `state.latest_crosslinks[shard].epoch > state.validator_registry_update_epoch` for every shard number `shard` in `[(state.current_epoch_start_shard + i) % SHARD_COUNT for i in range(get_current_epoch_committee_count(state))]` (that is, for every shard in the current committees) But this may actually make it _more_ correct, as it means that in the case where >512 shards are processed per epoch, and so a committee from the previous epoch could get finalized in the current epoch, that would no longer count toward every shard having received a "new" crosslink. --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f9c155c53..b98bdf3b8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -362,8 +362,8 @@ The following data structures are defined as [SimpleSerialize (SSZ)](https://git 'epoch_boundary_root': 'bytes32', # Shard block's hash of root 'shard_block_root': 'bytes32', - # Last crosslink's hash of root - 'latest_crosslink_root': 'bytes32', + # Last crosslink + 'latest_crosslink': Crosslink, # Last justified epoch in the beacon state 'justified_epoch': 'uint64', # Hash of the last justified beacon block @@ -1688,7 +1688,7 @@ For each `attestation` in `block.body.attestations`: * Verify that `attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY < attestation.data.slot + EPOCH_LENGTH`. * Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch if attestation.data.slot >= get_epoch_start_slot(get_current_epoch(state)) else state.previous_justified_epoch`. * Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, get_epoch_start_slot(attestation.data.justified_epoch))`. -* Verify that either `attestation.data.latest_crosslink_root` or `attestation.data.shard_block_root` equals `state.latest_crosslinks[attestation.data.shard].shard_block_root`. +* Verify that either (i) `state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink` or (ii) `state.latest_crosslinks[attestation.data.shard] == Crosslink(shard_block_root=attestation.data.shard_block_root, epoch=slot_to_epoch(attestation.data.slot))` * Verify bitfields and aggregate signature: ```python @@ -1860,7 +1860,7 @@ Finally, update the following: For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch))`, let `crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)`. For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot`, compute: -* Set `state.latest_crosslinks[shard] = Crosslink(epoch=current_epoch, shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * get_total_balance(crosslink_committee)`. +* Set `state.latest_crosslinks[shard] = Crosslink(epoch=slot_to_epoch(slot), shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * get_total_balance(crosslink_committee)`. #### Rewards and penalties From f797826ee29767ef3133a97ec1f28d347553ede7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 Feb 2019 21:51:56 -0600 Subject: [PATCH 50/96] Update specs/core/0_beacon-chain.md Co-Authored-By: vbuterin --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7167b93a6..50daad513 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -708,7 +708,7 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: i See the 'generalized domain' algorithm on page 3. """ for round in range(round_count): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))) % list_size + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size flip = (pivot - index) % list_size position = max(index, flip) source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) From 1c6ccac8fc1f02079f49bf9fd715b64764c304e0 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 7 Feb 2019 21:55:33 -0600 Subject: [PATCH 51/96] SHUFFLE_ROUND_COUNT as global constant --- specs/core/0_beacon-chain.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 50daad513..a7a21683c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -185,6 +185,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be | `BEACON_CHAIN_SHARD_NUMBER` | `2**64 - 1` | - | | `MAX_INDICES_PER_SLASHABLE_VOTE` | `2**12` (= 4,096) | votes | | `MAX_WITHDRAWALS_PER_EPOCH` | `2**2` (= 4) | withdrawals | +| `SHUFFLE_ROUND_COUNT` | 90 | - | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `EPOCH_LENGTH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -697,17 +698,15 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber ### `get_permuted_index` ```python -def get_permuted_index(index: int, list_size: int, seed: Bytes32, round_count: int=90) -> int: +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: """ Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. - Note that ``round_count`` is ``90`` in protocol and parameterized for the shuffling tests. - Utilizes 'swap or not' shuffling found in https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf See the 'generalized domain' algorithm on page 3. """ - for round in range(round_count): + for round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size flip = (pivot - index) % list_size position = max(index, flip) From f0cbacb828baa6ed454bea5d5efe52b0a73b67f5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 Feb 2019 20:22:28 -0800 Subject: [PATCH 52/96] add missing . --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b98bdf3b8..f66839349 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1688,7 +1688,7 @@ For each `attestation` in `block.body.attestations`: * Verify that `attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY < attestation.data.slot + EPOCH_LENGTH`. * Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch if attestation.data.slot >= get_epoch_start_slot(get_current_epoch(state)) else state.previous_justified_epoch`. * Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, get_epoch_start_slot(attestation.data.justified_epoch))`. -* Verify that either (i) `state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink` or (ii) `state.latest_crosslinks[attestation.data.shard] == Crosslink(shard_block_root=attestation.data.shard_block_root, epoch=slot_to_epoch(attestation.data.slot))` +* Verify that either (i) `state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink` or (ii) `state.latest_crosslinks[attestation.data.shard] == Crosslink(shard_block_root=attestation.data.shard_block_root, epoch=slot_to_epoch(attestation.data.slot))`. * Verify bitfields and aggregate signature: ```python From 6d9581281dba8c8d2cf127ddd07e6202759c405c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 7 Feb 2019 21:25:47 -0700 Subject: [PATCH 53/96] change latest_crosslink_root to latest_crosslink in validator guide --- specs/validator/0_beacon-chain-validator.md | 6 +- wire-api.md | 93 +++++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 wire-api.md diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 744df690f..d1246c6fa 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -42,7 +42,7 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers - [Beacon block root](#beacon-block-root) - [Epoch boundary root](#epoch-boundary-root) - [Shard block root](#shard-block-root) - - [Latest crosslink root](#latest-crosslink-root) + - [Latest crosslink](#latest-crosslink) - [Justified epoch](#justified-epoch) - [Justified block root](#justified-block-root) - [Construct attestation](#construct-attestation) @@ -270,9 +270,9 @@ Set `attestation_data.shard_block_root = ZERO_HASH`. _Note:_ This is a stub for phase 0. -##### Latest crosslink root +##### Latest crosslink -Set `attestation_data.latest_crosslink_root = state.latest_crosslinks[shard].shard_block_root` where `state` is the beacon state at `head` and `shard` is the validator's assigned shard. +Set `attestation_data.latest_crosslink = state.latest_crosslinks[shard]` where `state` is the beacon state at `head` and `shard` is the validator's assigned shard. ##### Justified epoch diff --git a/wire-api.md b/wire-api.md new file mode 100644 index 000000000..0eff650bd --- /dev/null +++ b/wire-api.md @@ -0,0 +1,93 @@ +# Phase 0 Wire API [WIP] + +This is the minimal wire API required for Phase 0 of Eth2.0. Note that this is _not_ the wire protocol but the interface right above. Once we settle on the API required, we can specify the underlying protocol. + +All API methods are specified as the plural `list` version, assuming that if singular objects are sent or requested that the input will just be a list of length 1. + +"Bad form" is any action that is not explicitly against the protocol but is not in the best interest of one's peers or the protocol in general. Messages/requests that are considered bad form may reduce the reputation of the sending node and may result in being dropped. + +## Network topology + +Ethereum 2.0 network topology consists of a pubsub mapping of peers to "topics". These topics along with peer mappings effectively form subnets. + +The primary topics of core protocol consideration are: +* `beacon`: All messages for the beacon chain are mapped to topic `beacon`. +* `shard-{number}` for all integers, `number` in `range(SHARD_SUBNET_COUNT)`: Messages for a given shard defined by `shard_number` are mapped to topic `shard-{shard_number % SHARD_SUBNET_COUNT}`. + +We use `discv5` to discover peers of select topics, and we use `gossipsub`, a libp2p routing protocol, to route messages of a particular topic to the subnet in question. + +Note: attempting to broadcast or request messages about a topic not subscribed to by the peer is considered bad form. For example, running `send_attestations(attestations)` where one or more of the attestations have `attestation.data.shard == 5` to a peer not subscribed to `shard-5` might result in that peer dropping the node. + +## Dependencies + +This document depends on: +* [SSZ spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md) +* [Phase 0 spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md) + +## API + +### Sync + +The following is a basic sync protocol akin to eth1.0. _This is very likely to change pending input from those intimately familiar with the pain points of 1.0 sync_. + +`status` message is sent in the initial handshake between two peers. After handshake and status exchange, the peer with higher `latest_finalized_epoch` or, if epochs are equal, the higher `best_slot` sends a list of `beacon_block_roots` via `send_beacon_block_roots`. + +Status handshake fields: +* `protocol_version` +* `network_id` +* `latest_finalized_root` +* `latest_finalized_epoch` +* `best_root` +* `best_slot` + +### Beacon Blocks + +Supported pubsub topics: +* `beacon` + +The following definitions are used in the API: +* `block_header`: a serialized `BeaconBlock` in which `BeaconBlock.body` is the `hash_tree_root` of the associated `BeaconBlockBody`. +* `block_body`: a serialied `BeaconBlockBody`. +* `block_root`: the `hash_tree_root` of a `BeaconBlock`. + +API: +* `send_beacon_block_roots(block_roots)`: Sends list of `block_roots` to peer. +* `send_beacon_block_headers(block_headers)`: Sends list of `block_headers` to peer. +* `request_beacon_block_headers(block_roots)`: Requests the associated `block_headers` for the given `block_roots` from peer. +* `send_beacon_block_bodies(block_bodies)`: Sends list of `block_bodies` to peer. +* `request_beacon_block_bodies(block_roots)`: Requests the associated `block_bodies` for the given `block_roots` from peer. + +Notes: +* It is assumed that both the associated `BeaconBlock` and `BeaconBlockBody` can be looked up via `block_root`. + +### Attestations + +Supported pubsub topics: +* `beacon` +* all `shard-{number}` topics + +The following definitions are used in the API: +* `attestation`: a serialized `Attestation` with full serialized `AttestationData` for `Attestation.data`. + +API: +* `send_attestations(attestations)`: Sends list of `attestations` to peer. + +Notes: +* It is expected that an attestation is only broadcast to either `beacon` topic or `shard-{attestation.data.shard}` topic. Broadcasting to mismatched shard topics is considered bad form. +* It is expected that only aggregate attestations are broadcast to the `beacon` topic. Repeated broadcasting of attestations with a signle signer to the `beacon` topic is considered bad form. +* There is a shard subnet design decision here. Due to the likelihood of `attestation.data` to be highly repeated across a committee during a given slot, it could be valuable to just pass the `attestation` with a `root` in the `attestation.data` field. If the recipient does not already have an `AttestationData` for the received `root`, then the recipient would explicitly request the root. This reduces the total data passed by 184 bytes in the case that the recipient has already received the `attestation.data` but increases the rounds of communication when they haven't. +* We do not currently specify a getter method for an attestation by its `root`. Due to the diverse ways attestations might both be aggregated and stored, it is not feasible to reliably lookup via a `root`. The attestations that a client cares about are (1) those that made it on-chain into a `BeaconBlock` and (2) the most recent set of attestations being actively broadcast on the wire. We might provide a `request_attestations(slot)` or `request_attestations(epoch)` but do not provide it in this minimal API specification. + +### Exits + +Supported pubsub topics: +* `beacon` + +The following definitions are used in the API: +* `exit`: a serialized `Exit`. + +API: +* `send_exit(exit)`: Sends `exit` to peer. + +Notes: +* We do not specify a getter for an exit by its `root`. Standard usage is for \ No newline at end of file From fb90fd7ac3b1accd9dc680439260dd25d59414c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 01:07:31 -0600 Subject: [PATCH 54/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f3324bbbd..f25844ecb 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -71,7 +71,7 @@ To validate a block header on shard `shard_id`, compute as follows: * Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, slot, shard_id)`. * Assert `verify_bitfield(participation_bitfield, len(persistent_committee))` * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_id) + int_to_bytes8(slot)) % len(validators)`. Let `msg` be the block but with the `block.signature` set to `[0, 0]`. Verify that `BLSVerify(pub=validators[proposer_index].pubkey, msg=hash(msg), sig=block.signature, domain=get_domain(state, slot, SHARD_PROPOSER_DOMAIN))` passes. -* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, msg=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. +* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. We define the helper `get_proposal_committee` as follows: From a1f542b6743bba384763cb0b76a67d2e919ed4e6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 01:07:46 -0600 Subject: [PATCH 55/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f25844ecb..a401c87c3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -78,7 +78,7 @@ We define the helper `get_proposal_committee` as follows: ```python def get_proposal_committee(seed: Bytes32, validators: List[Validator], - shard: int, + shard: ShardNumber, epoch: EpochNumber) -> List[ValidatorIndex]: earlier_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD * 2 From bdd7dc106af7e541da4f257d5233495f074aaff6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 01:08:01 -0600 Subject: [PATCH 56/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a401c87c3..cb262a1ee 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -90,7 +90,7 @@ def get_proposal_committee(seed: Bytes32, later_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD later_committee = split(shuffle( get_active_validator_indices(validators, earlier_committee_start), - generate_seed(state, earlier_committee_start) + generate_seed(state, later_committee_start) ), SHARD_COUNT)[shard] def get_switchover_epoch(index): From 5e84d7e3ffd411e31f284989393085b20941a1b4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 01:08:15 -0600 Subject: [PATCH 57/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index cb262a1ee..2984b3fb7 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -95,7 +95,7 @@ def get_proposal_committee(seed: Bytes32, def get_switchover_epoch(index): return ( - int.from_bytes(hash(generate_seed(state, earlier_committee_start) + bytes3(index)), 'little') % + bytes_to_int(hash(generate_seed(state, earlier_committee_start) + bytes3(index))[0:8]) % PROPOSAL_RESHUFFLE_PERIOD ) From 5a1b2c29f262460c10bee8f2658294ddc2015326 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 01:08:30 -0600 Subject: [PATCH 58/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 2984b3fb7..af49825bd 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -89,7 +89,7 @@ def get_proposal_committee(seed: Bytes32, later_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD later_committee = split(shuffle( - get_active_validator_indices(validators, earlier_committee_start), + get_active_validator_indices(validators, later_committee_start), generate_seed(state, later_committee_start) ), SHARD_COUNT)[shard] From 2cff0e6b7a3f3437bc040f1fd023385a6c1a743c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 8 Feb 2019 16:46:50 +0800 Subject: [PATCH 59/96] Use the new `bytesN` SSZ object `hash32` -> `bytes32` ['uint384'] -> `bytes96` --- specs/core/1_shard-data-chains.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index af49825bd..0af79235e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -16,11 +16,11 @@ Ethereum 2.0 consists of a central beacon chain along with `SHARD_COUNT` shard c Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md#constants) in addition to the following: -| Constant | Value | Unit | Approximation | -|-----------------------------|-----------------|--------|---------------| -| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | -| `SHARD_BLOCK_SIZE` | 2**14 (= 16384) | bytes | | -| `PROPOSAL_RESHUFFLE_PERIOD` | 2**11 (= 2048) | epochs | 9 days | +| Constant | Value | Unit | Approximation | +|-----------------------------|------------------|--------|---------------| +| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | +| `SHARD_BLOCK_SIZE` | 2**14 (= 16,384) | bytes | | +| `PROPOSAL_RESHUFFLE_PERIOD` | 2**11 (= 2,048) | epochs | 9 days | ### Flags, domains, etc. @@ -42,18 +42,18 @@ A `ShardBlock` object has the following fields: # What shard is it on 'shard_id': 'uint64', # Parent block's root - 'parent_root': 'hash32', + 'parent_root': 'bytes32', # Beacon chain block - 'beacon_chain_ref': 'hash32', + 'beacon_chain_ref': 'bytes32', # Merkle root of data - 'data_root': 'hash32' + 'data_root': 'bytes32' # State root (placeholder for now) - 'state_root': 'hash32', + 'state_root': 'bytes32', # Block signature - 'signature': ['uint384'], + 'signature': 'bytes96', # Attestation 'participation_bitfield': 'bytes', - 'aggregate_signature': ['uint384'], + 'aggregate_signature': 'bytes96', } ``` From 410372ff0ed909f7b5a1063a74f2eee70343cea4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 03:48:58 -0600 Subject: [PATCH 60/96] Proposal committee -> persistent committee --- specs/core/1_shard-data-chains.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 0af79235e..1224449e3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -16,11 +16,11 @@ Ethereum 2.0 consists of a central beacon chain along with `SHARD_COUNT` shard c Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md#constants) in addition to the following: -| Constant | Value | Unit | Approximation | -|-----------------------------|------------------|--------|---------------| -| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | -| `SHARD_BLOCK_SIZE` | 2**14 (= 16,384) | bytes | | -| `PROPOSAL_RESHUFFLE_PERIOD` | 2**11 (= 2,048) | epochs | 9 days | +| Constant | Value | Unit | Approximation | +|-------------------------------|------------------|--------|---------------| +| `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | +| `SHARD_BLOCK_SIZE` | 2**14 (= 16,384) | bytes | | +| `PERSISTENT_COMMITTEE_PERIOD` | 2**11 (= 2,048) | epochs | 9 days | ### Flags, domains, etc. @@ -73,21 +73,21 @@ To validate a block header on shard `shard_id`, compute as follows: * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_id) + int_to_bytes8(slot)) % len(validators)`. Let `msg` be the block but with the `block.signature` set to `[0, 0]`. Verify that `BLSVerify(pub=validators[proposer_index].pubkey, msg=hash(msg), sig=block.signature, domain=get_domain(state, slot, SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. -We define the helper `get_proposal_committee` as follows: +We define the helper `get_persistent_committee` as follows: ```python -def get_proposal_committee(seed: Bytes32, - validators: List[Validator], - shard: ShardNumber, - epoch: EpochNumber) -> List[ValidatorIndex]: +def get_persistent_commmitee(seed: Bytes32, + validators: List[Validator], + shard: ShardNumber, + epoch: EpochNumber) -> List[ValidatorIndex]: - earlier_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD * 2 + earlier_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 earlier_committee = split(shuffle( get_active_validator_indices(validators, earlier_committee_start), generate_seed(state, earlier_committee_start) ), SHARD_COUNT)[shard] - later_committee_start = epoch - (epoch % PROPOSAL_RESHUFFLE_PERIOD) - PROPOSAL_RESHUFFLE_PERIOD + later_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD later_committee = split(shuffle( get_active_validator_indices(validators, later_committee_start), generate_seed(state, later_committee_start) @@ -96,12 +96,12 @@ def get_proposal_committee(seed: Bytes32, def get_switchover_epoch(index): return ( bytes_to_int(hash(generate_seed(state, earlier_committee_start) + bytes3(index))[0:8]) % - PROPOSAL_RESHUFFLE_PERIOD + PERSISTENT_COMMITTEE_PERIOD ) return ( - [i for i in earlier_committee if epoch % PROPOSAL_RESHUFFLE_PERIOD < get_switchover_epoch(i)] + - [i for i in later_committee if epoch % PROPOSAL_RESHUFFLE_PERIOD >= get_switchover_epoch(i)] + [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ) ``` From 47c396f3f8839644db5337922c1d31468d3cba6c Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 03:50:00 -0600 Subject: [PATCH 61/96] Break lets into separate lines --- specs/core/1_shard-data-chains.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 1224449e3..3fa49697c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -68,7 +68,8 @@ To validate a block header on shard `shard_id`, compute as follows: * Verify that `beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. * Verify that `beacon_chain_ref` is equal to or a descendant of the `beacon_chain_ref` specified in the `ShardBlock` pointed to by `parent_root`. -* Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, slot, shard_id)`. +* Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. +* Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, slot, shard_id)`. * Assert `verify_bitfield(participation_bitfield, len(persistent_committee))` * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_id) + int_to_bytes8(slot)) % len(validators)`. Let `msg` be the block but with the `block.signature` set to `[0, 0]`. Verify that `BLSVerify(pub=validators[proposer_index].pubkey, msg=hash(msg), sig=block.signature, domain=get_domain(state, slot, SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. From 65ec418213c025d64d0d498dab964950dedf4fba Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 03:54:02 -0600 Subject: [PATCH 62/96] A few more cleanups --- specs/core/1_shard-data-chains.md | 93 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 3fa49697c..8b6d4ad5f 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -29,52 +29,9 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md | `SHARD_PROPOSER_DOMAIN`| 129 | | `SHARD_ATTESTER_DOMAIN`| 130 | -## Data Structures +## Helper functions -### Shard chain blocks - -A `ShardBlock` object has the following fields: - -```python -{ - # Slot number - 'slot': 'uint64', - # What shard is it on - 'shard_id': 'uint64', - # Parent block's root - 'parent_root': 'bytes32', - # Beacon chain block - 'beacon_chain_ref': 'bytes32', - # Merkle root of data - 'data_root': 'bytes32' - # State root (placeholder for now) - 'state_root': 'bytes32', - # Block signature - 'signature': 'bytes96', - # Attestation - 'participation_bitfield': 'bytes', - 'aggregate_signature': 'bytes96', -} -``` - -## Shard block processing - -For a block on a shard to be processed by a node, the following conditions must be met: - -* The `ShardBlock` pointed to by `parent_root` has already been processed and accepted -* The signature for the block from the _proposer_ (see below for definition) of that block is included along with the block in the network message object - -To validate a block header on shard `shard_id`, compute as follows: - -* Verify that `beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. -* Verify that `beacon_chain_ref` is equal to or a descendant of the `beacon_chain_ref` specified in the `ShardBlock` pointed to by `parent_root`. -* Let `state` be the state of the beacon chain block referred to by `beacon_chain_ref`. -* Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, slot, shard_id)`. -* Assert `verify_bitfield(participation_bitfield, len(persistent_committee))` -* Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_id) + int_to_bytes8(slot)) % len(validators)`. Let `msg` be the block but with the `block.signature` set to `[0, 0]`. Verify that `BLSVerify(pub=validators[proposer_index].pubkey, msg=hash(msg), sig=block.signature, domain=get_domain(state, slot, SHARD_PROPOSER_DOMAIN))` passes. -* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=parent_root, sig=block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. - -We define the helper `get_persistent_committee` as follows: +#### get_persistent_committee ```python def get_persistent_commmitee(seed: Bytes32, @@ -106,6 +63,52 @@ def get_persistent_commmitee(seed: Bytes32, ) ``` +## Data Structures + +### Shard chain blocks + +A `ShardBlock` object has the following fields: + +```python +{ + # Slot number + 'slot': 'uint64', + # What shard is it on + 'shard_id': 'uint64', + # Parent block's root + 'parent_root': 'bytes32', + # Beacon chain block + 'beacon_chain_ref': 'bytes32', + # Merkle root of data + 'data_root': 'bytes32' + # State root (placeholder for now) + 'state_root': 'bytes32', + # Block signature + 'signature': 'bytes96', + # Attestation + 'participation_bitfield': 'bytes', + 'aggregate_signature': 'bytes96', +} +``` + +## Shard block processing + +For a `shard_block` on a shard to be processed by a node, the following conditions must be met: + +* The `ShardBlock` pointed to by `shard_block.parent_root` has already been processed and accepted +* The signature for the block from the _proposer_ (see below for definition) of that block is included along with the block in the network message object + +To validate a block header on shard `shard_block.shard_id`, compute as follows: + +* Verify that `shard_block.beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. +* Verify that `shard_block.beacon_chain_ref` is equal to or a descendant of the `shard_block.beacon_chain_ref` specified in the `ShardBlock` pointed to by `shard_block.parent_root`. +* Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. +* Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. +* Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` +* Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot)) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, shard_block.slot, SHARD_PROPOSER_DOMAIN))` passes. +* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. + + ### Verifying shard block data At network layer, we expect a shard block header to be broadcast along with its `block_body`. From e5bd78e5e659208f89bce16c9a0aa4f82cd03fae Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 04:08:52 -0600 Subject: [PATCH 63/96] Persistent committee refactors --- specs/core/1_shard-data-chains.md | 38 +++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8b6d4ad5f..b4bbf3a7e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -31,6 +31,17 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md ## Helper functions +#### get_split_offset + +````python +def get_split_offset(list_size: int, chunks: int, index: int) -> int: + """ + Returns a value such that for a list L, chunk count k and index i, + split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k+1, i)] + """ + return (len(list_size) * index) // chunks +```` + #### get_persistent_committee ```python @@ -40,20 +51,28 @@ def get_persistent_commmitee(seed: Bytes32, epoch: EpochNumber) -> List[ValidatorIndex]: earlier_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - earlier_committee = split(shuffle( - get_active_validator_indices(validators, earlier_committee_start), - generate_seed(state, earlier_committee_start) - ), SHARD_COUNT)[shard] + earlier_validator_set = get_active_validator_indices(validators, earlier_committee_start) + earlier_seed = generate_seed(state, earlier_committee_start) + earlier_start_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard) + earlier_end_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard+1) + earlier_committee = [ + earlier_validator_set[get_permuted_index(i, len(earlier_validator_set), earlier_seed)] + for i in range(earlier_start_offset, earlier_end_offset) + ] later_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD - later_committee = split(shuffle( - get_active_validator_indices(validators, later_committee_start), - generate_seed(state, later_committee_start) - ), SHARD_COUNT)[shard] + later_validator_set = get_active_validator_indices(validators, later_committee_start) + later_seed = generate_seed(state, later_committee_start) + later_start_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard) + later_end_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard+1) + later_committee = [ + later_validator_set[get_permuted_index(i, len(later_validator_set), later_seed)] + for i in range(later_start_offset, later_end_offset) + ] def get_switchover_epoch(index): return ( - bytes_to_int(hash(generate_seed(state, earlier_committee_start) + bytes3(index))[0:8]) % + bytes_to_int(hash(earlier_seed + bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) @@ -108,7 +127,6 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot)) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, shard_block.slot, SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. - ### Verifying shard block data At network layer, we expect a shard block header to be broadcast along with its `block_body`. From f9eaab1d044e2da9854217099b4bb8325388740f Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Fri, 8 Feb 2019 18:04:32 +0300 Subject: [PATCH 64/96] Silently skip deposits with invalid proof in process_deposit --- specs/core/0_beacon-chain.md | 49 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 21a991c42..c08343ef9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1252,36 +1252,37 @@ def process_deposit(state: BeaconState, Note that this function mutates ``state``. """ # Validate the given `proof_of_possession` - assert validate_proof_of_possession( + valid_proof = validate_proof_of_possession( state, pubkey, proof_of_possession, withdrawal_credentials, ) - validator_pubkeys = [v.pubkey for v in state.validator_registry] - - if pubkey not in validator_pubkeys: - # Add new validator - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawal_epoch=FAR_FUTURE_EPOCH, - penalized_epoch=FAR_FUTURE_EPOCH, - status_flags=0, - ) - - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. - state.validator_registry.append(validator) - state.validator_balances.append(amount) - else: - # Increase balance by deposit amount - index = validator_pubkeys.index(pubkey) - assert state.validator_registry[index].withdrawal_credentials == withdrawal_credentials - - state.validator_balances[index] += amount + if valid_proof: + validator_pubkeys = [v.pubkey for v in state.validator_registry] + + if pubkey not in validator_pubkeys: + # Add new validator + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawal_epoch=FAR_FUTURE_EPOCH, + penalized_epoch=FAR_FUTURE_EPOCH, + status_flags=0, + ) + + # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. + state.validator_registry.append(validator) + state.validator_balances.append(amount) + else: + # Increase balance by deposit amount + index = validator_pubkeys.index(pubkey) + assert state.validator_registry[index].withdrawal_credentials == withdrawal_credentials + + state.validator_balances[index] += amount ``` ### Routines for updating validator status From 92471046a1879361d5c87443da0890db0e7d7846 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 08:28:41 -0700 Subject: [PATCH 65/96] remove wire protocol doc --- wire-api.md | 93 ----------------------------------------------------- 1 file changed, 93 deletions(-) delete mode 100644 wire-api.md diff --git a/wire-api.md b/wire-api.md deleted file mode 100644 index 0eff650bd..000000000 --- a/wire-api.md +++ /dev/null @@ -1,93 +0,0 @@ -# Phase 0 Wire API [WIP] - -This is the minimal wire API required for Phase 0 of Eth2.0. Note that this is _not_ the wire protocol but the interface right above. Once we settle on the API required, we can specify the underlying protocol. - -All API methods are specified as the plural `list` version, assuming that if singular objects are sent or requested that the input will just be a list of length 1. - -"Bad form" is any action that is not explicitly against the protocol but is not in the best interest of one's peers or the protocol in general. Messages/requests that are considered bad form may reduce the reputation of the sending node and may result in being dropped. - -## Network topology - -Ethereum 2.0 network topology consists of a pubsub mapping of peers to "topics". These topics along with peer mappings effectively form subnets. - -The primary topics of core protocol consideration are: -* `beacon`: All messages for the beacon chain are mapped to topic `beacon`. -* `shard-{number}` for all integers, `number` in `range(SHARD_SUBNET_COUNT)`: Messages for a given shard defined by `shard_number` are mapped to topic `shard-{shard_number % SHARD_SUBNET_COUNT}`. - -We use `discv5` to discover peers of select topics, and we use `gossipsub`, a libp2p routing protocol, to route messages of a particular topic to the subnet in question. - -Note: attempting to broadcast or request messages about a topic not subscribed to by the peer is considered bad form. For example, running `send_attestations(attestations)` where one or more of the attestations have `attestation.data.shard == 5` to a peer not subscribed to `shard-5` might result in that peer dropping the node. - -## Dependencies - -This document depends on: -* [SSZ spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md) -* [Phase 0 spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md) - -## API - -### Sync - -The following is a basic sync protocol akin to eth1.0. _This is very likely to change pending input from those intimately familiar with the pain points of 1.0 sync_. - -`status` message is sent in the initial handshake between two peers. After handshake and status exchange, the peer with higher `latest_finalized_epoch` or, if epochs are equal, the higher `best_slot` sends a list of `beacon_block_roots` via `send_beacon_block_roots`. - -Status handshake fields: -* `protocol_version` -* `network_id` -* `latest_finalized_root` -* `latest_finalized_epoch` -* `best_root` -* `best_slot` - -### Beacon Blocks - -Supported pubsub topics: -* `beacon` - -The following definitions are used in the API: -* `block_header`: a serialized `BeaconBlock` in which `BeaconBlock.body` is the `hash_tree_root` of the associated `BeaconBlockBody`. -* `block_body`: a serialied `BeaconBlockBody`. -* `block_root`: the `hash_tree_root` of a `BeaconBlock`. - -API: -* `send_beacon_block_roots(block_roots)`: Sends list of `block_roots` to peer. -* `send_beacon_block_headers(block_headers)`: Sends list of `block_headers` to peer. -* `request_beacon_block_headers(block_roots)`: Requests the associated `block_headers` for the given `block_roots` from peer. -* `send_beacon_block_bodies(block_bodies)`: Sends list of `block_bodies` to peer. -* `request_beacon_block_bodies(block_roots)`: Requests the associated `block_bodies` for the given `block_roots` from peer. - -Notes: -* It is assumed that both the associated `BeaconBlock` and `BeaconBlockBody` can be looked up via `block_root`. - -### Attestations - -Supported pubsub topics: -* `beacon` -* all `shard-{number}` topics - -The following definitions are used in the API: -* `attestation`: a serialized `Attestation` with full serialized `AttestationData` for `Attestation.data`. - -API: -* `send_attestations(attestations)`: Sends list of `attestations` to peer. - -Notes: -* It is expected that an attestation is only broadcast to either `beacon` topic or `shard-{attestation.data.shard}` topic. Broadcasting to mismatched shard topics is considered bad form. -* It is expected that only aggregate attestations are broadcast to the `beacon` topic. Repeated broadcasting of attestations with a signle signer to the `beacon` topic is considered bad form. -* There is a shard subnet design decision here. Due to the likelihood of `attestation.data` to be highly repeated across a committee during a given slot, it could be valuable to just pass the `attestation` with a `root` in the `attestation.data` field. If the recipient does not already have an `AttestationData` for the received `root`, then the recipient would explicitly request the root. This reduces the total data passed by 184 bytes in the case that the recipient has already received the `attestation.data` but increases the rounds of communication when they haven't. -* We do not currently specify a getter method for an attestation by its `root`. Due to the diverse ways attestations might both be aggregated and stored, it is not feasible to reliably lookup via a `root`. The attestations that a client cares about are (1) those that made it on-chain into a `BeaconBlock` and (2) the most recent set of attestations being actively broadcast on the wire. We might provide a `request_attestations(slot)` or `request_attestations(epoch)` but do not provide it in this minimal API specification. - -### Exits - -Supported pubsub topics: -* `beacon` - -The following definitions are used in the API: -* `exit`: a serialized `Exit`. - -API: -* `send_exit(exit)`: Sends `exit` to peer. - -Notes: -* We do not specify a getter for an exit by its `root`. Standard usage is for \ No newline at end of file From 334d47714df2b3f41b82ad2adea2aa6636369c3e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 10:35:57 -0700 Subject: [PATCH 66/96] fix a couple of nitpicks before release --- specs/core/0_beacon-chain.md | 7 ++++--- specs/validator/0_beacon-chain-validator.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 21a991c42..ebc12324c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -658,9 +658,10 @@ def get_previous_epoch(state: BeaconState) -> EpochNumber: Return the previous epoch of the given ``state``. If the current epoch is ``GENESIS_EPOCH``, return ``GENESIS_EPOCH``. """ - if slot_to_epoch(state.slot) > GENESIS_EPOCH: - return slot_to_epoch(state.slot) - 1 - return slot_to_epoch(state.slot) + current_epoch = get_current_epoch(state) + if current_epoch == GENESIS_EPOCH: + return GENESIS_EPOCH + return current_epoch - 1 ``` ### `get_current_epoch` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d1246c6fa..5b8a93af9 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -385,7 +385,7 @@ def get_next_epoch_committee_assignments( return potential_assignments ``` -`get_next_epoch_committee_assignments` should be called at the beginning of each epoch to plan for the next epoch. A validator should always plan for both values of `registry_change` as a possibility unless the validator can concretely eliminate one of the options. Planning for a future shuffling involves noting at which slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). +`get_next_epoch_committee_assignments` should be called at the start of each epoch to get potential assignments for the next epoch (slots during `current_epoch + 1`). A validator should always plan for both values of `registry_change` as a possibility unless the validator can concretely eliminate one of the options. Planning for future assignments involves noting at which future slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). ## How to avoid slashing From 663d38e9c8382e110022de6fcc4ca191ecadd40e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 11:54:17 -0700 Subject: [PATCH 67/96] simplify get_next_epoch_committee_assignment by adding registry_change arg --- specs/validator/0_beacon-chain-validator.md | 54 ++++++++++----------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 5b8a93af9..93c3fb408 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -341,15 +341,16 @@ There are three possibilities for the shuffling at the next epoch: Either (2) or (3) occurs if (1) fails. The choice between (2) and (3) is deterministic based upon `epochs_since_last_registry_update`. -`get_crosslink_committees_at_slot` is designed to be able to query slots in the next epoch. When querying slots in the next epoch there are two options -- with and without a `registry_change` -- which is the optional third parameter of the function. The following helper can be used to get the potential crosslink committees in the next epoch for a given `validator_index`. This function returns a list of 2 shard committee tuples. +`get_crosslink_committees_at_slot` is designed to be able to query slots in the next epoch. When querying slots in the next epoch there are two options -- with and without a `registry_change` -- which is the optional third parameter of the function. The following helper can be used to get the potential crosslink committee assignments in the next epoch for a given `validator_index` and `registry_change`. ```python -def get_next_epoch_committee_assignments( +def get_next_epoch_committee_assignment( state: BeaconState, - validator_index: ValidatorIndex) -> List[Tuple[List[ValidatorIndex], ShardNumber, SlotNumber, bool]]: + validator_index: ValidatorIndex, + registry_change: bool) -> Tuple[List[ValidatorIndex], ShardNumber, SlotNumber, bool]: """ - Return a list of the two possible committee assignments for ``validator_index`` at the next epoch. - Possible committee ``assignment`` is of the form (List[ValidatorIndex], ShardNumber, SlotNumber, bool). + Return the committee assignment in the next epoch for ``validator_index`` and ``registry_change``. + ``assignment`` returned is a tuple of the following form: * ``assignment[0]`` is the list of validators in the committee * ``assignment[1]`` is the shard to which the committee is assigned * ``assignment[2]`` is the slot at which the committee is assigned @@ -359,33 +360,28 @@ def get_next_epoch_committee_assignments( current_epoch = get_current_epoch(state) next_epoch = current_epoch + 1 next_epoch_start_slot = get_epoch_start_slot(next_epoch) - potential_assignments = [] - for registry_change in [False, True]: - for slot in range(next_epoch_start_slot, next_epoch_start_slot + EPOCH_LENGTH): - crosslink_committees = get_crosslink_committees_at_slot( - state, - slot, - registry_change=registry_change, - ) - selected_committees = [ - committee # Tuple[List[ValidatorIndex], ShardNumber] - for committee in crosslink_committees - if validator_index in committee[0] - ] - if len(selected_committees) > 0: - assignment = selected_committees[0] - assignment += (slot,) - first_committee_at_slot = crosslink_committees[0][0] # List[ValidatorIndex] - is_proposer = first_committee_at_slot[slot % len(first_committee_at_slot)] == validator_index - assignment += (is_proposer,) + for slot in range(next_epoch_start_slot, next_epoch_start_slot + EPOCH_LENGTH): + crosslink_committees = get_crosslink_committees_at_slot( + state, + slot, + registry_change=registry_change, + ) + selected_committees = [ + committee # Tuple[List[ValidatorIndex], ShardNumber] + for committee in crosslink_committees + if validator_index in committee[0] + ] + if len(selected_committees) > 0: + validators = selected_committees[0][0] + shard = selected_committees[0][1] + first_committee_at_slot = crosslink_committees[0][0] # List[ValidatorIndex] + is_proposer = first_committee_at_slot[slot % len(first_committee_at_slot)] == validator_index - potential_assignments.append(assignment) - break - - return potential_assignments + assignment = (validators, shard, slot, is_proposer) + return assignment ``` -`get_next_epoch_committee_assignments` should be called at the start of each epoch to get potential assignments for the next epoch (slots during `current_epoch + 1`). A validator should always plan for both values of `registry_change` as a possibility unless the validator can concretely eliminate one of the options. Planning for future assignments involves noting at which future slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). +`get_next_epoch_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (slots during `current_epoch + 1`). A validator should always plan for assignments from both values of `registry_change` unless the validator can concretely eliminate one of the options. Planning for future assignments involves noting at which future slot one might have to attest and propose and also which shard one should begin syncing (in phase 1+). ## How to avoid slashing From 057364a7d4552334060256679b89f43dd38dec10 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 19:37:34 -0600 Subject: [PATCH 68/96] Add mandatory deposit index ordering Co-requisite with #589 --- specs/core/0_beacon-chain.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ebc12324c..78388cad2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -526,6 +526,7 @@ The following data structures are defined as [SimpleSerialize (SSZ)](https://git # Ethereum 1.0 chain data 'latest_eth1_data': Eth1Data, 'eth1_data_votes': [Eth1DataVote], + 'deposit_count': 'uint64' } ``` @@ -1717,6 +1718,7 @@ Verify that `len(block.body.deposits) <= MAX_DEPOSITS`. For each `deposit` in `block.body.deposits`: * Let `serialized_deposit_data` be the serialized form of `deposit.deposit_data`. It should be 8 bytes for `deposit_data.amount` followed by 8 bytes for `deposit_data.timestamp` and then the `DepositInput` bytes. That is, it should match `deposit_data` in the [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) of which the hash was placed into the Merkle tree. +* Verify that `deposit.index == state.deposit_index`. * Verify that `verify_merkle_branch(hash(serialized_deposit_data), deposit.branch, DEPOSIT_CONTRACT_TREE_DEPTH, deposit.index, state.latest_eth1_data.deposit_root)` is `True`. ```python @@ -1745,6 +1747,8 @@ process_deposit( ) ``` +* Set `state.deposit_index += 1`. + ##### Exits Verify that `len(block.body.exits) <= MAX_EXITS`. From 36d5120deb20d24546c8877aea212b6e961df0e3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 19:38:09 -0600 Subject: [PATCH 69/96] Initialize deposit index --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 78388cad2..a7e2b6286 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1479,6 +1479,7 @@ def get_initial_beacon_state(initial_validator_deposits: List[Deposit], # Ethereum 1.0 chain data latest_eth1_data=latest_eth1_data, eth1_data_votes=[], + deposit_index=0 ) # Process initial deposits From 019fe8953e6aabd2176f93ab00b96904966882ca Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 22:02:55 -0600 Subject: [PATCH 70/96] Fixed as per @djrtwo's comments --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a7e2b6286..da2cce788 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -526,7 +526,7 @@ The following data structures are defined as [SimpleSerialize (SSZ)](https://git # Ethereum 1.0 chain data 'latest_eth1_data': Eth1Data, 'eth1_data_votes': [Eth1DataVote], - 'deposit_count': 'uint64' + 'deposit_index': 'uint64' } ``` @@ -1479,7 +1479,7 @@ def get_initial_beacon_state(initial_validator_deposits: List[Deposit], # Ethereum 1.0 chain data latest_eth1_data=latest_eth1_data, eth1_data_votes=[], - deposit_index=0 + deposit_index=len(initial_validator_deposits) ) # Process initial deposits From ad0ff80be2b0f3682abf4134bc1660a566790c46 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 22:06:03 -0600 Subject: [PATCH 71/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b4bbf3a7e..e910e24ac 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -122,7 +122,7 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Verify that `shard_block.beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. * Verify that `shard_block.beacon_chain_ref` is equal to or a descendant of the `shard_block.beacon_chain_ref` specified in the `ShardBlock` pointed to by `shard_block.parent_root`. * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. -* Let `persistent_committee` be `[persistent_committee[i] for i in get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. +* Let `persistent_committee = get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` * Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot)) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, shard_block.slot, SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. From 8dd111b7e62ba768c54d2af875dadb442303b103 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 22:06:46 -0600 Subject: [PATCH 72/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e910e24ac..9362d537e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -124,7 +124,7 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. * Let `persistent_committee = get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` -* Let `proposer_index = hash(state.randao_mix + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot)) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, shard_block.slot, SHARD_PROPOSER_DOMAIN))` passes. +* Let `proposer_index = bytes_to_int(hash(state.current_epoch_seed + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot))[0:8]) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. ### Verifying shard block data From e0d8cf42688b1011121cbc5f3b586e0513ab507c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 8 Feb 2019 22:07:14 -0600 Subject: [PATCH 73/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9362d537e..2752765c5 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -125,7 +125,7 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Let `persistent_committee = get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` * Let `proposer_index = bytes_to_int(hash(state.current_epoch_seed + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot))[0:8]) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. -* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot, SHARD_ATTESTER_DOMAIN))` passes. +* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. ### Verifying shard block data From 65c5a1a1b76d112e0d6ec2f7ac6fe1bf61ec6e27 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 8 Feb 2019 22:10:54 -0600 Subject: [PATCH 74/96] Fixed get_persistent_committee --- specs/core/1_shard-data-chains.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 2752765c5..132a3b7bc 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -45,13 +45,15 @@ def get_split_offset(list_size: int, chunks: int, index: int) -> int: #### get_persistent_committee ```python -def get_persistent_commmitee(seed: Bytes32, - validators: List[Validator], - shard: ShardNumber, - epoch: EpochNumber) -> List[ValidatorIndex]: +def get_persistent_commmitee(state: BeaconState, + shard: ShardNumber, + epoch: EpochNumber) -> List[ValidatorIndex]: + """ + Returns the persistent committee for the given shard at the given epoch + """ earlier_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - earlier_validator_set = get_active_validator_indices(validators, earlier_committee_start) + earlier_validator_set = get_active_validator_indices(state.validators, earlier_committee_start) earlier_seed = generate_seed(state, earlier_committee_start) earlier_start_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard) earlier_end_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard+1) @@ -61,7 +63,7 @@ def get_persistent_commmitee(seed: Bytes32, ] later_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD - later_validator_set = get_active_validator_indices(validators, later_committee_start) + later_validator_set = get_active_validator_indices(state.validators, later_committee_start) later_seed = generate_seed(state, later_committee_start) later_start_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard) later_end_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard+1) From 0157aa039cbef094facb32d35aa5c5feb1297325 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 9 Feb 2019 11:10:03 +0300 Subject: [PATCH 75/96] Change var name to avoid confusing the reader into thinking the `valid_proof` variable contains a valid proof as opposed to just being a bool. Co-Authored-By: Nashatyrev --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c08343ef9..604bab0f0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1252,7 +1252,7 @@ def process_deposit(state: BeaconState, Note that this function mutates ``state``. """ # Validate the given `proof_of_possession` - valid_proof = validate_proof_of_possession( + proof_is_valid = validate_proof_of_possession( state, pubkey, proof_of_possession, From e48010b77b9604104500e44e2100909d4575e0be Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 9 Feb 2019 11:10:54 +0300 Subject: [PATCH 76/96] Change var name to avoid confusing the reader into thinking the `valid_proof` variable contains a valid proof as opposed to just being a bool. Co-Authored-By: Nashatyrev --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 604bab0f0..03fa7c801 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1259,7 +1259,7 @@ def process_deposit(state: BeaconState, withdrawal_credentials, ) - if valid_proof: + if proof_is_valid: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: From 7886d9618606cfad88c5c8f9df2b99feecb1b79a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 9 Feb 2019 07:21:38 -0800 Subject: [PATCH 77/96] change proof_is_valid to exit condition --- specs/core/0_beacon-chain.md | 50 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 03fa7c801..d53ebc901 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1259,30 +1259,32 @@ def process_deposit(state: BeaconState, withdrawal_credentials, ) - if proof_is_valid: - validator_pubkeys = [v.pubkey for v in state.validator_registry] - - if pubkey not in validator_pubkeys: - # Add new validator - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=withdrawal_credentials, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawal_epoch=FAR_FUTURE_EPOCH, - penalized_epoch=FAR_FUTURE_EPOCH, - status_flags=0, - ) - - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. - state.validator_registry.append(validator) - state.validator_balances.append(amount) - else: - # Increase balance by deposit amount - index = validator_pubkeys.index(pubkey) - assert state.validator_registry[index].withdrawal_credentials == withdrawal_credentials - - state.validator_balances[index] += amount + if not proof_is_valid: + return + + validator_pubkeys = [v.pubkey for v in state.validator_registry] + + if pubkey not in validator_pubkeys: + # Add new validator + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=withdrawal_credentials, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawal_epoch=FAR_FUTURE_EPOCH, + penalized_epoch=FAR_FUTURE_EPOCH, + status_flags=0, + ) + + # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. + state.validator_registry.append(validator) + state.validator_balances.append(amount) + else: + # Increase balance by deposit amount + index = validator_pubkeys.index(pubkey) + assert state.validator_registry[index].withdrawal_credentials == withdrawal_credentials + + state.validator_balances[index] += amount ``` ### Routines for updating validator status From 8097b2373b1d0462ddc02309ceff4a9db958d23d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 00:09:34 -0600 Subject: [PATCH 78/96] Added deduplication --- specs/core/1_shard-data-chains.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 132a3b7bc..5213f7da7 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -77,11 +77,13 @@ def get_persistent_commmitee(state: BeaconState, bytes_to_int(hash(earlier_seed + bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) - - return ( + + # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from + # later committee; return a sorted list of the union of the two, deduplicated + return sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] - ) + ))) ``` ## Data Structures From abed5ffdaef901c49dcc51eeb4b1df6d6d7d8b38 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 10:16:22 -0600 Subject: [PATCH 79/96] Updated phase 1: fork choice rule (#586) Adds the crosslink committee to the fork choice rule. This is useful because it means that even if a proposal committee is byzantine and attempts to prevent a crosslink via a "balance attack" (alternating between chain A and chain B being the canonical chain), the crosslink committee can force the equilibrium to flip to one side or the other. --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index cabe2934e..a8d19ddd3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -121,4 +121,4 @@ Verify that the `shard_block_combined_data_root` is the output of these function ### Shard block fork choice rule -The fork choice rule for any shard is LMD GHOST using the validators currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (ie. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_ref` is the block in the main beacon chain at the specified `slot` should be considered (if the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than a slot). +The fork choice rule for any shard is LMD GHOST using the shard chain attestations of the persistent committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (ie. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_ref` is the block in the main beacon chain at the specified `slot` should be considered (if the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than a slot). From 1f97206dcf843613b43f6b99a32dc29cb9cb7bcf Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 10:17:21 -0600 Subject: [PATCH 80/96] Updated phase 1: commitments (#579) See #338 and #529 for discussion. --- specs/core/1_shard-data-chains.md | 40 +++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a8d19ddd3..352b89b40 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -20,6 +20,7 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md |------------------------|-----------------|-------|---------------| | `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | | `SHARD_BLOCK_SIZE` | 2**14 (= 16384) | bytes | | +| `CROSSLINK_LOOKBACK` | 2**5 (= 32) | slots | | ### Flags, domains, etc. @@ -98,26 +99,39 @@ A node should sign a crosslink only if the following conditions hold. **If a nod First, the conditions must recursively apply to the crosslink referenced in `last_crosslink_root` for the same shard (unless `last_crosslink_root` equals zero, in which case we are at the genesis). -Second, we verify the `shard_block_combined_data_root`. Let `h` be the slot _immediately after_ the slot of the shard block included by the last crosslink, and `h+n-1` be the slot number of the block directly referenced by the current `shard_block_root`. Let `B[i]` be the block at slot `h+i` in the shard chain. Let `bodies[0] .... bodies[n-1]` be the bodies of these blocks and `roots[0] ... roots[n-1]` the data roots. If there is a missing slot in the shard chain at position `h+i`, then `bodies[i] == b'\x00' * shard_block_maxbytes(state[i])` and `roots[i]` be the Merkle root of the empty data. Define `compute_merkle_root` be a simple Merkle root calculating function that takes as input a list of objects, where the list's length must be an exact power of two. We define the function for computing the combined data root as follows: +Second, we verify the `shard_chain_commitment`. +* Let `start_slot = state.latest_crosslinks[shard].epoch * EPOCH_LENGTH + EPOCH_LENGTH - CROSSLINK_LOOKBACK`. +* Let `end_slot = attestation.data.slot - attestation.data.slot % EPOCH_LENGTH - CROSSLINK_LOOKBACK`. +* Let `length = end_slot - start_slot`, `headers[0] .... headers[length-1]` be the serialized block headers in the canonical shard chain from the verifer's point of view (note that this implies that `headers` and `bodies` have been checked for validity). +* Let `bodies[0] ... bodies[length-1]` be the bodies of the blocks. +* Note: If there is a missing slot, then the header and body are the same as that of the block at the most recent slot that has a block. + +We define two helpers: ```python -ZERO_ROOT = merkle_root(bytes([0] * SHARD_BLOCK_SIZE)) - -def mk_combined_data_root(roots): - data = roots + [ZERO_ROOT for _ in range(len(roots), next_power_of_2(len(roots)))] - return compute_merkle_root(data) +def pad_to_power_of_2(values: List[bytes]) -> List[bytes]: + while not is_power_of_two(len(values)): + values = values + [SHARD_BLOCK_SIZE] + return values ``` -This outputs the root of a tree of the data roots, with the data roots all adjusted to have the same height if needed. The tree can also be viewed as a tree of all of the underlying data concatenated together, appropriately padded. Here is an equivalent definition that uses bodies instead of roots [TODO: check equivalence]: - ```python -def mk_combined_data_root(depths, bodies): - data = b''.join(bodies) - data += bytes([0] * (next_power_of_2(len(data)) - len(data)) - return compute_merkle_root([data[pos:pos+SHARD_CHUNK_SIZE] for pos in range(0, len(data), SHARD_CHUNK_SIZE)]) +def merkle_root_of_bytes(data: bytes) -> bytes: + return merkle_root([data[i:i+32] for i in range(0, len(data), 32)]) ``` -Verify that the `shard_block_combined_data_root` is the output of these functions. +We define the function for computing the commitment as follows: + +```python +def compute_commitment(headers: List[ShardBlock], bodies: List[bytes]) -> Bytes32: + return hash( + merkle_root(pad_to_power_of_2([merkle_root_of_bytes(zpad(serialize(h), SHARD_BLOCK_SIZE)) for h in headers])), + merkle_root(pad_to_power_of_2([merkle_root_of_bytes(h) for h in bodies])) + ) +``` + +The `shard_chain_commitment` is only valid if it equals `compute_commitment(headers, bodies)`. + ### Shard block fork choice rule From 2944a7ddfc67ededa4c3ad6f6a4f28b5a18b35ac Mon Sep 17 00:00:00 2001 From: Dean Eigenmann Date: Sun, 10 Feb 2019 17:19:12 +0100 Subject: [PATCH 81/96] Renamed `Helpers` -> `Variables`, seems to make more sense (#560) --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 78d24cde3..3c89cf1a9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -125,7 +125,7 @@ - [Deposits](#deposits-1) - [Exits](#exits-1) - [Per-epoch processing](#per-epoch-processing) - - [Helpers](#helpers) + - [Helper variables](#helper-variables) - [Eth1 data](#eth1-data-1) - [Justification](#justification) - [Crosslinks](#crosslinks) @@ -1770,7 +1770,7 @@ For each `exit` in `block.body.exits`: The steps below happen when `(state.slot + 1) % EPOCH_LENGTH == 0`. -#### Helpers +#### Helper variables * Let `current_epoch = get_current_epoch(state)`. * Let `previous_epoch = get_previous_epoch(state)`. From 046119fb2eb4252b4353edd95c70181fb40299e3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 15:44:58 -0600 Subject: [PATCH 82/96] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a4a67cd8b..c470395df 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -86,7 +86,27 @@ def get_persistent_commmitee(state: BeaconState, [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)] ))) ``` +#### get_shard_proposer_index +```python +def get_shard_proposer_index(state: BeaconState, + shard: ShardNumber, + slot: SlotNumber) -> ValidatorIndex: + seed = hash( + state.current_epoch_seed + + int_to_bytes8(shard) + + int_to_bytes8(slot) + ) + persistent_committee = get_persistent_committee(state, shard, slot_to_epoch(slot)) + # Default proposer + index = bytes_to_int(seed[0:8]) % len(persistent_committee) + # If default proposer exits, try the other proposers in order; if all are exited + # return None (ie. no block can be proposed) + validators_to_try = persistent_committee[index:] + persistent_committee[:index] + for index in validators_to_try: + if is_active_validator(state.validators[index], get_current_epoch(state)): + return index + return None ## Data Structures ### Shard chain blocks From ad3f43a4a6dfe472cc65fa9a2c5c934e18a45e24 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 10 Feb 2019 15:45:06 -0600 Subject: [PATCH 83/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index c470395df..cbbd98048 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -147,7 +147,7 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Verify that `shard_block.beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`. * Verify that `shard_block.beacon_chain_ref` is equal to or a descendant of the `shard_block.beacon_chain_ref` specified in the `ShardBlock` pointed to by `shard_block.parent_root`. * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. -* Let `persistent_committee = get_persistent_committee(state, shard_block.slot, shard_block.shard_id)`. +* Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, slot_to_epoch(shard_block.slot))`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` * Let `proposer_index = bytes_to_int(hash(state.current_epoch_seed + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot))[0:8]) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. From 9e66b069b223d714ca3e1da4c2f412f86d895367 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 15:45:16 -0600 Subject: [PATCH 84/96] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index cbbd98048..d07c6b1ed 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -149,6 +149,7 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`. * Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, slot_to_epoch(shard_block.slot))`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` +* For every `i in range(len(persistent_committee))` where `is_active_validator(state.validators[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0` * Let `proposer_index = bytes_to_int(hash(state.current_epoch_seed + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot))[0:8]) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. From 9bba3362eb18c9bcaff8353c34e254df7c628007 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 15:45:47 -0600 Subject: [PATCH 85/96] Update specs/core/1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d07c6b1ed..29d47dfed 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -150,7 +150,10 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, slot_to_epoch(shard_block.slot))`. * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))` * For every `i in range(len(persistent_committee))` where `is_active_validator(state.validators[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0` -* Let `proposer_index = bytes_to_int(hash(state.current_epoch_seed + int_to_bytes8(shard_block.shard_id) + int_to_bytes8(shard_block.slot))[0:8]) % len(validators)`. Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. +* Let `proposer_index = get_shard_proposer_index(state, shard_block.shard_id, shard_block.slot)`. +* Verify that `proposer_index` is not `None`. +* Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. +* Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. * Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. ### Verifying shard block data From ab44cbe380b57afaeb2121307cc4a83ae2f7b77c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 10 Feb 2019 15:45:57 -0600 Subject: [PATCH 86/96] Update specs/core/1_shard-data-chains.md Co-Authored-By: vbuterin --- specs/core/1_shard-data-chains.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 29d47dfed..0c90e0028 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -154,7 +154,8 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows: * Verify that `proposer_index` is not `None`. * Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`. * Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_PROPOSER_DOMAIN))` passes. -* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. +* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`. +* Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), SHARD_ATTESTER_DOMAIN))` passes. ### Verifying shard block data From db4f99d8992ab05236ba8ec35c23f5800ad677cf Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 10 Feb 2019 15:47:26 -0600 Subject: [PATCH 87/96] Fixed end of code block --- specs/core/1_shard-data-chains.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 0c90e0028..a4cf36134 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -107,6 +107,8 @@ def get_shard_proposer_index(state: BeaconState, if is_active_validator(state.validators[index], get_current_epoch(state)): return index return None +``` + ## Data Structures ### Shard chain blocks From afca1e21278260aba51766ed380abcca42efddea Mon Sep 17 00:00:00 2001 From: Jordan Andrews Date: Mon, 11 Feb 2019 22:01:05 +1100 Subject: [PATCH 88/96] Add C# implementation --- specs/simple-serialize.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b888ca76f..690974dde 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -452,6 +452,7 @@ return hash(b''.join([hash_tree_root_internal(getattr(x, field)) for field in va | Java | [ https://www.github.com/ConsenSys/cava/tree/master/ssz ](https://www.github.com/ConsenSys/cava/tree/master/ssz) | SSZ Java library part of the Cava suite | | Go | [ https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz ](https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz) | Go implementation of SSZ mantained by Prysmatic Labs | | Swift | [ https://github.com/yeeth/SimpleSerialize.swift ](https://github.com/yeeth/SimpleSerialize.swift) | Swift implementation maintained SSZ | +| C# | [ https://github.com/codingupastorm/csharp-ssz ](https://github.com/codingupastorm/csharp-ssz) | C# implementation maintained SSZ | ## Copyright Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). From 22755871491913ec9c4ddcf14a7115d520d76c6c Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 11 Feb 2019 13:29:54 +0000 Subject: [PATCH 89/96] Add note on leap seconds Clarify the local time is Unix time and add note on leap seconds. --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3c89cf1a9..dea56cc08 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -167,7 +167,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be * **Shard chain** - one of the chains on which user transactions take place and account data is stored. * **Block root** - a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". * **Crosslink** - a set of signatures from a committee attesting to a block in a shard chain, which can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. -* **Slot** - a period of `SLOT_DURATION` seconds, during which one proposer has the ability to create a beacon chain block and some attesters have the ability to make attestations +* **Slot** - a period during which one proposer has the ability to create a beacon chain block and some attesters have the ability to make attestations * **Epoch** - an aligned span of slots during which all [validators](#dfn-validator) get exactly one chance to make an attestation * **Finalized**, **justified** - see Casper FFG finalization [[casper-ffg]](#ref-casper-ffg) * **Withdrawal period** - the number of slots between a [validator](#dfn-validator) exit and the [validator](#dfn-validator) balance being withdrawable @@ -1522,7 +1522,7 @@ For a beacon chain block, `block`, to be processed by a node, the following cond * The parent block with root `block.parent_root` has been processed and accepted. * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. -* The node's local clock time is greater than or equal to `state.genesis_time + block.slot * SLOT_DURATION`. +* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SLOT_DURATION`. (Note that leap seconds mean that slots will occasionally last `SLOT_DURATION + 1` or `SLOT_DURATION - 1` seconds, possibly several times a year.) If these conditions are not met, the client should delay processing the beacon block until the conditions are all satisfied. From 0069241b9c20ddbb30262a23b6bf1a548841eaf2 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 11 Feb 2019 14:49:11 +0000 Subject: [PATCH 90/96] Alias `bytes` to `List[bytes1]` --- specs/simple-serialize.md | 91 ++++++++++----------------------------- 1 file changed, 22 insertions(+), 69 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 690974dde..65f1a7372 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -9,28 +9,24 @@ deserializing objects and data types. ## ToC * [About](#about) -* [Terminology](#terminology) +* [Variables and Functions](#variables-and-functions) * [Constants](#constants) * [Overview](#overview) + [Serialize/Encode](#serializeencode) - - [uint](#uint) - - [Bool](#bool) - - [Bytes](#bytes) - - [bytesN](#bytesn) - - [bytes](#bytes-1) + - [uintN](#uintn) + - [bool](#bool) + - [bytesN](#bytesn) - [List/Vectors](#listvectors) - [Container](#container) + [Deserialize/Decode](#deserializedecode) - - [uint](#uint-1) - - [Bool](#bool-1) - - [Bytes](#bytes-2) - - [bytesN](#bytesn-1) - - [bytes](#bytes-1) + - [uintN](#uintn-1) + - [bool](#bool-1) + - [bytesN](#bytesn-1) - [List/Vectors](#listvectors-1) - [Container](#container-1) + [Tree Hash](#tree-hash) - [`uint8`..`uint256`, `bool`, `bytes1`..`bytes32`](#uint8uint256-bool-bytes1bytes32) - - [`uint264`..`uintN`, `bytes`, `bytes33`..`bytesN`](#uint264uintn-bytes-bytes33bytesn) + - [`uint264`..`uintN`, `bytes33`..`bytesN`](#uint264uintn-bytes33bytesn) - [List/Vectors](#listvectors-2) - [Container](#container-2) * [Implementations](#implementations) @@ -68,11 +64,11 @@ overhead. ### Serialize/Encode -#### uint +#### uintN | uint Type | Usage | |:---------:|:-----------------------------------------------------------| -| `uintN` | Type of `N` bits unsigned integer, where ``N % 8 == 0``. | +| `uintN` | Type of `N` bits unsigned integer, where ``N % 8 == 0``. | Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``) @@ -88,7 +84,7 @@ buffer_size = int_size / 8 return value.to_bytes(buffer_size, 'little') ``` -#### Bool +#### bool Convert directly to a single 0x00 or 0x01 byte. @@ -101,18 +97,13 @@ assert(value in (True, False)) return b'\x01' if value is True else b'\x00' ``` -#### Bytes +#### bytesN -| Bytes Type | Usage | -|:---------:|:------------------------------------| -| `bytesN` | Explicit length `N` bytes data. | -| `bytes` | Bytes data with arbitrary length. | - -##### bytesN +A fixed-size byte array. | Checks to perform | Code | |:---------------------------------------|:---------------------| -| Length in bytes is correct for `bytesN` | ``len(value) == N`` | +| Length in bytes is correct for `bytesN` | ``len(value) == N`` | ```python assert(len(value) == N) @@ -120,21 +111,6 @@ assert(len(value) == N) return value ``` -##### bytes -For general `bytes` type: -1. Get the length/number of bytes; Encode into a `4-byte` integer. -2. Append the value to the length and return: ``[ length_bytes ] + [ value_bytes ]`` - -| Check to perform | Code | -|:-------------------------------------|:-----------------------| -| Length of bytes can fit into 4 bytes | ``len(value) < 2**32`` | - -```python -assert(len(value) < 2**32) -byte_length = (len(value)).to_bytes(LENGTH_BYTES, 'little') -return byte_length + value -``` - #### List/Vectors Lists are a collection of elements of the same homogeneous type. @@ -146,6 +122,8 @@ Lists are a collection of elements of the same homogeneous type. 1. Serialize all list elements individually and concatenate them. 2. Prefix the concatenation with its length encoded as a `4-byte` **little-endian** unsigned integer. +We define `bytes` to be a synonym of `List[bytes1]`. + **Example in Python** ```python @@ -168,8 +146,8 @@ A container represents a heterogenous, associative collection of key-value pairs To serialize a container, obtain the list of its field's names in the specified order. For each field name in this list, obtain the corresponding value and serialize it. Tightly pack the complete set of serialized values in the same order as the field names into a buffer. Calculate the size of this buffer of serialized bytes and encode as a `4-byte` **little endian** `uint32`. Prepend the encoded length to the buffer. The result of this concatenation is the final serialized value of the container. -| Check to perform | Code | -|:--------------------------------------------|:----------------------------| +| Check to perform | Code | +|:----------------------------------------------|:----------------------------| | Length of serialized fields fits into 4 bytes | ``len(serialized) < 2**32`` | To serialize: @@ -231,7 +209,7 @@ At the final step, the following checks should be made: |:-------------------------|:-------------------------------------| | Ensure no extra length | `new_index == len(rawbytes)` | -#### uint +#### uintN Convert directly from bytes into integer utilising the number of bytes the same size as the integer length. (e.g. ``uint16 == 2 bytes``) @@ -245,7 +223,7 @@ assert(len(rawbytes) >= new_index) return int.from_bytes(rawbytes[current_index:current_index+byte_length], 'little'), new_index ``` -#### Bool +#### bool Return True if 0x01, False if 0x00. @@ -254,9 +232,7 @@ assert rawbytes in (b'\x00', b'\x01') return True if rawbytes == b'\x01' else False ``` -#### Bytes - -##### bytesN +#### bytesN Return the `N` bytes. @@ -266,28 +242,6 @@ new_index = current_index + N return rawbytes[current_index:current_index+N], new_index ``` -##### bytes - -Get the length of the bytes, return the bytes. - -| Check to perform | code | -|:--------------------------------------------------|:-------------------------------------------------| -| rawbytes has enough left for length | ``len(rawbytes) > current_index + LENGTH_BYTES`` | -| bytes to return not greater than serialized bytes | ``len(rawbytes) > bytes_end `` | - -```python -assert(len(rawbytes) > current_index + LENGTH_BYTES) -bytes_length = int.from_bytes(rawbytes[current_index:current_index + LENGTH_BYTES], 'little') - -bytes_start = current_index + LENGTH_BYTES -bytes_end = bytes_start + bytes_length -new_index = bytes_end - -assert(len(rawbytes) >= bytes_end) - -return rawbytes[bytes_start:bytes_end], new_index -``` - #### List/Vectors Deserialize each element in the list. @@ -295,7 +249,6 @@ Deserialize each element in the list. 2. Loop through deserializing each item in the list until you reach the entire length of the list. - | Check to perform | code | |:------------------------------------------|:----------------------------------------------------------------| | ``rawbytes`` has enough left for length | ``len(rawbytes) > current_index + LENGTH_BYTES`` | @@ -384,7 +337,7 @@ Refer to [the helper function `hash`](https://github.com/ethereum/eth2.0-specs/b Return the serialization of the value. -#### `uint264`..`uintN`, `bytes`, `bytes33`..`bytesN` +#### `uint264`..`uintN`, `bytes33`..`bytesN` Return the hash of the serialization of the value. From 98902d12e33873f4e3aec7dd5e100379ba72934f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 11 Feb 2019 14:30:36 -0700 Subject: [PATCH 91/96] assert/unhandled exception note to state transition (#606) --- specs/core/0_beacon-chain.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dea56cc08..0f020be53 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -154,7 +154,7 @@ The primary source of load on the beacon chain is "attestations". Attestations a ## Notation -Code snippets appearing in `this style` are to be interpreted as Python code. Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed asserts are considered invalid. +Code snippets appearing in `this style` are to be interpreted as Python code. ## Terminology @@ -1593,6 +1593,8 @@ We now define the state transition function. At a high level the state transitio The per-slot transitions focus on the slot counter and block roots records updates; the per-block transitions generally focus on verifying aggregate signatures and saving temporary records relating to the per-block activity in the `BeaconState`; the per-epoch transitions focus on the [validator](#dfn-validator) registry, including adjusting balances and activating and exiting [validators](#dfn-validator), as well as processing crosslinks and managing block justification/finalization. +Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. + _Note_: If there are skipped slots between a block and its parent block, run the steps in the [per-slot](#per-slot-processing) and [per-epoch](#per-epoch-processing) sections once for each skipped slot and then once for the slot containing the new block. ### Per-slot processing From 3ea0c27be01650d2269e1b7bfb64c8cfc1242de0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 12 Feb 2019 22:06:26 +1100 Subject: [PATCH 92/96] Minor modification to reduce lines of code (#607) --- specs/core/0_beacon-chain.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0f020be53..11f5007ee 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -2030,12 +2030,10 @@ def process_penalties_and_exits(state: BeaconState) -> None: eligible_indices = filter(eligible, all_indices) # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) - withdrawn_so_far = 0 - for index in sorted_indices: - prepare_validator_for_withdrawal(state, index) - withdrawn_so_far += 1 + for withdrawn_so_far, index in enumerate(sorted_indices): if withdrawn_so_far >= MAX_WITHDRAWALS_PER_EPOCH: break + prepare_validator_for_withdrawal(state, index) ``` #### Final updates From de90d4475e30db83669e72b787c408f37b4f9d2a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 12 Feb 2019 22:09:27 +1100 Subject: [PATCH 93/96] Provide definition for undefined `index` (#608) The `index` variable is not defined and, IMO, it's difficult to discern. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 11f5007ee..0f0ad8f99 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1900,7 +1900,7 @@ For each `index` in `previous_epoch_attester_indices`, we determine the proposer For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(current_epoch))`: * Let `crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)`. -* For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot`: +* For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot` and every `index` in `crosslink_committee`: * If `index in attesting_validators(crosslink_committee)`, `state.validator_balances[index] += base_reward(state, index) * total_attesting_balance(crosslink_committee) // get_total_balance(state, crosslink_committee))`. * If `index not in attesting_validators(crosslink_committee)`, `state.validator_balances[index] -= base_reward(state, index)`. From 41e95cf9eaa2d7a75882913093b3ffa60a0fb95e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 12 Feb 2019 19:11:45 +0800 Subject: [PATCH 94/96] Refactor `get_persistent_committee` (#604) --- specs/core/1_shard-data-chains.md | 52 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a4cf36134..2503c8dad 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -43,42 +43,46 @@ def get_split_offset(list_size: int, chunks: int, index: int) -> int: return (len(list_size) * index) // chunks ```` +#### get_shuffled_committee + +```python +def get_shuffled_committee(state: BeaconState, + shard: ShardNumber, + committee_start_epoch: EpochNumber) -> List[ValidatorIndex]: + """ + Return shuffled committee. + """ + validator_indices = get_active_validator_indices(state.validators, committee_start_epoch) + seed = generate_seed(state, committee_start_epoch) + start_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard) + end_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard + 1) + return [ + validator_indices[get_permuted_index(i, len(validator_indices), seed)] + for i in range(start_offset, end_offset) + ] +``` + #### get_persistent_committee ```python -def get_persistent_commmitee(state: BeaconState, +def get_persistent_committee(state: BeaconState, shard: ShardNumber, epoch: EpochNumber) -> List[ValidatorIndex]: """ - Returns the persistent committee for the given shard at the given epoch + Return the persistent committee for the given ``shard`` at the given ``epoch``. """ - - earlier_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - earlier_validator_set = get_active_validator_indices(state.validators, earlier_committee_start) - earlier_seed = generate_seed(state, earlier_committee_start) - earlier_start_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard) - earlier_end_offset = get_split_offset(len(earlier_validator_set), SHARD_COUNT, shard+1) - earlier_committee = [ - earlier_validator_set[get_permuted_index(i, len(earlier_validator_set), earlier_seed)] - for i in range(earlier_start_offset, earlier_end_offset) - ] - - later_committee_start = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD - later_validator_set = get_active_validator_indices(state.validators, later_committee_start) - later_seed = generate_seed(state, later_committee_start) - later_start_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard) - later_end_offset = get_split_offset(len(later_validator_set), SHARD_COUNT, shard+1) - later_committee = [ - later_validator_set[get_permuted_index(i, len(later_validator_set), later_seed)] - for i in range(later_start_offset, later_end_offset) - ] - + earlier_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 + earlier_committee = get_shuffled_committee(state, shard, earlier_committee_start_epoch) + + later_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD + later_committee = get_shuffled_committee(state, shard, later_committee_start_epoch) + def get_switchover_epoch(index): return ( bytes_to_int(hash(earlier_seed + bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) - + # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated return sorted(list(set( From 9a4f4d53447de9d7998c8fd9d1504b126cccf7e3 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 12 Feb 2019 11:12:29 +0000 Subject: [PATCH 95/96] Fix SSZ merkleisation bug (#602) --- specs/simple-serialize.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 65f1a7372..d4bd9e78d 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -367,13 +367,16 @@ def merkle_hash(lst): # Leave large items alone chunkz = lst - # Tree-hash - while len(chunkz) > 1: - if len(chunkz) % 2 == 1: - chunkz.append(b'\x00' * SSZ_CHUNK_SIZE) + # Merkleise + def next_power_of_2(x): + return 1 if x == 0 else 2**(x - 1).bit_length() + + for i in range(len(chunkz), next_power_of_2(len(chunkz))): + chunkz.append(b'\x00' * SSZ_CHUNK_SIZE) + while len(chunkz) > 1: chunkz = [hash(chunkz[i] + chunkz[i+1]) for i in range(0, len(chunkz), 2)] - # Return hash of root and length data + # Return hash of root and data length return hash(chunkz[0] + datalen) ``` From f871b9a0d1cda472d620a368d8c3fb2cead33c12 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 12 Feb 2019 12:52:07 +0100 Subject: [PATCH 96/96] Use little endian bit order (#575) Slightly more simple and common to find bit n at position `1 << n` --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0f0ad8f99..03be5ee17 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1077,7 +1077,7 @@ def get_bitfield_bit(bitfield: bytes, i: int) -> int: """ Extract the bit in ``bitfield`` at position ``i``. """ - return (bitfield[i // 8] >> (7 - (i % 8))) % 2 + return (bitfield[i // 8] >> (i % 8)) % 2 ``` ### `verify_bitfield`