From 24468de23bf1e0e4059ff5eeb5f9a5c621bc4f75 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 14 Mar 2019 20:28:44 -0500 Subject: [PATCH 1/8] Change get_shuffling to compute_committee See #729 and #774 The behavior now is that the first committee will consist of `get_permuted_index(0..n-1)`, the second committee `get_permuted_index(n....2n-1)`, etc. --- specs/core/0_beacon-chain.md | 47 +++++++++++++++--------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index daa1bc108..206aebf76 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -63,7 +63,7 @@ - [`get_permuted_index`](#get_permuted_index) - [`split`](#split) - [`get_epoch_committee_count`](#get_epoch_committee_count) - - [`get_shuffling`](#get_shuffling) + - [`compute_committee`](#compute_committee) - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count) - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count) @@ -803,28 +803,26 @@ def get_epoch_committee_count(active_validator_count: int) -> int: ) * SLOTS_PER_EPOCH ``` -### `get_shuffling` +### `compute_committee` ```python -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: +def compute_committee(validator_indices: [int], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: """ - Shuffle active validators and split into crosslink committees. - Return a list of committees (each a list of validator indices). + Return the index'th shuffled committee out of a total `total_committees` + using the given validator_indices and seed """ - # Shuffle active validator indices - active_validator_indices = get_active_validator_indices(validators, epoch) - length = len(active_validator_indices) - shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] - - # Split the shuffled active validator indices - return split(shuffled_indices, get_epoch_committee_count(length)) + start_offset = get_split_offset(len(validator_indices), total_committees, index) + end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) + return [ + validator_indices[get_permuted_index(i, len(validator_indices), seed)] + for i in range(start_offset, end_offset) + ] ``` -**Invariant**: if `get_shuffling(seed, validators, epoch)` returns some value `x` for some `epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY`, it should return the same value `x` for the same `seed` and `epoch` and possible future modifications of `validators` forever in phase 0, and until the ~1 year deletion delay in phase 2 and in the future. - -**Note**: this definition and the next few definitions make heavy use of repetitive computing. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. +**Note**: this definition and the next few definitions are highly inefficient as algorithms as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work. ### `get_previous_epoch_committee_count` @@ -916,22 +914,17 @@ def get_crosslink_committees_at_slot(state: BeaconState, shuffling_epoch = state.current_shuffling_epoch shuffling_start_shard = state.current_shuffling_start_shard - shuffling = get_shuffling( - seed, - state.validator_registry, - shuffling_epoch, - ) - offset = slot % SLOTS_PER_EPOCH - committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT - + indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) + committee_count = get_epoch_committee_count(len(indices)) + committees_per_slot = committee_count // EPOCH_LENGTH return [ ( - shuffling[committees_per_slot * offset + i], + compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) ] + ``` ### `get_block_root` From 5d327b63646d8831412853e7c972f866b78e628e Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 09:43:38 +0000 Subject: [PATCH 2/8] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 206aebf76..2dfeb7d69 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -806,7 +806,7 @@ def get_epoch_committee_count(active_validator_count: int) -> int: ### `compute_committee` ```python -def compute_committee(validator_indices: [int], +def compute_committee(validator_indices: List[ValidatorIndex], seed: Bytes32, index: int, total_committees: int) -> List[ValidatorIndex]: From 68d1c74784b8d5a1daa05b8098fa8bfb2e17b009 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 Mar 2019 09:45:20 +0000 Subject: [PATCH 3/8] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2dfeb7d69..be3544ab8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -811,8 +811,8 @@ def compute_committee(validator_indices: List[ValidatorIndex], index: int, total_committees: int) -> List[ValidatorIndex]: """ - Return the index'th shuffled committee out of a total `total_committees` - using the given validator_indices and seed + Return the ``index``'th shuffled committee out of a total ``total_committees`` + using ``validator_indices`` and ``seed``. """ start_offset = get_split_offset(len(validator_indices), total_committees, index) end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) From 009563b2c35c9c9fd352e8026d4b1ff4ff9d2e69 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:15:51 -0600 Subject: [PATCH 4/8] fix a few bugs in testing compute_committee --- scripts/phase0/build_spec.py | 23 ++++++++++++----------- specs/core/0_beacon-chain.md | 3 ++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index ae5a5a4f2..6116f1ffe 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -37,22 +37,23 @@ Store = None code_lines += function_puller.get_lines(sourcefile) code_lines.append(""" -# Monkey patch validator shuffling cache -_get_shuffling = get_shuffling -shuffling_cache = {} -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: +# Monkey patch validator get committee code +_compute_committee = compute_committee +committee_cache = {} +def compute_committee(validator_indices: List[ValidatorIndex], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: - param_hash = (seed, hash_tree_root(validators, [Validator]), epoch) + param_hash = (hash_tree_root(validator_indices), seed, index, total_committees) - if param_hash in shuffling_cache: + if param_hash in committee_cache: # print("Cache hit, epoch={0}".format(epoch)) - return shuffling_cache[param_hash] + return committee_cache[param_hash] else: # print("Cache miss, epoch={0}".format(epoch)) - ret = _get_shuffling(seed, validators, epoch) - shuffling_cache[param_hash] = ret + ret = _compute_committee(validator_indices, seed, index, total_committees) + committee_cache[param_hash] = ret return ret diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9708ee591..27ae71c00 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -916,7 +916,8 @@ def get_crosslink_committees_at_slot(state: BeaconState, indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) committee_count = get_epoch_committee_count(len(indices)) - committees_per_slot = committee_count // EPOCH_LENGTH + committees_per_slot = committee_count // SLOTS_PER_EPOCH + offset = slot % SLOTS_PER_EPOCH return [ ( compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) From c8e9073414114cad7b276eb6623457ed8fb1bf86 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:24:36 -0600 Subject: [PATCH 5/8] define get_split_offset and squash a couple of bugs --- specs/core/0_beacon-chain.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 27ae71c00..b56a87ad5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -61,7 +61,7 @@ - [`is_active_validator`](#is_active_validator) - [`get_active_validator_indices`](#get_active_validator_indices) - [`get_permuted_index`](#get_permuted_index) - - [`split`](#split) + - [`get_split_offset`](#get_split_offset) - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`compute_committee`](#compute_committee) - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count) @@ -773,18 +773,11 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: return index ``` -### `split` +### `get_split_offset` ```python -def split(values: List[Any], split_count: int) -> List[List[Any]]: - """ - Splits ``values`` into ``split_count`` pieces. - """ - list_length = len(values) - return [ - values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] - for i in range(split_count) - ] +def get_split_offset(list_length: int, split_count: int, index: int) -> int: + return (list_length * index) // split_count ``` ### `get_epoch_committee_count` @@ -918,9 +911,11 @@ def get_crosslink_committees_at_slot(state: BeaconState, committee_count = get_epoch_committee_count(len(indices)) committees_per_slot = committee_count // SLOTS_PER_EPOCH offset = slot % SLOTS_PER_EPOCH + slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT + return [ ( - compute_committee(indices, seed, committees_per_slot * offset + i, committee_count) + compute_committee(indices, seed, committees_per_slot * offset + i, committee_count), (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) From f5826e7f1ce5e46358873514d3a1c9d173fe55aa Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:34:49 -0600 Subject: [PATCH 6/8] small lint --- README.md | 7 ++++++- specs/core/0_beacon-chain.md | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5c88daf9..8f561a9ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Ethereum 2.0 Specifications +Ethereum 2.0 Specifications +------------ [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -25,3 +26,7 @@ The following are the broad design goals for Ethereum 2.0: * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) + +# Executable spec + +The aim is to have the entirety of the Ethereum 2.0Current just the phase 0 spec is executable. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b56a87ad5..e21607010 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -920,7 +920,6 @@ def get_crosslink_committees_at_slot(state: BeaconState, ) for i in range(committees_per_slot) ] - ``` ### `get_block_root` From f7fab30772b6d70c4a2f84acc171a0432575394e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:40:24 -0600 Subject: [PATCH 7/8] minor adjustment to not repeat committe count calc --- specs/core/0_beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e21607010..9563e22bb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -908,14 +908,13 @@ def get_crosslink_committees_at_slot(state: BeaconState, shuffling_start_shard = state.current_shuffling_start_shard indices = get_active_validator_indices(state.validator_registry, shuffling_epoch) - committee_count = get_epoch_committee_count(len(indices)) - committees_per_slot = committee_count // SLOTS_PER_EPOCH + committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH offset = slot % SLOTS_PER_EPOCH slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT return [ ( - compute_committee(indices, seed, committees_per_slot * offset + i, committee_count), + compute_committee(indices, seed, committees_per_slot * offset + i, committees_per_epoch), (slot_start_shard + i) % SHARD_COUNT, ) for i in range(committees_per_slot) From ba57d91e7a31b0dab328c7582dd1159b85fdf5d7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 19 Mar 2019 11:41:27 -0600 Subject: [PATCH 8/8] undo readme commit --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 8f561a9ab..c5c88daf9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Ethereum 2.0 Specifications ------------- +# Ethereum 2.0 Specifications [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -26,7 +25,3 @@ The following are the broad design goals for Ethereum 2.0: * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) - -# Executable spec - -The aim is to have the entirety of the Ethereum 2.0Current just the phase 0 spec is executable.