status-im
/
eth2.0-specs
mirror of https://github.com/status-im/eth2.0-specs.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

update shuffling

This commit is contained in:
protolambda 2019-04-11 01:52:51 +10:00
parent e5fb9626e8
commit 41374957bb
No known key found for this signature in database
GPG Key ID: EC89FDBB2B4C7623
6 changed files with 112 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
test_generators/shuffling/constants.py
View File

@ -1,6 +0,0 @@
SLOTS_PER_EPOCH = 2**6 # 64 slots, 6.4 minutes
FAR_FUTURE_EPOCH = 2**64 - 1 # uint64 max
SHARD_COUNT = 2**10 # 1024
TARGET_COMMITTEE_SIZE = 2**7 # 128 validators
ACTIVATION_EXIT_DELAY = 2**2 # 4 epochs
SHUFFLE_ROUND_COUNT = 90

95
test_generators/shuffling/core_helpers.py
View File

@ -1,95 +0,0 @@
from typing import Any, List, NewType
from constants import SLOTS_PER_EPOCH, SHARD_COUNT, TARGET_COMMITTEE_SIZE, SHUFFLE_ROUND_COUNT
from utils import hash
from yaml_objects import Validator
Epoch = NewType("Epoch", int)
ValidatorIndex = NewType("ValidatorIndex", int)
Bytes32 = NewType("Bytes32", bytes)
def int_to_bytes1(x):
return x.to_bytes(1, 'little')
def int_to_bytes4(x):
return x.to_bytes(4, 'little')
def bytes_to_int(data: bytes) -> int:
return int.from_bytes(data, 'little')
def is_active_validator(validator: Validator, epoch: Epoch) -> bool:
"""
Check if ``validator`` is active.
"""
return validator.activation_epoch <= epoch < validator.exit_epoch
def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]:
"""
Get indices of active validators from ``validators``.
"""
return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)]
def split(values: List[Any], split_count: int) -> List[List[Any]]:
"""
Splits ``values`` into ``split_count`` pieces.
"""
list_length = len(values)
return [
values[(list_length * i // split_count): (list_length * (i + 1) // split_count)]
for i in range(split_count)
]
def get_epoch_committee_count(active_validator_count: int) -> int:
"""
Return the number of committees in one epoch.
"""
return max(
1,
min(
SHARD_COUNT // SLOTS_PER_EPOCH,
active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE,
)
) * SLOTS_PER_EPOCH
def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int:
"""
Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
Utilizes 'swap or not' shuffling found in
https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
See the 'generalized domain' algorithm on page 3.
"""
for round in range(SHUFFLE_ROUND_COUNT):
pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size
flip = (pivot - index) % list_size
position = max(index, flip)
source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256))
byte = source[(position % 256) // 8]
bit = (byte >> (position % 8)) % 2
index = flip if bit else index
return index
def get_shuffling(seed: Bytes32,
validators: List[Validator],
epoch: Epoch) -> List[List[ValidatorIndex]]:
"""
Shuffle active validators and split into crosslink committees.
Return a list of committees (each a list of validator indices).
"""
# Shuffle active validator indices
active_validator_indices = get_active_validator_indices(validators, epoch)
length = len(active_validator_indices)
shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)]
# Split the shuffled active validator indices
return split(shuffled_indices, get_epoch_committee_count(length))

251
test_generators/shuffling/main.py
View File

@ -1,160 +1,127 @@
import random import random
import sys
import os
import yaml from eth2spec.phase0.spec import *
from eth_utils import (
from constants import ACTIVATION_EXIT_DELAY, FAR_FUTURE_EPOCH to_dict, to_tuple
from core_helpers import get_shuffling )
from yaml_objects import Validator from gen_base import gen_runner, gen_suite, gen_typing
from preset_loader import loader
def noop(self, *args, **kw): @to_dict
# Prevent !!str or !!binary tags def active_exited_validator_case(idx_max: int):
pass validators = []
# Standard deviation, around 8% validators will activate or exit within
# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator
# shuffling
RAND_EPOCH_STD = 35
yaml.emitter.Emitter.process_tag = noop # TODO: fix epoch numbers
slot = 1000 * SLOTS_PER_EPOCH
# The epoch, also a mean for the normal distribution
epoch = slot_to_epoch(slot)
MAX_EXIT_EPOCH = epoch + 5000 # Maximum exit_epoch for easier reading
for idx in range(idx_max):
v = Validator(
pubkey=bytes(random.randint(0, 255) for _ in range(48)),
withdrawal_credentials=bytes(random.randint(0, 255) for _ in range(32)),
activation_epoch=FAR_FUTURE_EPOCH,
exit_epoch=FAR_FUTURE_EPOCH,
withdrawable_epoch=FAR_FUTURE_EPOCH,
initiated_exit=False,
slashed=False,
high_balance=0
)
# 4/5 of all validators are active
if random.random() < 0.8:
# Choose a normally distributed epoch number
rand_epoch = round(random.gauss(epoch, RAND_EPOCH_STD))
EPOCH = 1000 # The epoch, also a mean for the normal distribution # for 1/2 of *active* validators rand_epoch is the activation epoch
if random.random() < 0.5:
v.activation_epoch = rand_epoch
# Standard deviation, around 8% validators will activate or exit within # 1/4 of active validators will exit in forseeable future
# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator
# shuffling
RAND_EPOCH_STD = 35
MAX_EXIT_EPOCH = 5000 # Maximum exit_epoch for easier reading
def active_exited_validators_generator():
"""
Random cases with variety of validator's activity status
"""
# Order not preserved - https://github.com/yaml/pyyaml/issues/110
metadata = {
'title': 'Shuffling Algorithm Tests 1',
'summary': 'Test vectors for validator shuffling with different validator\'s activity status.'
' Note: only relevant validator fields are defined.',
'test_suite': 'shuffle',
'fork': 'phase0-0.5.0',
}
# Config
random.seed(int("0xEF00BEAC", 16))
num_cases = 10
test_cases = []
for case in range(num_cases):
seedhash = bytes(random.randint(0, 255) for byte in range(32))
idx_max = random.randint(128, 512)
validators = []
for idx in range(idx_max):
v = Validator(original_index=idx)
# 4/5 of all validators are active
if random.random() < 0.8:
# Choose a normally distributed epoch number
rand_epoch = round(random.gauss(EPOCH, RAND_EPOCH_STD))
# for 1/2 of *active* validators rand_epoch is the activation epoch
if random.random() < 0.5: if random.random() < 0.5:
v.activation_epoch = rand_epoch v.exit_epoch = random.randint(
rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH)
# 1/4 of active validators will exit in forseeable future # 1/4 of active validators in theory remain in the set indefinitely
if random.random() < 0.5:
v.exit_epoch = random.randint(
rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH)
# 1/4 of active validators in theory remain in the set indefinitely
else:
v.exit_epoch = FAR_FUTURE_EPOCH
# for the other active 1/2 rand_epoch is the exit epoch
else: else:
v.activation_epoch = random.randint( v.exit_epoch = FAR_FUTURE_EPOCH
0, rand_epoch - ACTIVATION_EXIT_DELAY) # for the other active 1/2 rand_epoch is the exit epoch
v.exit_epoch = rand_epoch
# The remaining 1/5 of all validators is not activated
else: else:
v.activation_epoch = FAR_FUTURE_EPOCH v.activation_epoch = random.randint(
v.exit_epoch = FAR_FUTURE_EPOCH 0, rand_epoch - ACTIVATION_EXIT_DELAY)
v.exit_epoch = rand_epoch
validators.append(v) # The remaining 1/5 of all validators is not activated
else:
input_ = { v.activation_epoch = FAR_FUTURE_EPOCH
'validators': validators,
'epoch': EPOCH
}
output = get_shuffling(
seedhash, validators, input_['epoch'])
test_cases.append({
'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output
})
return {
'metadata': metadata,
'filename': 'test_vector_shuffling.yml',
'test_cases': test_cases
}
def validators_set_size_variety_generator():
"""
Different validator set size cases, inspired by removed manual `permutated_index` tests
https://github.com/ethereum/eth2.0-test-generators/tree/bcd9ab2933d9f696901d1dfda0828061e9d3093f/permutated_index
"""
# Order not preserved - https://github.com/yaml/pyyaml/issues/110
metadata = {
'title': 'Shuffling Algorithm Tests 2',
'summary': 'Test vectors for validator shuffling with different validator\'s set size.'
' Note: only relevant validator fields are defined.',
'test_suite': 'shuffle',
'fork': 'tchaikovsky',
'version': 1.0
}
# Config
random.seed(int("0xEF00BEAC", 16))
test_cases = []
seedhash = bytes(random.randint(0, 255) for byte in range(32))
idx_max = 4096
set_sizes = [1, 2, 3, 1024, idx_max]
for size in set_sizes:
validators = []
for idx in range(size):
v = Validator(original_index=idx)
v.activation_epoch = EPOCH
v.exit_epoch = FAR_FUTURE_EPOCH v.exit_epoch = FAR_FUTURE_EPOCH
validators.append(v)
input_ = {
'validators': validators,
'epoch': EPOCH
}
output = get_shuffling(
seedhash, validators, input_['epoch'])
test_cases.append({ validators.append(v)
'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output
})
return { query_slot = slot + random.randint(-1, 1)
'metadata': metadata, state = get_genesis_beacon_state([], 0, None)
'filename': 'shuffling_set_size.yml', state.validator_registry = validators
'test_cases': test_cases state.latest_randao_mixes = [b'\xde\xad\xbe\xef' * 8 for _ in range(LATEST_RANDAO_MIXES_LENGTH)]
} state.slot = slot
state.latest_start_shard = random.randint(0, SHARD_COUNT - 1)
randao_mix = bytes(random.randint(0, 255) for _ in range(32))
state.latest_randao_mixes[slot_to_epoch(query_slot) % LATEST_RANDAO_MIXES_LENGTH] = randao_mix
committees = get_crosslink_committees_at_slot(state, query_slot)
yield 'validator_registry', [
{
'activation_epoch': v.activation_epoch,
'exit_epoch': v.exit_epoch
} for v in state.validator_registry
]
yield 'randao_mix', '0x'+randao_mix.hex()
yield 'state_slot', state.slot
yield 'query_slot', query_slot
yield 'latest_start_shard', state.latest_start_shard
yield 'crosslink_committees', committees
if __name__ == '__main__': @to_tuple
output_dir = sys.argv[2] def active_exited_validator_cases():
for generator in [active_exited_validators_generator, validators_set_size_variety_generator]: for i in range(3):
result = generator() yield active_exited_validator_case(random.randint(100, min(200, SHARD_COUNT * 2)))
filename = os.path.join(output_dir, result['filename'])
with open(filename, 'w') as outfile:
# Dump at top level def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
yaml.dump(result['metadata'], outfile, default_flow_style=False) presets = loader.load_presets(configs_path, 'minimal')
# default_flow_style will unravel "ValidatorRecord" and "committee" line, exploding file size apply_constants_preset(presets)
yaml.dump({'test_cases': result['test_cases']}, outfile)
return ("shuffling_minimal", "core", gen_suite.render_suite(
title="Shuffling Algorithm Tests with minimal config",
summary="Test vectors for validator shuffling with different validator registry activity status and set size."
" Note: only relevant fields are defined.",
forks_timeline="testing",
forks=["phase0"],
config="minimal",
handler="core",
test_cases=active_exited_validator_cases()))
def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
presets = loader.load_presets(configs_path, 'mainnet')
apply_constants_preset(presets)
return ("shuffling_full", "core", gen_suite.render_suite(
title="Shuffling Algorithm Tests with mainnet config",
summary="Test vectors for validator shuffling with different validator registry activity status and set size."
" Note: only relevant fields are defined.",
forks_timeline="mainnet",
forks=["phase0"],
config="mainnet",
handler="core",
test_cases=active_exited_validator_cases()))
if __name__ == "__main__":
gen_runner.run_generator("shuffling", [mini_shuffling_suite, full_shuffling_suite])

6
test_generators/shuffling/requirements.txt
View File

@ -1,4 +1,4 @@
eth-hash[pycryptodome]==0.2.0
eth-typing==2.0.0
eth-utils==1.4.1 eth-utils==1.4.1
PyYAML==4.2b1 ../../test_libs/gen_helpers
../../test_libs/config_helpers
../../test_libs/pyspec

6
test_generators/shuffling/utils.py
View File

@ -1,6 +0,0 @@
from eth_typing import Hash32
from eth_utils import keccak
def hash(x: bytes) -> Hash32:
return keccak(x)

25
test_generators/shuffling/yaml_objects.py
View File

@ -1,25 +0,0 @@
from typing import Any
import yaml
class Validator(yaml.YAMLObject):
"""
A validator stub containing only the fields relevant for get_shuffling()
"""
fields = {
'activation_epoch': 'uint64',
'exit_epoch': 'uint64',
# Extra index field to ease testing/debugging
'original_index': 'uint64',
}
def __init__(self, **kwargs):
for k in self.fields.keys():
setattr(self, k, kwargs.get(k))
def __setattr__(self, name: str, value: Any) -> None:
super().__setattr__(name, value)
def __getattribute__(self, name: str) -> Any:
return super().__getattribute__(name)