update shuffling
This commit is contained in:
protolambda
parent
e5fb9626e8
commit
41374957bb
|
|
@ -1,6 +0,0 @@
|
|||
SLOTS_PER_EPOCH = 2**6 # 64 slots, 6.4 minutes
|
||||
FAR_FUTURE_EPOCH = 2**64 - 1 # uint64 max
|
||||
SHARD_COUNT = 2**10 # 1024
|
||||
TARGET_COMMITTEE_SIZE = 2**7 # 128 validators
|
||||
ACTIVATION_EXIT_DELAY = 2**2 # 4 epochs
|
||||
SHUFFLE_ROUND_COUNT = 90
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
from typing import Any, List, NewType
|
||||
|
||||
from constants import SLOTS_PER_EPOCH, SHARD_COUNT, TARGET_COMMITTEE_SIZE, SHUFFLE_ROUND_COUNT
|
||||
from utils import hash
|
||||
from yaml_objects import Validator
|
||||
|
||||
Epoch = NewType("Epoch", int)
|
||||
ValidatorIndex = NewType("ValidatorIndex", int)
|
||||
Bytes32 = NewType("Bytes32", bytes)
|
||||
|
||||
|
||||
def int_to_bytes1(x):
|
||||
return x.to_bytes(1, 'little')
|
||||
|
||||
|
||||
def int_to_bytes4(x):
|
||||
return x.to_bytes(4, 'little')
|
||||
|
||||
|
||||
def bytes_to_int(data: bytes) -> int:
|
||||
return int.from_bytes(data, 'little')
|
||||
|
||||
|
||||
def is_active_validator(validator: Validator, epoch: Epoch) -> bool:
|
||||
"""
|
||||
Check if ``validator`` is active.
|
||||
"""
|
||||
return validator.activation_epoch <= epoch < validator.exit_epoch
|
||||
|
||||
|
||||
def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]:
|
||||
"""
|
||||
Get indices of active validators from ``validators``.
|
||||
"""
|
||||
return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)]
|
||||
|
||||
|
||||
def split(values: List[Any], split_count: int) -> List[List[Any]]:
|
||||
"""
|
||||
Splits ``values`` into ``split_count`` pieces.
|
||||
"""
|
||||
list_length = len(values)
|
||||
return [
|
||||
values[(list_length * i // split_count): (list_length * (i + 1) // split_count)]
|
||||
for i in range(split_count)
|
||||
]
|
||||
|
||||
|
||||
def get_epoch_committee_count(active_validator_count: int) -> int:
|
||||
"""
|
||||
Return the number of committees in one epoch.
|
||||
"""
|
||||
return max(
|
||||
1,
|
||||
min(
|
||||
SHARD_COUNT // SLOTS_PER_EPOCH,
|
||||
active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE,
|
||||
)
|
||||
) * SLOTS_PER_EPOCH
|
||||
|
||||
|
||||
def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int:
|
||||
"""
|
||||
Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
|
||||
|
||||
Utilizes 'swap or not' shuffling found in
|
||||
https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
|
||||
See the 'generalized domain' algorithm on page 3.
|
||||
"""
|
||||
for round in range(SHUFFLE_ROUND_COUNT):
|
||||
pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size
|
||||
flip = (pivot - index) % list_size
|
||||
position = max(index, flip)
|
||||
source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256))
|
||||
byte = source[(position % 256) // 8]
|
||||
bit = (byte >> (position % 8)) % 2
|
||||
index = flip if bit else index
|
||||
|
||||
return index
|
||||
|
||||
|
||||
def get_shuffling(seed: Bytes32,
|
||||
validators: List[Validator],
|
||||
epoch: Epoch) -> List[List[ValidatorIndex]]:
|
||||
"""
|
||||
Shuffle active validators and split into crosslink committees.
|
||||
Return a list of committees (each a list of validator indices).
|
||||
"""
|
||||
# Shuffle active validator indices
|
||||
active_validator_indices = get_active_validator_indices(validators, epoch)
|
||||
length = len(active_validator_indices)
|
||||
shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)]
|
||||
|
||||
# Split the shuffled active validator indices
|
||||
return split(shuffled_indices, get_epoch_committee_count(length))
|
||||
|
|
@ -1,160 +1,127 @@
|
|||
import random
|
||||
import sys
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
from constants import ACTIVATION_EXIT_DELAY, FAR_FUTURE_EPOCH
|
||||
from core_helpers import get_shuffling
|
||||
from yaml_objects import Validator
|
||||
from eth2spec.phase0.spec import *
|
||||
from eth_utils import (
|
||||
to_dict, to_tuple
|
||||
)
|
||||
from gen_base import gen_runner, gen_suite, gen_typing
|
||||
from preset_loader import loader
|
||||
|
||||
|
||||
def noop(self, *args, **kw):
|
||||
# Prevent !!str or !!binary tags
|
||||
pass
|
||||
@to_dict
|
||||
def active_exited_validator_case(idx_max: int):
|
||||
validators = []
|
||||
|
||||
# Standard deviation, around 8% validators will activate or exit within
|
||||
# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator
|
||||
# shuffling
|
||||
RAND_EPOCH_STD = 35
|
||||
|
||||
yaml.emitter.Emitter.process_tag = noop
|
||||
# TODO: fix epoch numbers
|
||||
|
||||
slot = 1000 * SLOTS_PER_EPOCH
|
||||
# The epoch, also a mean for the normal distribution
|
||||
epoch = slot_to_epoch(slot)
|
||||
MAX_EXIT_EPOCH = epoch + 5000 # Maximum exit_epoch for easier reading
|
||||
|
||||
for idx in range(idx_max):
|
||||
v = Validator(
|
||||
pubkey=bytes(random.randint(0, 255) for _ in range(48)),
|
||||
withdrawal_credentials=bytes(random.randint(0, 255) for _ in range(32)),
|
||||
activation_epoch=FAR_FUTURE_EPOCH,
|
||||
exit_epoch=FAR_FUTURE_EPOCH,
|
||||
withdrawable_epoch=FAR_FUTURE_EPOCH,
|
||||
initiated_exit=False,
|
||||
slashed=False,
|
||||
high_balance=0
|
||||
)
|
||||
# 4/5 of all validators are active
|
||||
if random.random() < 0.8:
|
||||
# Choose a normally distributed epoch number
|
||||
rand_epoch = round(random.gauss(epoch, RAND_EPOCH_STD))
|
||||
|
||||
EPOCH = 1000 # The epoch, also a mean for the normal distribution
|
||||
# for 1/2 of *active* validators rand_epoch is the activation epoch
|
||||
if random.random() < 0.5:
|
||||
v.activation_epoch = rand_epoch
|
||||
|
||||
# Standard deviation, around 8% validators will activate or exit within
|
||||
# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator
|
||||
# shuffling
|
||||
RAND_EPOCH_STD = 35
|
||||
MAX_EXIT_EPOCH = 5000 # Maximum exit_epoch for easier reading
|
||||
|
||||
|
||||
def active_exited_validators_generator():
|
||||
"""
|
||||
Random cases with variety of validator's activity status
|
||||
"""
|
||||
# Order not preserved - https://github.com/yaml/pyyaml/issues/110
|
||||
metadata = {
|
||||
'title': 'Shuffling Algorithm Tests 1',
|
||||
'summary': 'Test vectors for validator shuffling with different validator\'s activity status.'
|
||||
' Note: only relevant validator fields are defined.',
|
||||
'test_suite': 'shuffle',
|
||||
'fork': 'phase0-0.5.0',
|
||||
}
|
||||
|
||||
# Config
|
||||
random.seed(int("0xEF00BEAC", 16))
|
||||
num_cases = 10
|
||||
|
||||
test_cases = []
|
||||
|
||||
for case in range(num_cases):
|
||||
seedhash = bytes(random.randint(0, 255) for byte in range(32))
|
||||
idx_max = random.randint(128, 512)
|
||||
|
||||
validators = []
|
||||
for idx in range(idx_max):
|
||||
v = Validator(original_index=idx)
|
||||
# 4/5 of all validators are active
|
||||
if random.random() < 0.8:
|
||||
# Choose a normally distributed epoch number
|
||||
rand_epoch = round(random.gauss(EPOCH, RAND_EPOCH_STD))
|
||||
|
||||
# for 1/2 of *active* validators rand_epoch is the activation epoch
|
||||
# 1/4 of active validators will exit in forseeable future
|
||||
if random.random() < 0.5:
|
||||
v.activation_epoch = rand_epoch
|
||||
|
||||
# 1/4 of active validators will exit in forseeable future
|
||||
if random.random() < 0.5:
|
||||
v.exit_epoch = random.randint(
|
||||
rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH)
|
||||
# 1/4 of active validators in theory remain in the set indefinitely
|
||||
else:
|
||||
v.exit_epoch = FAR_FUTURE_EPOCH
|
||||
# for the other active 1/2 rand_epoch is the exit epoch
|
||||
v.exit_epoch = random.randint(
|
||||
rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH)
|
||||
# 1/4 of active validators in theory remain in the set indefinitely
|
||||
else:
|
||||
v.activation_epoch = random.randint(
|
||||
0, rand_epoch - ACTIVATION_EXIT_DELAY)
|
||||
v.exit_epoch = rand_epoch
|
||||
|
||||
# The remaining 1/5 of all validators is not activated
|
||||
v.exit_epoch = FAR_FUTURE_EPOCH
|
||||
# for the other active 1/2 rand_epoch is the exit epoch
|
||||
else:
|
||||
v.activation_epoch = FAR_FUTURE_EPOCH
|
||||
v.exit_epoch = FAR_FUTURE_EPOCH
|
||||
v.activation_epoch = random.randint(
|
||||
0, rand_epoch - ACTIVATION_EXIT_DELAY)
|
||||
v.exit_epoch = rand_epoch
|
||||
|
||||
validators.append(v)
|
||||
|
||||
input_ = {
|
||||
'validators': validators,
|
||||
'epoch': EPOCH
|
||||
}
|
||||
output = get_shuffling(
|
||||
seedhash, validators, input_['epoch'])
|
||||
|
||||
test_cases.append({
|
||||
'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output
|
||||
})
|
||||
|
||||
return {
|
||||
'metadata': metadata,
|
||||
'filename': 'test_vector_shuffling.yml',
|
||||
'test_cases': test_cases
|
||||
}
|
||||
|
||||
|
||||
def validators_set_size_variety_generator():
|
||||
"""
|
||||
Different validator set size cases, inspired by removed manual `permutated_index` tests
|
||||
https://github.com/ethereum/eth2.0-test-generators/tree/bcd9ab2933d9f696901d1dfda0828061e9d3093f/permutated_index
|
||||
"""
|
||||
# Order not preserved - https://github.com/yaml/pyyaml/issues/110
|
||||
metadata = {
|
||||
'title': 'Shuffling Algorithm Tests 2',
|
||||
'summary': 'Test vectors for validator shuffling with different validator\'s set size.'
|
||||
' Note: only relevant validator fields are defined.',
|
||||
'test_suite': 'shuffle',
|
||||
'fork': 'tchaikovsky',
|
||||
'version': 1.0
|
||||
}
|
||||
|
||||
# Config
|
||||
random.seed(int("0xEF00BEAC", 16))
|
||||
|
||||
test_cases = []
|
||||
|
||||
seedhash = bytes(random.randint(0, 255) for byte in range(32))
|
||||
idx_max = 4096
|
||||
set_sizes = [1, 2, 3, 1024, idx_max]
|
||||
|
||||
for size in set_sizes:
|
||||
validators = []
|
||||
for idx in range(size):
|
||||
v = Validator(original_index=idx)
|
||||
v.activation_epoch = EPOCH
|
||||
# The remaining 1/5 of all validators is not activated
|
||||
else:
|
||||
v.activation_epoch = FAR_FUTURE_EPOCH
|
||||
v.exit_epoch = FAR_FUTURE_EPOCH
|
||||
validators.append(v)
|
||||
input_ = {
|
||||
'validators': validators,
|
||||
'epoch': EPOCH
|
||||
}
|
||||
output = get_shuffling(
|
||||
seedhash, validators, input_['epoch'])
|
||||
|
||||
test_cases.append({
|
||||
'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output
|
||||
})
|
||||
validators.append(v)
|
||||
|
||||
return {
|
||||
'metadata': metadata,
|
||||
'filename': 'shuffling_set_size.yml',
|
||||
'test_cases': test_cases
|
||||
}
|
||||
query_slot = slot + random.randint(-1, 1)
|
||||
state = get_genesis_beacon_state([], 0, None)
|
||||
state.validator_registry = validators
|
||||
state.latest_randao_mixes = [b'\xde\xad\xbe\xef' * 8 for _ in range(LATEST_RANDAO_MIXES_LENGTH)]
|
||||
state.slot = slot
|
||||
state.latest_start_shard = random.randint(0, SHARD_COUNT - 1)
|
||||
randao_mix = bytes(random.randint(0, 255) for _ in range(32))
|
||||
state.latest_randao_mixes[slot_to_epoch(query_slot) % LATEST_RANDAO_MIXES_LENGTH] = randao_mix
|
||||
|
||||
committees = get_crosslink_committees_at_slot(state, query_slot)
|
||||
yield 'validator_registry', [
|
||||
{
|
||||
'activation_epoch': v.activation_epoch,
|
||||
'exit_epoch': v.exit_epoch
|
||||
} for v in state.validator_registry
|
||||
]
|
||||
yield 'randao_mix', '0x'+randao_mix.hex()
|
||||
yield 'state_slot', state.slot
|
||||
yield 'query_slot', query_slot
|
||||
yield 'latest_start_shard', state.latest_start_shard
|
||||
yield 'crosslink_committees', committees
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
output_dir = sys.argv[2]
|
||||
for generator in [active_exited_validators_generator, validators_set_size_variety_generator]:
|
||||
result = generator()
|
||||
filename = os.path.join(output_dir, result['filename'])
|
||||
with open(filename, 'w') as outfile:
|
||||
# Dump at top level
|
||||
yaml.dump(result['metadata'], outfile, default_flow_style=False)
|
||||
# default_flow_style will unravel "ValidatorRecord" and "committee" line, exploding file size
|
||||
yaml.dump({'test_cases': result['test_cases']}, outfile)
|
||||
@to_tuple
|
||||
def active_exited_validator_cases():
|
||||
for i in range(3):
|
||||
yield active_exited_validator_case(random.randint(100, min(200, SHARD_COUNT * 2)))
|
||||
|
||||
|
||||
def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
|
||||
presets = loader.load_presets(configs_path, 'minimal')
|
||||
apply_constants_preset(presets)
|
||||
|
||||
return ("shuffling_minimal", "core", gen_suite.render_suite(
|
||||
title="Shuffling Algorithm Tests with minimal config",
|
||||
summary="Test vectors for validator shuffling with different validator registry activity status and set size."
|
||||
" Note: only relevant fields are defined.",
|
||||
forks_timeline="testing",
|
||||
forks=["phase0"],
|
||||
config="minimal",
|
||||
handler="core",
|
||||
test_cases=active_exited_validator_cases()))
|
||||
|
||||
|
||||
def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
|
||||
presets = loader.load_presets(configs_path, 'mainnet')
|
||||
apply_constants_preset(presets)
|
||||
|
||||
return ("shuffling_full", "core", gen_suite.render_suite(
|
||||
title="Shuffling Algorithm Tests with mainnet config",
|
||||
summary="Test vectors for validator shuffling with different validator registry activity status and set size."
|
||||
" Note: only relevant fields are defined.",
|
||||
forks_timeline="mainnet",
|
||||
forks=["phase0"],
|
||||
config="mainnet",
|
||||
handler="core",
|
||||
test_cases=active_exited_validator_cases()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gen_runner.run_generator("shuffling", [mini_shuffling_suite, full_shuffling_suite])
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
eth-hash[pycryptodome]==0.2.0
|
||||
eth-typing==2.0.0
|
||||
eth-utils==1.4.1
|
||||
PyYAML==4.2b1
|
||||
../../test_libs/gen_helpers
|
||||
../../test_libs/config_helpers
|
||||
../../test_libs/pyspec
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from eth_typing import Hash32
|
||||
from eth_utils import keccak
|
||||
|
||||
|
||||
def hash(x: bytes) -> Hash32:
|
||||
return keccak(x)
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class Validator(yaml.YAMLObject):
|
||||
"""
|
||||
A validator stub containing only the fields relevant for get_shuffling()
|
||||
"""
|
||||
fields = {
|
||||
'activation_epoch': 'uint64',
|
||||
'exit_epoch': 'uint64',
|
||||
# Extra index field to ease testing/debugging
|
||||
'original_index': 'uint64',
|
||||
}
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
for k in self.fields.keys():
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
def __setattr__(self, name: str, value: Any) -> None:
|
||||
super().__setattr__(name, value)
|
||||
|
||||
def __getattribute__(self, name: str) -> Any:
|
||||
return super().__getattribute__(name)
|
||||
Loading…
Reference in New Issue