speed up epoch processing by caching shuffling results (#153)
* speed up epoch processing by caching shuffling results
This commit is contained in:
Dustin Brody
GitHub
parent
38f48753b2
commit
b8402e9809
|
|
@ -151,6 +151,26 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
|
|||
validator.withdrawable_epoch = get_current_epoch(state) +
|
||||
LATEST_SLASHED_EXIT_LENGTH
|
||||
|
||||
func update_shuffling_cache*(state: var BeaconState) =
|
||||
let
|
||||
list_size = state.validator_registry.len.uint64
|
||||
shuffling_seq = mapIt(
|
||||
get_shuffled_seq(state.current_shuffling_seed, list_size),
|
||||
# No intrinsic reason for this conversion; SSZ requirement artifact.
|
||||
it.int)
|
||||
|
||||
doAssert state.shuffling_cache.index in [0, 1]
|
||||
|
||||
# Do a dance to keep everything JSON-encodable.
|
||||
state.shuffling_cache.seeds[state.shuffling_cache.index] =
|
||||
state.current_shuffling_seed
|
||||
state.shuffling_cache.list_sizes[state.shuffling_cache.index] = list_size
|
||||
if state.shuffling_cache.index == 0:
|
||||
state.shuffling_cache.shuffling_0 = shuffling_seq
|
||||
else:
|
||||
state.shuffling_cache.shuffling_1 = shuffling_seq
|
||||
state.shuffling_cache.index = 1 - state.shuffling_cache.index
|
||||
|
||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#on-genesis
|
||||
func get_genesis_beacon_state*(
|
||||
initial_validator_deposits: openArray[Deposit],
|
||||
|
|
@ -238,6 +258,8 @@ func get_genesis_beacon_state*(
|
|||
state.latest_active_index_roots[index] = genesis_active_index_root
|
||||
state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH)
|
||||
|
||||
update_shuffling_cache(state)
|
||||
|
||||
state
|
||||
|
||||
# TODO candidate for spec?
|
||||
|
|
|
|||
|
|
@ -417,6 +417,9 @@ type
|
|||
eth1_data_votes*: seq[Eth1DataVote]
|
||||
deposit_index*: uint64
|
||||
|
||||
# Not in spec. TODO: don't serialize or deserialize this.
|
||||
shuffling_cache*: ShufflingCache
|
||||
|
||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#validator
|
||||
Validator* = object
|
||||
pubkey*: ValidatorPubKey ##\
|
||||
|
|
@ -493,6 +496,21 @@ type
|
|||
|
||||
# TODO: not in spec
|
||||
CrosslinkCommittee* = tuple[committee: seq[ValidatorIndex], shard: uint64]
|
||||
ShufflingCache* = object
|
||||
## https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
|
||||
## Note: this definition and the next few definitions make heavy use of
|
||||
## repetitive computing. Production implementations are expected to
|
||||
## appropriately use caching/memoization to avoid redoing work.
|
||||
##
|
||||
## TODO use native ValidatorIndex, once this doesn't need to be serialized.
|
||||
## `seed` and `list_size` determine the shuffle. For now, only need two, at
|
||||
## any given time. If the next_epoch variations of shuffling get called, it
|
||||
## might increase to three at once.
|
||||
seeds*: array[2, Eth2Digest]
|
||||
list_sizes*: array[2, uint64]
|
||||
index*: int
|
||||
shuffling_0*: seq[int]
|
||||
shuffling_1*: seq[int]
|
||||
|
||||
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
|
||||
($state.validator_registry[validatorIdx].pubkey)[0..7]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import
|
|||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
|
||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_permuted_index
|
||||
func get_shuffled_seq*(seed: Eth2Digest,
|
||||
validators: openArray[Validator],
|
||||
list_size: uint64,
|
||||
): seq[ValidatorIndex] =
|
||||
## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go
|
||||
## Shuffles ``validators`` into crosslink committees seeded by ``seed`` and
|
||||
|
|
@ -25,7 +25,6 @@ func get_shuffled_seq*(seed: Eth2Digest,
|
|||
##
|
||||
## Invert the inner/outer loops from the spec, essentially. Most useful
|
||||
## hash result re-use occurs within a round.
|
||||
let list_size = validators.len.uint64
|
||||
var
|
||||
# Share these buffers.
|
||||
pivot_buffer: array[(32+1), byte]
|
||||
|
|
@ -75,21 +74,31 @@ func get_shuffled_seq*(seed: Eth2Digest,
|
|||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
|
||||
func get_shuffling*(seed: Eth2Digest,
|
||||
validators: openArray[Validator],
|
||||
epoch: Epoch
|
||||
epoch: Epoch,
|
||||
shuffling_cache: ShufflingCache
|
||||
): seq[seq[ValidatorIndex]] =
|
||||
## This function is factored to facilitate testing with
|
||||
## https://github.com/ethereum/eth2.0-test-generators/tree/master/permutated_index
|
||||
## test vectors, which the split of get_shuffling obfuscates.
|
||||
let list_size = validators.len.uint64
|
||||
|
||||
let
|
||||
active_validator_indices = get_active_validator_indices(validators, epoch)
|
||||
committees_per_epoch = get_epoch_committee_count(
|
||||
len(active_validator_indices)).int
|
||||
# Both mapIt-type-conversions are an SSZ artifact. TODO remove.
|
||||
shuffled_seq =
|
||||
if shuffling_cache.seeds[0] == seed and
|
||||
shuffling_cache.list_sizes[0] == list_size:
|
||||
mapIt(shuffling_cache.shuffling_0, it.ValidatorIndex)
|
||||
elif shuffling_cache.seeds[1] == seed and
|
||||
shuffling_cache.list_sizes[1] == list_size:
|
||||
mapIt(shuffling_cache.shuffling_1, it.ValidatorIndex)
|
||||
else:
|
||||
get_shuffled_seq(seed, list_size)
|
||||
|
||||
# Split the shuffled list into committees_per_epoch pieces
|
||||
result = split(
|
||||
get_shuffled_seq(seed, validators),
|
||||
committees_per_epoch)
|
||||
result = split(shuffled_seq, committees_per_epoch)
|
||||
assert result.len() == committees_per_epoch # what split should do..
|
||||
|
||||
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_previous_epoch_committee_count
|
||||
|
|
@ -196,6 +205,7 @@ func get_crosslink_committees_at_slot*(state: BeaconState, slot: Slot,
|
|||
seed,
|
||||
state.validator_registry,
|
||||
shuffling_epoch,
|
||||
state.shuffling_cache
|
||||
)
|
||||
offset = slot mod SLOTS_PER_EPOCH
|
||||
committees_per_slot = committees_per_epoch div SLOTS_PER_EPOCH
|
||||
|
|
|
|||
|
|
@ -916,6 +916,8 @@ func processEpoch(state: var BeaconState) =
|
|||
state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch)
|
||||
# /Note/ that state.current_shuffling_start_shard is left unchanged
|
||||
|
||||
updateShufflingCache(state)
|
||||
|
||||
## Regardless of whether or not a validator set change happens run
|
||||
## process_slashings(state) and process_exit_queue(state)
|
||||
process_slashings(state)
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ func sumCommittees(v: openArray[seq[ValidatorIndex]], reqCommitteeLen: int): int
|
|||
func checkPermutation(index: int, list_size: uint64,
|
||||
permuted_index: int, seed: Eth2Digest): bool =
|
||||
let
|
||||
validators = repeat(
|
||||
Validator(exit_epoch: FAR_FUTURE_EPOCH), list_size)
|
||||
s = get_shuffled_seq(seed, validators)
|
||||
s = get_shuffled_seq(seed, list_size)
|
||||
s[index] == permuted_index.ValidatorIndex
|
||||
|
||||
suite "Validators":
|
||||
|
|
@ -35,7 +33,7 @@ suite "Validators":
|
|||
Validator(
|
||||
exit_epoch: FAR_FUTURE_EPOCH
|
||||
), num_validators)
|
||||
s = get_shuffling(Eth2Digest(), validators, 0)
|
||||
s = get_shuffling(Eth2Digest(), validators, 0, ShufflingCache())
|
||||
committees = get_epoch_committee_count(len(validators)).int
|
||||
check:
|
||||
# def b(s): return "Eth2Digest(data: [0x" + "'u8, 0x".join((s[i:i+2] for i in range(0, 64, 2))) + "'u8])"
|
||||
|
|
|
|||
Loading…
Reference in New Issue