Caching updates/refactoring & state_sim defaults updates (#235)

* rm now-superceded shuffling cache (shuffling is only called from get_crosslinks), which was badly architected due to trying to exist in state; rm one more vestige of previous light-client regime (one more to go, from datatypes)

* fix wrong shuffling list size (active validator size, not validator size) to make consistent with 0.5.1 (will be inconsistent with testnet0); fix typo and change defaults in state_sim

* doAssert a couple of constant relationships necessary to avoid underflow; rm non-spec, unused helper function get_new_recent_block_roots

* refactor separate crosslink_committee_cache and winning_root_participants_cache(s) into StateData object; remove last vestige of previous shuffling cache

* separate out caching parts of StateData to new StateCache object
This commit is contained in:
Dustin Brody 2019-04-05 14:18:13 +00:00 committed by Jacek Sieka
parent 605dd0a0e9
commit c53de3e550
9 changed files with 65 additions and 131 deletions

View File

@ -222,6 +222,13 @@ type
## The block associated with the state found in data - in particular,
## blck.state_root == root
StateCache* = object
crosslink_committee_cache*:
Table[tuple[a: uint64, b: bool], seq[CrosslinkCommittee]]
winning_root_participants_cache*:
Table[Shard, HashSet[ValidatorIndex]]
BlockSlot* = object
## Unique identifier for a particular fork in the block chain - normally,
## there's a block for every slot, but in the case a block is not produced,

View File

@ -7,7 +7,7 @@
import
chronicles, math, options, sequtils,
../extras, ../ssz,
../extras, ../ssz, ../beacon_node_types,
./bitfield, ./crypto, ./datatypes, ./digest, ./helpers, ./validator,
tables
@ -195,26 +195,6 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
validator.withdrawable_epoch =
get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH
func update_shuffling_cache*(state: var BeaconState) =
let
list_size = state.validator_registry.len.uint64
shuffling_seq = mapIt(
get_shuffled_seq(state.current_shuffling_seed, list_size),
# No intrinsic reason for this conversion; SSZ requirement artifact.
it.int)
doAssert state.shuffling_cache.index in [0, 1]
# Do a dance to keep everything JSON-encodable.
state.shuffling_cache.seeds[state.shuffling_cache.index] =
state.current_shuffling_seed
state.shuffling_cache.list_sizes[state.shuffling_cache.index] = list_size
if state.shuffling_cache.index == 0:
state.shuffling_cache.shuffling_0 = shuffling_seq
else:
state.shuffling_cache.shuffling_1 = shuffling_seq
state.shuffling_cache.index = 1 - state.shuffling_cache.index
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#get_temporary_block_header
func get_temporary_block_header*(blck: BeaconBlock): BeaconBlockHeader =
## Return the block header corresponding to a block with ``state_root`` set
@ -318,9 +298,6 @@ func get_genesis_beacon_state*(
state.latest_active_index_roots[index] = genesis_active_index_root
state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH)
# Not in spec.
update_shuffling_cache(state)
state
# TODO candidate for spec?
@ -383,7 +360,7 @@ func get_attestation_participants*(state: BeaconState,
iterator get_attestation_participants_cached*(state: BeaconState,
attestation_data: AttestationData,
bitfield: BitField,
crosslink_committees_cached: var auto): ValidatorIndex =
cache: var StateCache): ValidatorIndex =
## Return the participant indices at for the ``attestation_data`` and
## ``bitfield``.
## Attestation participants in the attestation data are called out in a
@ -402,7 +379,7 @@ iterator get_attestation_participants_cached*(state: BeaconState,
var found = false
for crosslink_committee in get_crosslink_committees_at_slot_cached(
state, attestation_data.slot, false, crosslink_committees_cached):
state, attestation_data.slot, false, cache):
if crosslink_committee.shard == attestation_data.shard:
# TODO this and other attestation-based fields need validation so we don't
# crash on a malicious attestation!

View File

@ -374,10 +374,6 @@ type
validator_registry_update_epoch*: Epoch
# TODO remove or conditionally compile; not in spec anymore
validator_registry_delta_chain_tip*: Eth2Digest ##\
## For light clients to easily track delta
# Randomness and committees
latest_randao_mixes*: array[LATEST_RANDAO_MIXES_LENGTH, Eth2Digest]
previous_shuffling_start_shard*: uint64
@ -417,9 +413,6 @@ type
eth1_data_votes*: seq[Eth1DataVote]
deposit_index*: uint64
# Not in spec. TODO: don't serialize or deserialize this.
shuffling_cache*: ShufflingCache
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#validator
Validator* = object
pubkey*: ValidatorPubKey ##\
@ -510,21 +503,6 @@ type
# TODO: not in spec
CrosslinkCommittee* = tuple[committee: seq[ValidatorIndex], shard: uint64]
ShufflingCache* = object
## https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#get_shuffling
## Note: this definition and the next few definitions make heavy use of
## repetitive computing. Production implementations are expected to
## appropriately use caching/memoization to avoid redoing work.
##
## TODO use native ValidatorIndex, once this doesn't need to be serialized.
## `seed` and `list_size` determine the shuffle. For now, only need two, at
## any given time. If the next_epoch variations of shuffling get called, it
## might increase to three at once.
seeds*: array[2, Eth2Digest]
list_sizes*: array[2, uint64]
index*: int
shuffling_0*: seq[int]
shuffling_1*: seq[int]
func shortValidatorKey*(state: BeaconState, validatorIdx: int): string =
($state.validator_registry[validatorIdx].pubkey)[0..7]

View File

@ -18,17 +18,6 @@ func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] =
for i in 0 ..< N:
result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray
func get_new_recent_block_roots*(old_block_roots: seq[Eth2Digest],
parent_slot, current_slot: int64,
parent_hash: Eth2Digest
): seq[Eth2Digest] =
# Should throw for `current_slot - CYCLE_LENGTH * 2 - 1` according to spec comment
let d = current_slot - parent_slot
result = old_block_roots[d .. ^1]
for _ in 0 ..< min(d, old_block_roots.len):
result.add parent_hash
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#integer_squareroot
func integer_squareroot*(n: SomeInteger): SomeInteger =
## The largest integer ``x`` such that ``x**2`` is less than ``n``.
@ -53,6 +42,7 @@ func get_fork_version*(fork: Fork, epoch: Epoch): array[4, byte] =
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#is_power_of_two
func is_power_of_2*(v: uint64): bool = (v > 0'u64) and (v and (v-1)) == 0
# TODO reuse as necessary/useful for merkle proof building
func merkle_root*(values: openArray[Eth2Digest]): Eth2Digest =
## Merkleize ``values`` (where ``len(values)`` is a power of two) and return
## the Merkle root.
@ -170,6 +160,9 @@ func get_active_index_root(state: BeaconState, epoch: Epoch): Eth2Digest =
## Cannot underflow, since GENESIS_EPOCH > LATEST_RANDAO_MIXES_LENGTH
## and ACTIVATION_EXIT_DELAY > 0.
doAssert GENESIS_EPOCH > LATEST_RANDAO_MIXES_LENGTH
doAssert ACTIVATION_EXIT_DELAY > 0
doAssert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH +
ACTIVATION_EXIT_DELAY < epoch
doAssert epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY
@ -228,7 +221,10 @@ func generate_seed*(state: BeaconState, epoch: Epoch): Eth2Digest =
# Generate a seed for the given ``epoch``.
var seed_input : array[32*3, byte]
# Cannot underflow, since GENESIS_EPOCH > MIN_SEED_LOOKAHEAD
doAssert GENESIS_EPOCH > MIN_SEED_LOOKAHEAD
seed_input[0..31] = get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD).data
seed_input[32..63] = get_active_index_root(state, epoch).data
seed_input[64..95] = int_to_bytes32(epoch)

View File

@ -9,7 +9,7 @@
import
options, nimcrypto, sequtils, math, chronicles,
eth/common,
../ssz,
../ssz, ../beacon_node_types,
./crypto, ./datatypes, ./digest, ./helpers
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#get_shuffling
@ -71,33 +71,21 @@ func get_shuffled_seq*(seed: Eth2Digest,
result = shuffled_active_validator_indices
# https://github.com/ethereum/eth2.0-specs/blob/v0.4.0/specs/core/0_beacon-chain.md#get_shuffling
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#get_shuffling
func get_shuffling*(seed: Eth2Digest,
validators: openArray[Validator],
epoch: Epoch,
shuffling_cache: ShufflingCache
): seq[seq[ValidatorIndex]] =
## This function is factored to facilitate testing with
## https://github.com/ethereum/eth2.0-test-generators/tree/master/permutated_index
## test vectors, which the split of get_shuffling obfuscates.
## TODO fix bad list size but keep consistent with cached values,
## once epoch processing reordering comes around
let list_size = validators.len.uint64
let
active_validator_indices = get_active_validator_indices(validators, epoch)
list_size = active_validator_indices.len.uint64
committees_per_epoch = get_epoch_committee_count(
len(active_validator_indices)).int
# Both mapIt-type-conversions are an SSZ artifact. TODO remove.
shuffled_seq =
if shuffling_cache.seeds[0] == seed and
shuffling_cache.list_sizes[0] == list_size:
mapIt(shuffling_cache.shuffling_0, it.ValidatorIndex)
elif shuffling_cache.seeds[1] == seed and
shuffling_cache.list_sizes[1] == list_size:
mapIt(shuffling_cache.shuffling_1, it.ValidatorIndex)
else:
get_shuffled_seq(seed, list_size)
shuffled_seq = get_shuffled_seq(seed, list_size)
# Split the shuffled list into committees_per_epoch pieces
result = split(shuffled_seq, committees_per_epoch)
@ -214,10 +202,9 @@ func get_crosslink_committees_at_slot*(state: BeaconState, slot: Slot|uint64,
shuffling = get_shuffling(
seed,
state.validator_registry,
shuffling_epoch,
shuffling_epoch
# Not in spec
state.shuffling_cache
)
offset = slot mod SLOTS_PER_EPOCH
committees_per_slot = committees_per_epoch div SLOTS_PER_EPOCH
@ -231,14 +218,14 @@ func get_crosslink_committees_at_slot*(state: BeaconState, slot: Slot|uint64,
iterator get_crosslink_committees_at_slot_cached*(
state: BeaconState, slot: Slot|uint64,
registry_change: bool = false, cache: var auto):
registry_change: bool = false, cache: var StateCache):
CrosslinkCommittee =
let key = (slot.uint64, registry_change)
if key in cache:
for v in cache[key]: yield v
if key in cache.crosslink_committee_cache:
for v in cache.crosslink_committee_cache[key]: yield v
#debugEcho "get_crosslink_committees_at_slot_cached: MISS"
let result = get_crosslink_committees_at_slot(state, slot, registry_change)
cache[key] = result
cache.crosslink_committee_cache[key] = result
for v in result: yield v
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#get_beacon_proposer_index

View File

@ -19,7 +19,7 @@
# the others use NEP-1 - helps grepping identifiers in spec
# * We mix procedural and functional styles for no good reason, except that the
# spec does so also.
# * There are no tests, and likely lots of bugs.
# * There are likely lots of bugs.
# * For indices, we get a mix of uint64, ValidatorIndex and int - this is currently
# swept under the rug with casts
# * The spec uses uint64 for data types, but functions in the spec often assume
@ -32,7 +32,7 @@
import
algorithm, collections/sets, chronicles, math, options, sequtils, tables,
./extras, ./ssz,
./extras, ./ssz, ./beacon_node_types,
./spec/[beaconstate, bitfield, crypto, datatypes, digest, helpers, validator]
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#block-header
@ -533,14 +533,14 @@ func get_attesting_indices(
func get_attesting_indices_cached(
state: BeaconState,
attestations: openArray[PendingAttestation],
crosslink_committee_cache: var auto): HashSet[ValidatorIndex] =
attestations: openArray[PendingAttestation], cache: var StateCache):
HashSet[ValidatorIndex] =
# Union of attesters that participated in some attestations
result = initSet[ValidatorIndex]()
for attestation in attestations:
for validator_index in get_attestation_participants_cached(
state, attestation.data, attestation.aggregation_bitfield,
crosslink_committee_cache):
cache):
result.incl validator_index
func get_attesting_balance(state: BeaconState,
@ -548,11 +548,10 @@ func get_attesting_balance(state: BeaconState,
get_total_balance(state, get_attesting_indices(state, attestations))
func get_attesting_balance_cached(
state: BeaconState,
attestations: seq[PendingAttestation],
crosslink_committees_cache: var auto): Gwei =
state: BeaconState, attestations: seq[PendingAttestation],
cache: var StateCache): Gwei =
get_total_balance(state, get_attesting_indices_cached(
state, attestations, crosslink_committees_cache))
state, attestations, cache))
func get_current_epoch_boundary_attestations(state: BeaconState):
seq[PendingAttestation] =
@ -583,7 +582,7 @@ func lowerThan(candidate, current: Eth2Digest): bool =
false
func get_winning_root_and_participants(
state: BeaconState, shard: Shard, crosslink_committees_cache: var auto):
state: BeaconState, shard: Shard, cache: var StateCache):
tuple[a: Eth2Digest, b: HashSet[ValidatorIndex]] =
let
all_attestations =
@ -617,7 +616,7 @@ func get_winning_root_and_participants(
for r in all_roots:
let root_balance = get_attesting_balance_cached(
state, attestations_for.getOrDefault(r), crosslink_committees_cache)
state, attestations_for.getOrDefault(r), cache)
if (root_balance > winning_root_balance or
(root_balance == winning_root_balance and
lowerThan(winning_root, r))):
@ -627,7 +626,7 @@ func get_winning_root_and_participants(
(winning_root,
get_attesting_indices_cached(
state,
attestations_for.getOrDefault(winning_root), crosslink_committees_cache))
attestations_for.getOrDefault(winning_root), cache))
# Combination of earliest_attestation and inclusion_slot avoiding O(n^2)
# TODO merge/refactor these two functions, which differ only very slightly.
@ -725,8 +724,7 @@ func update_justification_and_finalization(state: var BeaconState) =
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#crosslinks
func process_crosslinks(
state: var BeaconState, crosslink_committee_cache: var auto,
winning_root_participants_cache: var auto) =
state: var BeaconState, per_epoch_cache: var StateCache) =
let
current_epoch = get_current_epoch(state)
previous_epoch = current_epoch - 1
@ -742,22 +740,21 @@ func process_crosslinks(
GENESIS_SLOT.uint64, get_epoch_start_slot(previous_epoch).uint64) ..<
get_epoch_start_slot(next_epoch).uint64:
for cas in get_crosslink_committees_at_slot_cached(
state, slot, false, crosslink_committee_cache):
state, slot, false, per_epoch_cache):
let
(crosslink_committee, shard) = cas
# In general, it'll loop over the same shards twice, and
# get_winning_root_and_participants is defined to return
# the same results from the previous epoch as current.
(winning_root, participants) =
if shard notin winning_root_participants_cache:
get_winning_root_and_participants(
state, shard, crosslink_committee_cache)
if shard notin per_epoch_cache.winning_root_participants_cache:
get_winning_root_and_participants(state, shard, per_epoch_cache)
else:
(ZERO_HASH, winning_root_participants_cache[shard])
(ZERO_HASH, per_epoch_cache.winning_root_participants_cache[shard])
participating_balance = get_total_balance(state, participants)
total_balance = get_total_balance(state, crosslink_committee)
winning_root_participants_cache[shard] = participants
per_epoch_cache.winning_root_participants_cache[shard] = participants
if 3'u64 * participating_balance >= 2'u64 * total_balance:
# Check not from spec; seems kludgy
@ -936,9 +933,7 @@ func get_justification_and_finalization_deltas(state: BeaconState):
compute_inactivity_leak_deltas(state)
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#crosslinks-1
func get_crosslink_deltas(
state: BeaconState, crosslink_committees_cache: var auto,
winning_root_participants_cache: var auto):
func get_crosslink_deltas(state: BeaconState, cache: var StateCache):
tuple[a: seq[Gwei], b: seq[Gwei]] =
# deltas[0] for rewards
# deltas[1] for penalties
@ -953,15 +948,14 @@ func get_crosslink_deltas(
get_epoch_start_slot(get_current_epoch(state))
for slot in previous_epoch_start_slot.uint64 ..<
current_epoch_start_slot.uint64:
for cas in get_crosslink_committees_at_slot_cached(state, slot, false, crosslink_committees_cache):
for cas in get_crosslink_committees_at_slot_cached(state, slot, false, cache):
let
(crosslink_committee, shard) = cas
(winning_root, participants) =
if shard notin winning_root_participants_cache:
get_winning_root_and_participants(
state, shard, crosslink_committees_cache)
if shard notin cache.winning_root_participants_cache:
get_winning_root_and_participants(state, shard, cache)
else:
(ZERO_HASH, winning_root_participants_cache[shard])
(ZERO_HASH, cache.winning_root_participants_cache[shard])
participating_balance = get_total_balance(state, participants)
total_balance = get_total_balance(state, crosslink_committee)
for index in crosslink_committee:
@ -975,13 +969,10 @@ func get_crosslink_deltas(
deltas
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#apply-rewards
func apply_rewards(
state: var BeaconState, crosslink_committees_cache: var auto,
winning_root_participants_cache: var auto) =
func apply_rewards(state: var BeaconState, cache: var StateCache) =
let
deltas1 = get_justification_and_finalization_deltas(state)
deltas2 = get_crosslink_deltas(
state, crosslink_committees_cache, winning_root_participants_cache)
deltas2 = get_crosslink_deltas(state, cache)
for i in 0 ..< len(state.validator_registry):
state.validator_balances[i] =
max(
@ -1087,6 +1078,12 @@ func finish_epoch_update(state: var BeaconState) =
state.current_epoch_attestations = @[]
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#per-epoch-processing
func get_empty_per_epoch_cache(): StateCache =
result.crosslink_committee_cache =
initTable[tuple[a: uint64, b: bool], seq[CrosslinkCommittee]]()
result.winning_root_participants_cache =
initTable[Shard, HashSet[ValidatorIndex]]()
func processEpoch(state: var BeaconState) =
if not (state.slot > GENESIS_SLOT and
(state.slot + 1) mod SLOTS_PER_EPOCH == 0):
@ -1095,21 +1092,16 @@ func processEpoch(state: var BeaconState) =
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#justification
update_justification_and_finalization(state)
var
crosslink_committee_cache =
initTable[tuple[a: uint64, b: bool], seq[CrosslinkCommittee]]()
winning_root_participants_cache =
initTable[Shard, HashSet[ValidatorIndex]]()
var per_epoch_cache = get_empty_per_epoch_cache()
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#crosslinks
process_crosslinks(
state, crosslink_committee_cache, winning_root_participants_cache)
process_crosslinks(state, per_epoch_cache)
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#eth1-data
maybe_reset_eth1_period(state)
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#apply-rewards
apply_rewards(
state, crosslink_committee_cache, winning_root_participants_cache)
apply_rewards(state, per_epoch_cache)
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#ejections
process_ejections(state)
@ -1117,9 +1109,6 @@ func processEpoch(state: var BeaconState) =
# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#validator-registry-and-shuffling-seed-data
update_registry_and_shuffling_data(state)
# Not from spec.
updateShufflingCache(state)
## Regardless of whether or not a validator set change happens run
## process_slashings(state) and process_exit_queue(state)
process_slashings(state)

View File

@ -15,7 +15,7 @@ type Timers = enum
tBlock = "Process non-epoch slot with block"
tEpoch = "Process epoch slot with block"
tHashBlock = "Tree-hash block"
tShuffle = "Retrieve committe once using get_crosslink_committees_at_slot"
tShuffle = "Retrieve committee once using get_crosslink_committees_at_slot"
tAttest = "Combine committee attestations"
template withTimer(stats: var RunningStat, body: untyped) =
@ -46,11 +46,11 @@ proc writeJson*(prefix, slot, v: auto) =
write(f, pretty(%*(v)))
cli do(slots = 1945,
validators = SLOTS_PER_EPOCH, # One per shard is minimum
validators = SLOTS_PER_EPOCH * 5, # One per shard is minimum
json_interval = SLOTS_PER_EPOCH,
prefix = 0,
attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.9,
validate = false):
validate = true):
let
flags = if validate: {} else: {skipValidation}
genesisState = get_genesis_beacon_state(

View File

@ -92,4 +92,4 @@ suite "Tree hashing":
let vr = BeaconState()
check:
$hash_tree_root(vr) ==
"DC751EF09987283D52483C75690234DDD75FFDAF1A844CD56FE1173465B5597A"
"17E30FC7BC442CEF2045C4FB3CC6B1D975F041C4629DA7395B607281FD1521A6"

View File

@ -33,7 +33,7 @@ suite "Validators":
Validator(
exit_epoch: FAR_FUTURE_EPOCH
), num_validators)
s = get_shuffling(Eth2Digest(), validators, GENESIS_EPOCH, ShufflingCache())
s = get_shuffling(Eth2Digest(), validators, GENESIS_EPOCH)
committees = get_epoch_committee_count(len(validators)).int
check:
# def b(s): return "Eth2Digest(data: [0x" + "'u8, 0x".join((s[i:i+2] for i in range(0, 64, 2))) + "'u8])"