convert some asserts to doAsserts to keep them in release mode builds; rename get_initial_beacon_state to get_genesis_beacon_state to track spec; switch target spec version to 0.3.0; switch references to penalize_validator to slash_validator/slashValidator to track spec; make some function returns safer by omitting 'return' (#132)

* convert some asserts to doAsserts to keep them in release mode builds; rename get_initial_beacon_state to get_genesis_beacon_state to track spec; switch target spec version to 0.3.0; switch references to penalize_validator to slash_validator/slashValidator to track spec; make some function returns safer by omitting 'return'

* 2x shuffling speedup by hoisting pivot calculations per https://github.com/protolambda/eth2-shuffle
This commit is contained in:
Dustin Brody 2019-02-22 09:56:45 +00:00 committed by Mamy Ratsimbazafy
parent 1d13007627
commit 2d52d5cbfe
12 changed files with 147 additions and 51 deletions

View File

@ -132,7 +132,7 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
## Note that this function mutates ``state``.
let validator = addr state.validator_registry[index]
assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
doAssert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
## [TO BE REMOVED IN PHASE 2]
exit_validator(state, index)
@ -153,12 +153,13 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
validator.withdrawable_epoch = get_current_epoch(state) +
LATEST_SLASHED_EXIT_LENGTH
func get_initial_beacon_state*(
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#on-genesis
func get_genesis_beacon_state*(
initial_validator_deposits: openArray[Deposit],
genesis_time: uint64,
latest_eth1_data: Eth1Data,
flags: UpdateFlags = {}): BeaconState =
## Get the initial ``BeaconState``.
## Get the genesis ``BeaconState``.
##
## Before the beacon chain starts, validators will register in the Eth1 chain
## and deposit ETH. When enough many validators have registered, a
@ -186,10 +187,12 @@ func get_initial_beacon_state*(
validator_registry_update_epoch: GENESIS_EPOCH,
# validator_registry and validator_balances automatically initalized
# TODO remove or conditionally compile; not in spec anymore
validator_registry_delta_chain_tip: ZERO_HASH,
# Randomness and committees
# latest_randao_mixes automatically initialized
previous_shuffling_start_shard: GENESIS_START_SHARD,
current_shuffling_start_shard: GENESIS_START_SHARD,
previous_shuffling_epoch: GENESIS_EPOCH,
@ -205,6 +208,11 @@ func get_initial_beacon_state*(
# Deposit root
latest_eth1_data: latest_eth1_data,
# Recent state
# TODO properly initialize latest_crosslinks
# latest_block_roots, latest_active_index_roots, latest_slashed_balances,
# latest_attestations, and batched_block_roots automatically initialized.
)
# Process initial deposits
@ -223,6 +231,12 @@ func get_initial_beacon_state*(
if get_effective_balance(state, vi) >= MAX_DEPOSIT_AMOUNT:
activate_validator(state, vi, true)
let genesis_active_index_root = Eth2Digest(data: hash_tree_root(
get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)))
for index in 0 ..< LATEST_ACTIVE_INDEX_ROOTS_LENGTH:
state.latest_active_index_roots[index] = genesis_active_index_root
state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH)
state
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_block_root

View File

@ -40,7 +40,7 @@ import
# TODO Many of these constants should go into a config object that can be used
# to run.. well.. a chain with different constants!
const
SPEC_VERSION* = "0.2.0" ## \
SPEC_VERSION* = "0.3.0" ## \
## Spec version we're aiming to be compatible with, right now
## TODO: improve this scheme once we can negotiate versions in protocol
@ -146,7 +146,7 @@ const
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#state-list-lengths
LATEST_BLOCK_ROOTS_LENGTH* = 2'u64^13
LATEST_RANDAO_MIXES_LENGTH* = 2'u64^13
LATEST_ACTIVE_INDEX_ROOTS_LENGTH* = 2'u64^13
LATEST_ACTIVE_INDEX_ROOTS_LENGTH* = 8192 # 2'u64^13, epochs
LATEST_SLASHED_EXIT_LENGTH* = 8192 # epochs
# Reward and penalty quotients

View File

@ -88,12 +88,13 @@ func is_power_of_2*(v: uint64): bool = (v and (v-1)) == 0
func merkle_root*(values: openArray[Eth2Digest]): Eth2Digest =
## Merkleize ``values`` (where ``len(values)`` is a power of two) and return
## the Merkle root.
## https://crypto.stackexchange.com/questions/43430/what-is-the-reason-to-separate-domains-in-the-internal-hash-algorithm-of-a-merkl
let num_values = len(values)
# Simplifies boundary conditions
assert is_power_of_two(num_values)
assert num_values >= 2
assert num_values mod 2 == 0
doAssert is_power_of_two(num_values)
doAssert num_values >= 2
doAssert num_values mod 2 == 0
# TODO reverse ``o`` order and use newSeqWith to avoid pointless zero-filling.
var o = repeat(ZERO_HASH, len(values))

View File

@ -12,21 +12,15 @@ import
../ssz,
./crypto, ./datatypes, ./digest, ./helpers
# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#get_permuted_index
func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest): uint64 =
# TODO remove once there are test vectors to check with directly
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_permuted_index
func get_permuted_index_spec(index: uint64, list_size: uint64, seed: Eth2Digest): uint64 =
## Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1`
## with ``seed`` as entropy.
##
## Utilizes 'swap or not' shuffling found in
## https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
## See the 'generalized domain' algorithm on page 3.
##
## This is very slow. Lots of pointless memory allocations. Some methods
## can use updating-with-incremental-chunks-of-preallocated source data,
## while others might have caller-allocated buffers, etc. What this does
## is egregious though. It is allocating tens of thousands of identical,
## small buffers, etc. There's nothing fundamentally inefficient about a
## per-index approach, but naive implementations require optimization.
result = index
var pivot_buffer: array[(32+1), byte]
var source_buffer: array[(32+1+4), byte]
@ -55,21 +49,14 @@ func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest): uin
if bit != 0:
result = flip
# TODO remove once there are test vectors to check with directly
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
func get_shuffling*(seed: Eth2Digest,
validators: openArray[Validator],
epoch: Epoch
): seq[seq[ValidatorIndex]] =
func get_shuffling_spec*(seed: Eth2Digest, validators: openArray[Validator],
epoch: Epoch): seq[seq[ValidatorIndex]] =
## Shuffles ``validators`` into crosslink committees seeded by ``seed`` and
## ``slot``.
## Returns a list of ``SLOTS_PER_EPOCH * committees_per_slot`` committees where
## each committee is itself a list of validator indices.
##
## TODO: write unit tests for if this produces an "interesting" permutation
## i.e. every number in input exists once and >95%, say, in a different
## place than it began (there are more rigorous approaches, but something)
## By design, get_permuted_index is somewhat difficult to test on its own;
## get_shuffling is first layer where that's straightforward.
let
active_validator_indices = get_active_validator_indices(validators, epoch)
@ -79,13 +66,93 @@ func get_shuffling*(seed: Eth2Digest,
shuffled_active_validator_indices = mapIt(
active_validator_indices,
active_validator_indices[get_permuted_index(
active_validator_indices[get_permuted_index_spec(
it, len(active_validator_indices).uint64, seed).int])
# Split the shuffled list into committees_per_epoch pieces
result = split(shuffled_active_validator_indices, committees_per_epoch)
assert result.len() == committees_per_epoch # what split should do..
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_permuted_index
func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest,
pivots: seq[uint64]): uint64 =
## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go
## Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1`
## with ``seed`` as entropy.
##
## Utilizes 'swap or not' shuffling found in
## https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
## See the 'generalized domain' algorithm on page 3.
result = index
var source_buffer: array[(32+1+4), byte]
doAssert len(pivots) == SHUFFLE_ROUND_COUNT
for round in 0 ..< SHUFFLE_ROUND_COUNT:
let
pivot = pivots[round]
flip = (pivot - index) mod list_size
position = max(index, flip)
round_bytes1 = int_to_bytes1(round)[0]
## Tradeoff between slicing (if reusing one larger buffer) and additional
## copies here of seed and `int_to_bytes1(round)`.
source_buffer[0..31] = seed.data
source_buffer[32] = round_bytes1
source_buffer[33..36] = int_to_bytes4(position div 256)
let
source = eth2hash(source_buffer).data
byte_value = source[(position mod 256) div 8]
bit = (byte_value shr (position mod 8)) mod 2
if bit != 0:
result = flip
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
func get_shuffling*(seed: Eth2Digest,
validators: openArray[Validator],
epoch: Epoch
): seq[seq[ValidatorIndex]] =
## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go
## Shuffles ``validators`` into crosslink committees seeded by ``seed`` and
## ``slot``.
## Returns a list of ``SLOTS_PER_EPOCH * committees_per_slot`` committees where
## each committee is itself a list of validator indices.
## The pivot's a function of seed and round only, so precalculate all
## SHUFFLE_ROUND_COUNT pivots, using one buffer.
let
active_validator_indices = get_active_validator_indices(validators, epoch)
list_size = active_validator_indices.len.uint64
var
pivot_buffer: array[(32+1), byte]
# Allow Nim stdlib to preallocate the correct seq size.
pivots = repeat(0'u64, SHUFFLE_ROUND_COUNT)
# This doesn't change across rounds.
pivot_buffer[0..31] = seed.data
for round in 0 ..< SHUFFLE_ROUND_COUNT:
let round_bytes1 = int_to_bytes1(round)[0]
pivot_buffer[32] = round_bytes1
let pivot = bytes_to_int(eth2hash(pivot_buffer).data[0..7]) mod list_size
pivots[round] = pivot
let
committees_per_epoch = get_epoch_committee_count(
len(active_validator_indices)).int
shuffled_active_validator_indices = mapIt(
active_validator_indices,
active_validator_indices[get_permuted_index(
it, len(active_validator_indices).uint64, seed, pivots).int])
# Split the shuffled list into committees_per_epoch pieces
result = split(shuffled_active_validator_indices, committees_per_epoch)
assert result.len() == committees_per_epoch # what split should do..
func get_new_validator_registry_delta_chain_tip*(
current_validator_registry_delta_chain_tip: Eth2Digest,
index: ValidatorIndex,

View File

@ -39,7 +39,7 @@ func flatten[T](v: openArray[seq[T]]): seq[T] =
# TODO not in nim - doh.
for x in v: result.add x
# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#proposer-signature
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#proposer-signature
func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
## When creating a block, the proposer will sign a version of the block that
## doesn't contain the data (chicken and egg), then add the signature to that
@ -62,7 +62,7 @@ func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
proposal_root.data, blck.signature,
get_domain(state.fork, get_current_epoch(state), DOMAIN_PROPOSAL))
# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#randao
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#randao
proc processRandao(
state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
let
@ -93,9 +93,9 @@ proc processRandao(
true
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#eth1-data
func processDepositRoot(state: var BeaconState, blck: BeaconBlock) =
## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1-data
# TODO verify that there's at most one match
for x in state.eth1_data_votes.mitems():
if blck.eth1_data == x.eth1_data:
x.vote_count += 1
@ -106,28 +106,34 @@ func processDepositRoot(state: var BeaconState, blck: BeaconBlock) =
vote_count: 1
)
# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#penalize_validator
func penalizeValidator(state: var BeaconState, index: ValidatorIndex) =
## Penalize the validator of the given ``index``.
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slashValidator
func slashValidator(state: var BeaconState, index: ValidatorIndex) =
## Slash the validator of the given ``index``.
## Note that this function mutates ``state``.
exit_validator(state, index)
var validator = addr state.validator_registry[index]
doAssert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
## [TO BE REMOVED IN PHASE 2]
exit_validator(state, index)
state.latest_slashed_balances[(get_current_epoch(state) mod
LATEST_SLASHED_EXIT_LENGTH).int] += get_effective_balance(state,
index.ValidatorIndex)
let
whistleblower_index = get_beacon_proposer_index(state, state.slot)
whistleblower_reward = get_effective_balance(state, index) div WHISTLEBLOWER_REWARD_QUOTIENT
whistleblower_reward = get_effective_balance(state, index) div
WHISTLEBLOWER_REWARD_QUOTIENT
state.validator_balances[whistleblower_index] += whistleblower_reward
state.validator_balances[index] -= whistleblower_reward
validator.slashed_epoch = get_current_epoch(state)
# v0.3.0 spec bug, fixed later, involving renamed constants. Use v0.3.0 name.
validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#proposer-slashings-1
proc processProposerSlashings(
state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposer-slashings-1
if len(blck.body.proposer_slashings) > MAX_PROPOSER_SLASHINGS:
notice "PropSlash: too many!",
proposer_slashings = len(blck.body.proposer_slashings)
@ -174,7 +180,7 @@ proc processProposerSlashings(
notice "PropSlash: penalized slot"
return false
penalizeValidator(state, proposer_slashing.proposer_index.ValidatorIndex)
slashValidator(state, proposer_slashing.proposer_index.ValidatorIndex)
return true
@ -227,8 +233,8 @@ func verify_slashable_attestation(state: BeaconState, slashable_attestation: Sla
),
)
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#attester-slashings-1
proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock): bool =
## https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attester-slashings-1
if len(blck.body.attester_slashings) > MAX_ATTESTER_SLASHINGS:
notice "CaspSlash: too many!"
return false
@ -267,9 +273,9 @@ proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock): bool =
for index in slashable_indices:
if state.validator_registry[index.int].slashed_epoch > get_current_epoch(state):
penalize_validator(state, index.ValidatorIndex)
slash_validator(state, index.ValidatorIndex)
return true
true
proc processAttestations(
state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
@ -355,7 +361,11 @@ func processSlot(state: var BeaconState, previous_block_root: Eth2Digest) =
## slot, a proposer creates a block to represent the state of the beacon
## chain at that time. In case the proposer is missing, it may happen that
## the no block is produced during the slot.
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slot
state.slot += 1
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#block-roots
state.latest_block_roots[(state.slot - 1) mod LATEST_BLOCK_ROOTS_LENGTH] =
previous_block_root
if state.slot mod LATEST_BLOCK_ROOTS_LENGTH == 0:
@ -369,7 +379,7 @@ proc processBlock(
# TODO when there's a failure, we should reset the state!
# TODO probably better to do all verification first, then apply state changes
# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#slot-1
# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slot-1
if not (blck.slot == state.slot):
notice "Unexpected block slot number",
blockSlot = blck.slot,

View File

@ -31,7 +31,7 @@ proc obtainTrustedStateSnapshot*(db: BeaconChainDB): Future[BeaconState] {.async
proc createStateSnapshot*(
startup: ChainStartupData, genesisOffset: int, outFile: string) =
let initialState = get_initial_beacon_state(
let initialState = get_genesis_beacon_state(
startup.validatorDeposits,
uint64(int(fastEpochTime() div 1000) + genesisOffset),
Eth1Data(), {})

View File

@ -5,7 +5,7 @@ import
../tests/testutil
proc stateSize(deposits: int, maxContent = false) =
var state = get_initial_beacon_state(
var state = get_genesis_beacon_state(
makeInitialDeposits(deposits), 0, Eth1Data(), {skipValidation})
if maxContent:

View File

@ -24,7 +24,7 @@ cli do(slots = 1945,
validate = false):
let
flags = if validate: {} else: {skipValidation}
genesisState = get_initial_beacon_state(
genesisState = get_genesis_beacon_state(
makeInitialDeposits(validators, flags), 0, Eth1Data(), flags)
genesisBlock = makeGenesisBlock(genesisState)

View File

@ -18,7 +18,7 @@ suite "Attestation pool processing":
let
# Genesis state with minimal number of deposits
# TODO bls verification is a bit of a bottleneck here
genesisState = get_initial_beacon_state(
genesisState = get_genesis_beacon_state(
makeInitialDeposits(), 0, Eth1Data(), {skipValidation})
genesisBlock = makeGenesisBlock(genesisState)
genesisRoot = hash_tree_root_final(genesisBlock)

View File

@ -12,7 +12,7 @@ import
../beacon_chain/spec/[beaconstate, datatypes, digest]
suite "Beacon state":
test "Smoke test get_initial_beacon_state":
let state = get_initial_beacon_state(
test "Smoke test get_genesis_beacon_state":
let state = get_genesis_beacon_state(
makeInitialDeposits(SLOTS_PER_EPOCH, {}), 0, Eth1Data(), {})
check: state.validator_registry.len == SLOTS_PER_EPOCH

View File

@ -18,7 +18,7 @@ suite "Block processing":
let
# Genesis state with minimal number of deposits
# TODO bls verification is a bit of a bottleneck here
genesisState = get_initial_beacon_state(
genesisState = get_genesis_beacon_state(
makeInitialDeposits(), 0, Eth1Data(), {})
genesisBlock = makeGenesisBlock(genesisState)

View File

@ -28,8 +28,12 @@ suite "Validators":
exit_epoch: FAR_FUTURE_EPOCH
), num_validators)
s = get_shuffling(Eth2Digest(), validators, 0)
#s_spec = get_shuffling_spec(Eth2Digest(), validators, 0)
committees = get_epoch_committee_count(len(validators)).int
check:
## Enable checking equivalence of spec and optimized versions.
## TODO enable checking against YAML test vectors
## s == s_spec
s.len == committees
# 32k validators: SLOTS_PER_EPOCH slots * committee_count_per_slot =
# get_epoch_committee_count committees.