From 46cfce652dcee2c3381daa50e571a98288d79726 Mon Sep 17 00:00:00 2001 From: Dustin Brody Date: Thu, 23 May 2019 11:13:02 +0000 Subject: [PATCH] initial new get_beacon_proposer_index and dependencies (get_shuffled_index, get_shard_delta, get_epoch_start_shard, get_crosslink_committee) (#266) * initial new get_beacon_proposer_index and dependencies (get_shuffled_index, get_shard_delta, get_epoch_start_shard, get_crosslink_committee) * more 0.6.1 updates * clean up debugEchos and fix validator keygen to create correct indices --- beacon_chain/beacon_node.nim | 3 +- beacon_chain/spec/beaconstate.nim | 2 +- beacon_chain/spec/datatypes.nim | 1 + beacon_chain/spec/helpers.nim | 29 +++---- beacon_chain/spec/validator.nim | 136 +++++++++++++++++++++++------- beacon_chain/state_transition.nim | 6 +- beacon_chain/validator_keygen.nim | 3 +- tests/testutil.nim | 4 +- 8 files changed, 128 insertions(+), 56 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 60db1234c..740297617 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -497,7 +497,8 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot): # revisit this - we should be able to advance behind node.blockPool.withState(node.stateCache, BlockSlot(blck: head, slot: slot)): let - proposerIdx = get_beacon_proposer_index(state, slot) + # TODO this probably isn't correct, check re blob/v0.5.1 + proposerIdx = get_beacon_proposer_index(state) validator = node.getAttachedValidator(state, proposerIdx) if validator != nil: diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 735a7f60f..d10426d01 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -178,7 +178,7 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) = ] += get_effective_balance(state, index) let - whistleblower_index = get_beacon_proposer_index(state, state.slot) + whistleblower_index = get_beacon_proposer_index(state) whistleblower_reward = get_effective_balance(state, index) div WHISTLEBLOWER_REWARD_QUOTIENT diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index b7b82cd17..4aefb55c8 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -391,6 +391,7 @@ type # Randomness and committees latest_randao_mixes*: array[LATEST_RANDAO_MIXES_LENGTH, Eth2Digest] + latest_start_shard*: Shard previous_shuffling_start_shard*: uint64 current_shuffling_start_shard*: uint64 previous_shuffling_epoch*: Epoch diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 3cf0cb090..6130b19ba 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -146,29 +146,20 @@ func get_current_epoch*(state: BeaconState): Epoch = doAssert state.slot >= GENESIS_SLOT, $state.slot slot_to_epoch(state.slot) -# https://github.com/ethereum/eth2.0-specs/blob/v0.6.0/specs/core/0_beacon-chain.md#get_randao_mix +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_randao_mix func get_randao_mix*(state: BeaconState, epoch: Epoch): Eth2Digest = ## Returns the randao mix at a recent ``epoch``. - - # Cannot underflow, since GENESIS_EPOCH > LATEST_RANDAO_MIXES_LENGTH - doAssert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch - doAssert epoch <= get_current_epoch(state) - + ## ``epoch`` expected to be between + ## (current_epoch - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, + ## current_epoch + ACTIVATION_EXIT_DELAY]. state.latest_randao_mixes[epoch mod LATEST_RANDAO_MIXES_LENGTH] -# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#get_active_index_root +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_active_index_root func get_active_index_root(state: BeaconState, epoch: Epoch): Eth2Digest = # Returns the index root at a recent ``epoch``. - - ## Cannot underflow, since GENESIS_EPOCH > LATEST_RANDAO_MIXES_LENGTH - ## and ACTIVATION_EXIT_DELAY > 0. - doAssert GENESIS_EPOCH > LATEST_RANDAO_MIXES_LENGTH - doAssert ACTIVATION_EXIT_DELAY > 0 - - doAssert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + - ACTIVATION_EXIT_DELAY < epoch - doAssert epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY + ## ``epoch`` expected to be between + ## (current_epoch - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, current_epoch + ACTIVATION_EXIT_DELAY]. state.latest_active_index_roots[epoch mod LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#bytes_to_int @@ -235,9 +226,9 @@ func generate_seed*(state: BeaconState, epoch: Epoch): Eth2Digest = var seed_input : array[32*3, byte] - doAssert GENESIS_EPOCH > MIN_SEED_LOOKAHEAD - - seed_input[0..31] = get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD).data + seed_input[0..31] = + get_randao_mix(state, + epoch + LATEST_RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD).data seed_input[32..63] = get_active_index_root(state, epoch).data seed_input[64..95] = int_to_bytes32(epoch) eth2hash(seed_input) diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 0e98c9a70..264de1bd6 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -80,6 +80,44 @@ func get_shuffled_seq*(seed: Eth2Digest, result = shuffled_active_validator_indices +func get_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Eth2Digest): uint64 = + ## Return the shuffled validator index corresponding to ``seed`` (and + ## ``index_count``). + ## https://github.com/status-im/nim-beacon-chain/blob/f77016af6818ad2c853f6c9e2751b17548e0222e/beacon_chain/spec/validator.nim#L15 + + doAssert index.uint64 < index_count + doAssert index_count <= 2'u64^40 + + result = index + var pivot_buffer: array[(32+1), byte] + var source_buffer: array[(32+1+4), byte] + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for round in 0 ..< SHUFFLE_ROUND_COUNT: + pivot_buffer[0..31] = seed.data + let round_bytes1 = int_to_bytes1(round)[0] + pivot_buffer[32] = round_bytes1 + + let + pivot = bytes_to_int(eth2hash(pivot_buffer).data[0..7]) mod index_count + flip = (pivot - index) mod index_count + position = max(index.uint64, flip) + + ## Tradeoff between slicing (if reusing one larger buffer) and additional + ## copies here of seed and `int_to_bytes1(round)`. + source_buffer[0..31] = seed.data + source_buffer[32] = round_bytes1 + source_buffer[33..36] = int_to_bytes4(position div 256) + + let + source = eth2hash(source_buffer).data + byte_value = source[(position mod 256) div 8] + bit = (byte_value shr (position mod 8)) mod 2 + + if bit != 0: + result = flip + # https://github.com/ethereum/eth2.0-specs/blob/v0.5.1/specs/core/0_beacon-chain.md#get_shuffling func get_shuffling*(seed: Eth2Digest, validators: openArray[Validator], @@ -212,36 +250,76 @@ iterator get_crosslink_committees_at_slot_cached*( cache.crosslink_committee_cache[key] = result for v in result: yield v -# https://github.com/ethereum/eth2.0-specs/blob/v0.5.0/specs/core/0_beacon-chain.md#get_beacon_proposer_index -# TODO remove/merge these back together once 0.5.1 callers removed -func get_beacon_proposer_index*(state: BeaconState, slot: Slot): ValidatorIndex = - ## From Casper RPJ mini-spec: - ## When slot i begins, validator Vidx is expected - ## to create ("propose") a block, which contains a pointer to some parent block - ## that they perceive as the "head of the chain", - ## and includes all of the **attestations** that they know about - ## that have not yet been included into that chain. - ## - ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) - ## Returns the beacon proposer index for the ``slot``. - # TODO this index is invalid outside of the block state transition function - # because presently, `state.slot += 1` happens before this function - # is called - see also testutil.getNextBeaconProposerIndex - # TODO is the above still true? the shuffling has changed since it was written +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_shard_delta +func get_shard_delta(state: BeaconState, epoch: Epoch): uint64 = + ## Return the number of shards to increment ``state.latest_start_shard`` + ## during ``epoch``. + min(get_epoch_committee_count(state, epoch), + (SHARD_COUNT - SHARD_COUNT div SLOTS_PER_EPOCH).uint64) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_epoch_start_shard +func get_epoch_start_shard(state: BeaconState, epoch: Epoch): Shard = + doAssert epoch <= get_current_epoch(state) + 1 + var + check_epoch = get_current_epoch(state) + 1 + shard = + (state.latest_start_shard + + get_shard_delta(state, get_current_epoch(state))) mod SHARD_COUNT + while check_epoch > epoch: + check_epoch -= 1 + shard = (shard + SHARD_COUNT - get_shard_delta(state, check_epoch)) mod + SHARD_COUNT + return shard + +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#compute_committee +func compute_committee(indices: seq[ValidatorIndex], seed: Eth2Digest, + index: uint64, count: uint64): seq[ValidatorIndex] = let - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 + start = (len(indices).uint64 * index) div count + endIdx = (len(indices).uint64 * (index + 1)) div count + doAssert endIdx.int - start.int > 0 + mapIt( + start.int .. (endIdx.int-1), + indices[ + get_shuffled_index(it.ValidatorIndex, len(indices).uint64, seed).int]) - doAssert previous_epoch <= epoch - doAssert epoch <= next_epoch - - let (first_committee, _) = get_crosslink_committees_at_slot(state, slot)[0] - let idx = int(slot mod uint64(first_committee.len)) - first_committee[idx] +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_crosslink_committee +func get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard): + seq[ValidatorIndex] = + compute_committee( + get_active_validator_indices(state, epoch), + generate_seed(state, epoch), + (shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) mod SHARD_COUNT, + get_epoch_committee_count(state, epoch), + ) +# https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/specs/core/0_beacon-chain.md#get_beacon_proposer_index func get_beacon_proposer_index*(state: BeaconState): ValidatorIndex = - # Return the beacon proposer index at ``state.slot``. - # TODO there are some other changes here - get_beacon_proposer_index(state, state.slot) + # Return the current beacon proposer index. + const + MAX_RANDOM_BYTE = 255 + + let + epoch = get_current_epoch(state) + committees_per_slot = + get_epoch_committee_count(state, epoch) div SLOTS_PER_EPOCH + offset = committees_per_slot * (state.slot mod SLOTS_PER_EPOCH) + shard = (get_epoch_start_shard(state, epoch) + offset) mod SHARD_COUNT + first_committee = get_crosslink_committee(state, epoch, shard) + seed = generate_seed(state, epoch) + + var + i = 0 + buffer: array[(32+8), byte] + buffer[0..31] = seed.data + while true: + buffer[32..39] = int_to_bytes8(i.uint64 div 32) + let + candidate_index = first_committee[((epoch + i.uint64) mod + len(first_committee).uint64).int] + random_byte = (eth2hash(buffer).data)[i mod 32] + effective_balance = + state.validator_registry[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return candidate_index + i += 1 diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim index d552b6516..9365c0207 100644 --- a/beacon_chain/state_transition.nim +++ b/beacon_chain/state_transition.nim @@ -57,7 +57,7 @@ proc processBlockHeader( state.latest_block_header = get_temporary_block_header(blck) let proposer = - state.validator_registry[get_beacon_proposer_index(state, state.slot)] + state.validator_registry[get_beacon_proposer_index(state)] if skipValidation notin flags and not bls_verify( proposer.pubkey, signing_root(blck).data, @@ -75,7 +75,7 @@ proc processBlockHeader( proc processRandao( state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool = let - proposer_index = get_beacon_proposer_index(state, state.slot) + proposer_index = get_beacon_proposer_index(state) proposer = addr state.validator_registry[proposer_index] if skipValidation notin flags: @@ -849,7 +849,7 @@ func compute_normal_justification_and_finalization_deltas(state: BeaconState): # Proposer bonus if index in previous_epoch_attestation_indices: let proposer_index = - get_beacon_proposer_index(state, inclusion_slot[index]) + get_beacon_proposer_index(state) deltas[0][proposer_index] += get_base_reward(state, index) div PROPOSER_REWARD_QUOTIENT deltas diff --git a/beacon_chain/validator_keygen.nim b/beacon_chain/validator_keygen.nim index f5ee131db..5e33ca8fd 100644 --- a/beacon_chain/validator_keygen.nim +++ b/beacon_chain/validator_keygen.nim @@ -39,7 +39,8 @@ cli do (totalValidators: int = 125000, data: DepositData( amount: MAX_EFFECTIVE_BALANCE, pubkey: pubKey, - withdrawal_credentials: withdrawalCredentials)) + withdrawal_credentials: withdrawalCredentials), + index: i.uint64) deposit.data.signature = bls_sign(privkey, signing_root(deposit.data).data, diff --git a/tests/testutil.nim b/tests/testutil.nim index 23e46d49b..eda14281c 100644 --- a/tests/testutil.nim +++ b/tests/testutil.nim @@ -63,7 +63,7 @@ func getNextBeaconProposerIndex*(state: BeaconState): ValidatorIndex = # see get_shard_committees_index var next_state = state next_state.slot += 1 - get_beacon_proposer_index(next_state, next_state.slot) + get_beacon_proposer_index(next_state) proc addBlock*( state: var BeaconState, previous_block_root: Eth2Digest, @@ -74,7 +74,7 @@ proc addBlock*( # but avoids some slow block copies state.slot += 1 - let proposer_index = get_beacon_proposer_index(state, state.slot) + let proposer_index = get_beacon_proposer_index(state) state.slot -= 1 # Ferret out remaining GENESIS_EPOCH == 0 assumptions in test code