From 3ce94623e3c77ef4337089b59e115d207657e4a7 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 23 Nov 2018 13:42:47 -0600 Subject: [PATCH 1/4] spec updates --- beacon_chain/datatypes.nim | 38 +++++++----- beacon_chain/private/helpers.nim | 103 +++++++++++++++++-------------- beacon_chain/validator.nim | 47 ++++++++++++++ 3 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 beacon_chain/validator.nim diff --git a/beacon_chain/datatypes.nim b/beacon_chain/datatypes.nim index 5f0350695..3872d2e41 100644 --- a/beacon_chain/datatypes.nim +++ b/beacon_chain/datatypes.nim @@ -98,8 +98,8 @@ type last_finalized_slot*: uint64 # Last finalized slot last_justified_slot*: uint64 # Last justified slot justified_streak*: uint64 # Number of consecutive justified slots - shard_and_committee_for_slots*: seq[ShardAndCommittee] # Committee members and their assigned shard, per slot - persistent_committees*: Uint24 # Persistent shard committees + shard_and_committee_for_slots*: seq[seq[ShardAndCommittee]] # Committee members and their assigned shard, per slot + persistent_committees*: seq[seq[ValidatorRecord]] # Persistent shard committees persistent_committee_reassignments*: seq[ShardReassignmentRecord] next_shuffling_seed*: Blake2_256_Digest # Randao seed used for next shuffling deposits_penalized_in_period*: uint32 # Total deposits penalized in the given withdrawal period @@ -127,6 +127,13 @@ type exit_slot*: uint64 # Slot when validator exited (or 0) exit_seq*: uint64 # Sequence number when validator exited (or 0) + InitialValidator* = object + pubkey*: BLSPublicKey + proof_of_possession*: seq[byte] + withdrawal_shard*: uint16 + withdrawal_address*: EthAddress + randao_commitment*: Blake2_256_Digest + ValidatorStatusCodes* {.pure.} = enum PendingActivation = 0 Active = 1 @@ -162,26 +169,29 @@ type const SHARD_COUNT* = 1024 # a constant referring to the number of shards DEPOSIT_SIZE* = 2^5 # You need to deposit 32 ETH to be a validator in Casper - SLOT_DURATION* = 16 # seconds - CYCLE_LENGTH* = 64 # slots - MIN_COMMITTEE_SIZE* = 2^7 # validators; 2018-11-05 version of spec also says: - # See a recommended `MIN_COMMITTEE_SIZE` of 111 here - # https://vitalik.ca/files/Ithaca201807_Sharding.pdf). + MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH + GWEI_PER_ETH* = 10^9 # Gwei/ETH + TARGET_COMMITTEE_SIZE* = 2^8 # validators + SLOT_DURATION* = 6 # seconds + CYCLE_LENGTH* = 64 # slots (~ 6 minutes) + MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~25 minutes) + SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^17 # slots (~9 days) + MIN_ATTESTATION_INCLUSION_DELAY* = 2^2 # slots (~25 minutes) SQRT_E_DROP_TIME* = 2^16 # slots (~12 days); amount of time it takes for the # quadratic leak to cut deposits of non-participating # validators by ~39.4% + WITHDRAWALS_PER_CYCLE* = 2^2 # validators (5.2m ETH in ~6 months) + MIN_WITHDRAWAL_PERIOD* = 2^13 # slots (~14 hours) + DELETION_PERIOD* = 2^22 # slots (~290 days) + COLLECTIVE_PENALTY_CALCULATION_PERIOD* = 2^20 # slots (~2.4 months) + SLASHING_WHISTLEBLOWER_REWARD_DENOMINATOR* = 2^9 # ? BASE_REWARD_QUOTIENT* = 2^15 # per-slot interest rate assuming all validators are # participating, assuming total deposits of 1 ETH. It # corresponds to ~3.88% annual interest assuming 10 # million participating ETH. - MIN_BALANCE* = 2^4 # ETH - MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH - GWEI_PER_ETH* = 10^9 # Gwei/ETH - MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~1.1 hours) - RANDAO_SLOTS_PER_LAYER* = 2^12 # slots (~18 hours) - WITHDRAWAL_PERIOD* = 2^19 # slots (~97 days) - SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^16 # slots (~12 days) MAX_VALIDATOR_CHURN_QUOTIENT* = 2^5 # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the # validators can change during each validator set # change. + POW_HASH_VOTING_PERIOD* = 2^10 # ? + POW_CONTRACT_MERKLE_TREE_DEPTH* = 2^5 # INITIAL_FORK_VERSION* = 0 # currently behaves like a constant diff --git a/beacon_chain/private/helpers.nim b/beacon_chain/private/helpers.nim index 95a271da9..2f726fabc 100644 --- a/beacon_chain/private/helpers.nim +++ b/beacon_chain/private/helpers.nim @@ -8,43 +8,48 @@ # Helper functions import ../datatypes, sequtils, nimcrypto, math -func get_active_validator_indices(validators: seq[ValidatorRecord]): seq[Uint24] = +func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] = ## Select the active validators result = @[] for idx, val in validators: if val.status == ACTIVE: result.add idx.Uint24 -func shuffle(values: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit.}= +func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = ## Returns the shuffled ``values`` with seed as entropy. ## TODO: this calls out for tests, but I odn't particularly trust spec ## right now. let values_count = values.len - # Entropy is consumed from the seed in 3-byte (24 bit) chunks - const rand_bytes = 3 - let rand_max = 2^(rand_bytes * 8) - 1 + const + # Entropy is consumed from the seed in 3-byte (24 bit) chunks. + rand_bytes = 3 + # The highest possible result of the RNG. + rand_max = 2^(rand_bytes * 8) - 1 # The range of the RNG places an upper-bound on the size of the list that # may be shuffled. It is a logic error to supply an oversized list. assert values_count < rand_max - deepCopy(result, values) - var source = seed - - var i = 0 - while i < values.len - 1: - # Re-hash the `source` to obtain a new pattern of bytes + result = values + var + source = seed + index = 0 + while index < values_count - 1: + # Re-hash the `source` to obtain a new pattern of bytes. source = blake2_256.digest source.data - # Iterate through the `source` bytes in 3-byte chunks + # Iterate through the `source` bytes in 3-byte chunks. for pos in countup(0, 29, 3): - let remaining = values_count - i + let remaining = values_count - index if remaining == 1: break # Read 3-bytes of `source` as a 24-bit big-endian integer. - let sample_from_source = source.data[pos].Uint24 shl 16 or source.data[pos+1].Uint24 shl 8 or source.data[pos+2].Uint24 + let sample_from_source = + source.data[pos].Uint24 shl 16 or + source.data[pos+1].Uint24 shl 8 or + source.data[pos+2].Uint24 # Sample values greater than or equal to `sample_max` will cause # modulo bias when mapped into the `remaining` range. @@ -52,50 +57,50 @@ func shuffle(values: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit # Perform a swap if the consumed entropy will not cause modulo bias. if sample_from_source < sample_max: - let replacement_position = sample_from_source mod remaining + i - swap result[i], result[replacement_position] - inc i + # Select a replacement index for the current index. + let replacement_position = sample_from_source mod remaining + index + swap result[index], result[replacement_position] + inc index -func split[T](lst: seq[T], N: Positive): seq[seq[T]] = +func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] = # TODO: implement as an iterator result = newSeq[seq[T]](N) for i in 0 ..< N: result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray -func get_new_shuffling*(seed: Blake2_256_Digest, validators: seq[ValidatorRecord], - dynasty: int64, crosslinking_start_shard: int16): seq[seq[ShardAndCommittee]] {.noInit.} = +func get_new_shuffling*(seed: Blake2_256_Digest, + validators: openArray[ValidatorRecord], + crosslinking_start_shard: int + ): seq[seq[ShardAndCommittee]] = ## Split up validators into groups at the start of every epoch, ## determining at what height they can make attestations and what shard they are making crosslinks for ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png - let avs = get_active_validator_indices(validators) - var committees_per_slot, slots_per_committee: uint16 + let + active_validators = get_active_validator_indices(validators) + committees_per_slot = clamp( + len(active_validators) div CYCLE_LENGTH div TARGET_COMMITTEE_SIZE, + 1, SHARD_COUNT div CYCLE_LENGTH) + # Shuffle with seed + shuffled_active_validator_indices = shuffle(active_validators, seed) + # Split the shuffled list into cycle_length pieces + validators_per_slot = split(shuffled_active_validator_indices, CYCLE_LENGTH) - if avs.len >= CYCLE_LENGTH * MIN_COMMITTEE_SIZE: - committees_per_slot = uint16 avs.len div CYCLE_LENGTH div (MIN_COMMITTEE_SIZE * 2) + 1 - slots_per_committee = 1 - else: - committees_per_slot = 1 - slots_per_committee = 1 - while avs.len.uint16 * slots_per_committee < CYCLE_LENGTH * MIN_COMMITTEE_SIZE and - slots_per_committee < CYCLE_LENGTH: - slots_per_committee *= 2 - - result = @[] - for slot, slot_indices in shuffle(avs, seed).split(CYCLE_LENGTH): - let shard_indices = slot_indices.split(committees_per_slot) - let shard_id_start = crosslinking_start_shard.uint16 + - slot.uint16 * committees_per_slot div slots_per_committee + for slot, slot_indices in validators_per_slot: + let + shard_indices = split(slot_indices, committees_per_slot) + shard_id_start = crosslinking_start_shard + slot * committees_per_slot var committees = newSeq[ShardAndCommittee](shard_indices.len) - for j, indices in shard_indices: - committees[j].shard_id = (shard_id_start + j.uint16) mod SHARD_COUNT - committees[j].committee = indices + for shard_position, indices in shard_indices: + committees[shard_position].shard_id = (shard_id_start + shard_position).uint16 mod SHARD_COUNT + committees[shard_position].committee = indices result.add committees func get_shards_and_committees_for_slot*(state: BeaconState, - slot: uint64): seq[ShardAndCommittee] = + slot: uint64 + ): seq[ShardAndCommittee] = # TODO: Spec why is active_state an argument? # TODO: this returns a scalar, not vector, but its return type in spec is a seq/list? @@ -103,7 +108,7 @@ func get_shards_and_committees_for_slot*(state: BeaconState, assert earliest_slot_in_array <= slot assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2 - return @[state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]] + return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array] # TODO, slot is a uint64; will be an issue on int32 arch. # Clarify with EF if light clients will need the beacon chain @@ -114,11 +119,10 @@ func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): return state.recent_block_hashes[slot - earliest_slot_in_array] -func get_new_recent_block_hashes*( - old_block_hashes: seq[Blake2_256_Digest], - parent_slot, current_slot: int64, - parent_hash: Blake2_256_Digest - ): seq[Blake2_256_Digest] = +func get_new_recent_block_hashes*(old_block_hashes: seq[Blake2_256_Digest], + parent_slot, current_slot: int64, + parent_hash: Blake2_256_Digest + ): seq[Blake2_256_Digest] = # Should throw for `current_slot - CYCLE_LENGTH * 2 - 1` according to spec comment let d = current_slot - parent_slot @@ -126,3 +130,8 @@ func get_new_recent_block_hashes*( for _ in 0 ..< min(d, old_block_hashes.len): result.add parent_hash +func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord = + let + first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee + index = first_committee[(slot mod len(first_committee).uint64).int] + state.validators[index] diff --git a/beacon_chain/validator.nim b/beacon_chain/validator.nim new file mode 100644 index 000000000..4110bdf0f --- /dev/null +++ b/beacon_chain/validator.nim @@ -0,0 +1,47 @@ +import + options, + eth_common, + ./datatypes, ./private/helpers + +func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] = + for i, v in validators: + if v.status == WITHDRAWN and v.exit_slot + DELETION_PERIOD.uint64 <= current_slot: + return some(i) + +func add_validator*(validators: var seq[ValidatorRecord], + pubkey: BLSPublicKey, + proof_of_possession: seq[byte], + withdrawal_shard: uint16, + withdrawal_address: EthAddress, + randao_commitment: Blake2_256_Digest, + status: ValidatorStatusCodes, + current_slot: uint64 + ): int = + # Check that validator really did register + # let signed_message = as_bytes32(pubkey) + as_bytes2(withdrawal_shard) + withdrawal_address + randao_commitment + # assert BLSVerify(pub=pubkey, + # msg=hash(signed_message), + # sig=proof_of_possession) + + # Pubkey uniqueness + # assert pubkey not in [v.pubkey for v in validators] + let + rec = ValidatorRecord( + pubkey: pubkey, + withdrawal_shard: withdrawal_shard, + withdrawal_address: withdrawal_address, + randao_commitment: randao_commitment, + randao_last_change: current_slot, + balance: DEPOSIT_SIZE * GWEI_PER_ETH, + status: status, + exit_slot: 0, + exit_seq: 0 + ) + + let index = min_empty_validator(validators, current_slot) + if index.isNone: + validators.add(rec) + return len(validators) - 1 + else: + validators[index.get()] = rec + return index.get() From d1839dad9a77cb1b93f81879806d6b8b88a215f3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 23 Nov 2018 16:44:43 -0600 Subject: [PATCH 2/4] more small fixes * move consts to top * move some functions to validator.nim * add some validator functoin smoke tests --- beacon_chain/datatypes.nim | 65 ++++++++++++++++---------------- beacon_chain/private/helpers.nim | 44 +++------------------ beacon_chain/validator.nim | 47 +++++++++++++++++++++++ tests/all_tests.nim | 3 +- tests/test_validator.nim | 30 +++++++++++++++ 5 files changed, 118 insertions(+), 71 deletions(-) create mode 100644 tests/test_validator.nim diff --git a/beacon_chain/datatypes.nim b/beacon_chain/datatypes.nim index 3872d2e41..19510018d 100644 --- a/beacon_chain/datatypes.nim +++ b/beacon_chain/datatypes.nim @@ -21,6 +21,36 @@ import milagro_crypto # - Signature (48 bytes - 384-bit) # - VerKey (public key) (192 bytes) +const + SHARD_COUNT* = 1024 # a constant referring to the number of shards + DEPOSIT_SIZE* = 2^5 # You need to deposit 32 ETH to be a validator in Casper + MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH + GWEI_PER_ETH* = 10^9 # Gwei/ETH + TARGET_COMMITTEE_SIZE* = 2^8 # validators + SLOT_DURATION* = 6 # seconds + CYCLE_LENGTH* = 64 # slots (~ 6 minutes) + MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~25 minutes) + SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^17 # slots (~9 days) + MIN_ATTESTATION_INCLUSION_DELAY* = 2^2 # slots (~25 minutes) + SQRT_E_DROP_TIME* = 2^16 # slots (~12 days); amount of time it takes for the + # quadratic leak to cut deposits of non-participating + # validators by ~39.4% + WITHDRAWALS_PER_CYCLE* = 2^2 # validators (5.2m ETH in ~6 months) + MIN_WITHDRAWAL_PERIOD* = 2^13 # slots (~14 hours) + DELETION_PERIOD* = 2^22 # slots (~290 days) + COLLECTIVE_PENALTY_CALCULATION_PERIOD* = 2^20 # slots (~2.4 months) + SLASHING_WHISTLEBLOWER_REWARD_DENOMINATOR* = 2^9 # ? + BASE_REWARD_QUOTIENT* = 2^15 # per-slot interest rate assuming all validators are + # participating, assuming total deposits of 1 ETH. It + # corresponds to ~3.88% annual interest assuming 10 + # million participating ETH. + MAX_VALIDATOR_CHURN_QUOTIENT* = 2^5 # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the + # validators can change during each validator set + # change. + POW_HASH_VOTING_PERIOD* = 2^10 # ? + POW_CONTRACT_MERKLE_TREE_DEPTH* = 2^5 # + INITIAL_FORK_VERSION* = 0 # currently behaves like a constant + type # Alias BLSPublicKey* = VerKey @@ -98,7 +128,9 @@ type last_finalized_slot*: uint64 # Last finalized slot last_justified_slot*: uint64 # Last justified slot justified_streak*: uint64 # Number of consecutive justified slots - shard_and_committee_for_slots*: seq[seq[ShardAndCommittee]] # Committee members and their assigned shard, per slot + shard_and_committee_for_slots*: array[2 * CYCLE_LENGTH, seq[ShardAndCommittee]] ## \ + ## Committee members and their assigned shard, per slot, covers 2 cycles + ## worth of assignments persistent_committees*: seq[seq[ValidatorRecord]] # Persistent shard committees persistent_committee_reassignments*: seq[ShardReassignmentRecord] next_shuffling_seed*: Blake2_256_Digest # Randao seed used for next shuffling @@ -164,34 +196,3 @@ type # with room to spare. # # Also, IntSets uses machine int size while we require int64 even on 32-bit platform. - - -const - SHARD_COUNT* = 1024 # a constant referring to the number of shards - DEPOSIT_SIZE* = 2^5 # You need to deposit 32 ETH to be a validator in Casper - MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH - GWEI_PER_ETH* = 10^9 # Gwei/ETH - TARGET_COMMITTEE_SIZE* = 2^8 # validators - SLOT_DURATION* = 6 # seconds - CYCLE_LENGTH* = 64 # slots (~ 6 minutes) - MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~25 minutes) - SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^17 # slots (~9 days) - MIN_ATTESTATION_INCLUSION_DELAY* = 2^2 # slots (~25 minutes) - SQRT_E_DROP_TIME* = 2^16 # slots (~12 days); amount of time it takes for the - # quadratic leak to cut deposits of non-participating - # validators by ~39.4% - WITHDRAWALS_PER_CYCLE* = 2^2 # validators (5.2m ETH in ~6 months) - MIN_WITHDRAWAL_PERIOD* = 2^13 # slots (~14 hours) - DELETION_PERIOD* = 2^22 # slots (~290 days) - COLLECTIVE_PENALTY_CALCULATION_PERIOD* = 2^20 # slots (~2.4 months) - SLASHING_WHISTLEBLOWER_REWARD_DENOMINATOR* = 2^9 # ? - BASE_REWARD_QUOTIENT* = 2^15 # per-slot interest rate assuming all validators are - # participating, assuming total deposits of 1 ETH. It - # corresponds to ~3.88% annual interest assuming 10 - # million participating ETH. - MAX_VALIDATOR_CHURN_QUOTIENT* = 2^5 # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the - # validators can change during each validator set - # change. - POW_HASH_VOTING_PERIOD* = 2^10 # ? - POW_CONTRACT_MERKLE_TREE_DEPTH* = 2^5 # - INITIAL_FORK_VERSION* = 0 # currently behaves like a constant diff --git a/beacon_chain/private/helpers.nim b/beacon_chain/private/helpers.nim index 2f726fabc..07e9b0554 100644 --- a/beacon_chain/private/helpers.nim +++ b/beacon_chain/private/helpers.nim @@ -8,13 +8,6 @@ # Helper functions import ../datatypes, sequtils, nimcrypto, math -func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] = - ## Select the active validators - result = @[] - for idx, val in validators: - if val.status == ACTIVE: - result.add idx.Uint24 - func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = ## Returns the shuffled ``values`` with seed as entropy. ## TODO: this calls out for tests, but I odn't particularly trust spec @@ -38,7 +31,10 @@ func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = index = 0 while index < values_count - 1: # Re-hash the `source` to obtain a new pattern of bytes. - source = blake2_256.digest source.data + + # XXX "attempting to call undeclared routine init" + # source = blake2_256.digest source.data + # Iterate through the `source` bytes in 3-byte chunks. for pos in countup(0, 29, 3): let remaining = values_count - index @@ -63,41 +59,13 @@ func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = inc index func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] = + ## split lst in N pieces, with each piece having `len(lst) div N` or + ## `len(lst) div N + 1` pieces # TODO: implement as an iterator result = newSeq[seq[T]](N) for i in 0 ..< N: result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray -func get_new_shuffling*(seed: Blake2_256_Digest, - validators: openArray[ValidatorRecord], - crosslinking_start_shard: int - ): seq[seq[ShardAndCommittee]] = - ## Split up validators into groups at the start of every epoch, - ## determining at what height they can make attestations and what shard they are making crosslinks for - ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png - - let - active_validators = get_active_validator_indices(validators) - committees_per_slot = clamp( - len(active_validators) div CYCLE_LENGTH div TARGET_COMMITTEE_SIZE, - 1, SHARD_COUNT div CYCLE_LENGTH) - # Shuffle with seed - shuffled_active_validator_indices = shuffle(active_validators, seed) - # Split the shuffled list into cycle_length pieces - validators_per_slot = split(shuffled_active_validator_indices, CYCLE_LENGTH) - - for slot, slot_indices in validators_per_slot: - let - shard_indices = split(slot_indices, committees_per_slot) - shard_id_start = crosslinking_start_shard + slot * committees_per_slot - - var committees = newSeq[ShardAndCommittee](shard_indices.len) - for shard_position, indices in shard_indices: - committees[shard_position].shard_id = (shard_id_start + shard_position).uint16 mod SHARD_COUNT - committees[shard_position].committee = indices - - result.add committees - func get_shards_and_committees_for_slot*(state: BeaconState, slot: uint64 ): seq[ShardAndCommittee] = diff --git a/beacon_chain/validator.nim b/beacon_chain/validator.nim index 4110bdf0f..c88d124b8 100644 --- a/beacon_chain/validator.nim +++ b/beacon_chain/validator.nim @@ -1,3 +1,11 @@ +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# Helpers and functions pertaining to managing the validator set + import options, eth_common, @@ -45,3 +53,42 @@ func add_validator*(validators: var seq[ValidatorRecord], else: validators[index.get()] = rec return index.get() + +func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] = + ## Select the active validators + result = @[] + for idx, val in validators: + if val.status == ACTIVE: + result.add idx.Uint24 + +func get_new_shuffling*(seed: Blake2_256_Digest, + validators: openArray[ValidatorRecord], + crosslinking_start_shard: int + ): array[CYCLE_LENGTH, seq[ShardAndCommittee]] = + ## Split up validators into groups at the start of every epoch, + ## determining at what height they can make attestations and what shard they are making crosslinks for + ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png + + let + active_validators = get_active_validator_indices(validators) + committees_per_slot = clamp( + len(active_validators) div CYCLE_LENGTH div TARGET_COMMITTEE_SIZE, + 1, SHARD_COUNT div CYCLE_LENGTH) + # Shuffle with seed + shuffled_active_validator_indices = shuffle(active_validators, seed) + # Split the shuffled list into cycle_length pieces + validators_per_slot = split(shuffled_active_validator_indices, CYCLE_LENGTH) + + assert validators_per_slot.len() == CYCLE_LENGTH # what split should do.. + + for slot, slot_indices in validators_per_slot: + let + shard_indices = split(slot_indices, committees_per_slot) + shard_id_start = crosslinking_start_shard + slot * committees_per_slot + + var committees = newSeq[ShardAndCommittee](shard_indices.len) + for shard_position, indices in shard_indices: + committees[shard_position].shard_id = (shard_id_start + shard_position).uint16 mod SHARD_COUNT + committees[shard_position].committee = indices + + result[slot] = committees diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 89bb553b3..497b1e3e4 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -6,5 +6,6 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import + ./test_block_processing, ./test_ssz, - ./test_block_processing + ./test_validator diff --git a/tests/test_validator.nim b/tests/test_validator.nim new file mode 100644 index 000000000..03dc37ebd --- /dev/null +++ b/tests/test_validator.nim @@ -0,0 +1,30 @@ +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + math, nimcrypto, unittest, sequtils, + ../beacon_chain/[datatypes, validator] + +func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int = + for x in v: + for y in x: + inc result, y.committee.len + +suite "Validators": + ## For now just test that we can compile and execute block processing with mock data. + + test "Smoke validator shuffling": + let + validators = repeat( + ValidatorRecord( + status: ACTIVE + ), 1024) + + # XXX the shuffling looks really odd, probably buggy + let s = get_new_shuffling(Blake2_256_Digest(), validators, 0) + check: + s.len == CYCLE_LENGTH + sumCommittees(s) == validators.len() # all validators accounted for From da22f2065f0d3c98f3f614d242a225c87eb6bfcd Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 26 Nov 2018 09:44:49 -0600 Subject: [PATCH 3/4] docs & comments * link to resources for shuffling * describe get_beacon_proposer * hints for solving init issue --- beacon_chain/private/helpers.nim | 18 +++++++++++++----- tests/test_validator.nim | 3 ++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/beacon_chain/private/helpers.nim b/beacon_chain/private/helpers.nim index 07e9b0554..246ab1c88 100644 --- a/beacon_chain/private/helpers.nim +++ b/beacon_chain/private/helpers.nim @@ -32,8 +32,11 @@ func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = while index < values_count - 1: # Re-hash the `source` to obtain a new pattern of bytes. - # XXX "attempting to call undeclared routine init" - # source = blake2_256.digest source.data + # XXX: attempting to call undeclared routine: 'init' + # https://github.com/nim-lang/Nim/issues/8677 + # + # mixin init? doesn't seem to be enough + #source = blake2_256.digest source.data # Iterate through the `source` bytes in 3-byte chunks. for pos in countup(0, 29, 3): @@ -69,9 +72,6 @@ func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] = func get_shards_and_committees_for_slot*(state: BeaconState, slot: uint64 ): seq[ShardAndCommittee] = - # TODO: Spec why is active_state an argument? - # TODO: this returns a scalar, not vector, but its return type in spec is a seq/list? - let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH assert earliest_slot_in_array <= slot assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2 @@ -99,6 +99,14 @@ func get_new_recent_block_hashes*(old_block_hashes: seq[Blake2_256_Digest], result.add parent_hash func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord = + ## From Casper RPJ mini-spec: + ## When slot i begins, validator Vidx is expected + ## to create ("propose") a block, which contains a pointer to some parent block + ## that they perceive as the "head of the chain", + ## and includes all of the **attestations** that they know about + ## that have not yet been included into that chain. + ## + ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) let first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee index = first_committee[(slot mod len(first_committee).uint64).int] diff --git a/tests/test_validator.nim b/tests/test_validator.nim index 03dc37ebd..8d6eae6d3 100644 --- a/tests/test_validator.nim +++ b/tests/test_validator.nim @@ -15,7 +15,8 @@ func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int = suite "Validators": ## For now just test that we can compile and execute block processing with mock data. - + ## https://github.com/status-im/nim-beacon-chain/issues/1 + ## https://github.com/sigp/lighthouse/blob/ba548e49a52687a655c61b443b6835d79c6d4236/beacon_chain/validator_shuffling/src/shuffle.rs test "Smoke validator shuffling": let validators = repeat( From 2137b084b2e309bad3090f37476d0d4d49546557 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 26 Nov 2018 18:41:03 +0200 Subject: [PATCH 4/4] work-around a generics compilation issue --- beacon_chain/private/helpers.nim | 7 +------ beacon_chain/validator.nim | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/beacon_chain/private/helpers.nim b/beacon_chain/private/helpers.nim index 246ab1c88..c4b70b3ec 100644 --- a/beacon_chain/private/helpers.nim +++ b/beacon_chain/private/helpers.nim @@ -31,12 +31,7 @@ func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = index = 0 while index < values_count - 1: # Re-hash the `source` to obtain a new pattern of bytes. - - # XXX: attempting to call undeclared routine: 'init' - # https://github.com/nim-lang/Nim/issues/8677 - # - # mixin init? doesn't seem to be enough - #source = blake2_256.digest source.data + source = blake2_256.digest source.data # Iterate through the `source` bytes in 3-byte chunks. for pos in countup(0, 29, 3): diff --git a/beacon_chain/validator.nim b/beacon_chain/validator.nim index c88d124b8..1b5b13abc 100644 --- a/beacon_chain/validator.nim +++ b/beacon_chain/validator.nim @@ -8,7 +8,7 @@ import options, - eth_common, + eth_common, nimcrypto/blake2, ./datatypes, ./private/helpers func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =