diff --git a/beacon_chain/datatypes.nim b/beacon_chain/datatypes.nim index 5f0350695..19510018d 100644 --- a/beacon_chain/datatypes.nim +++ b/beacon_chain/datatypes.nim @@ -21,6 +21,36 @@ import milagro_crypto # - Signature (48 bytes - 384-bit) # - VerKey (public key) (192 bytes) +const + SHARD_COUNT* = 1024 # a constant referring to the number of shards + DEPOSIT_SIZE* = 2^5 # You need to deposit 32 ETH to be a validator in Casper + MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH + GWEI_PER_ETH* = 10^9 # Gwei/ETH + TARGET_COMMITTEE_SIZE* = 2^8 # validators + SLOT_DURATION* = 6 # seconds + CYCLE_LENGTH* = 64 # slots (~ 6 minutes) + MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~25 minutes) + SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^17 # slots (~9 days) + MIN_ATTESTATION_INCLUSION_DELAY* = 2^2 # slots (~25 minutes) + SQRT_E_DROP_TIME* = 2^16 # slots (~12 days); amount of time it takes for the + # quadratic leak to cut deposits of non-participating + # validators by ~39.4% + WITHDRAWALS_PER_CYCLE* = 2^2 # validators (5.2m ETH in ~6 months) + MIN_WITHDRAWAL_PERIOD* = 2^13 # slots (~14 hours) + DELETION_PERIOD* = 2^22 # slots (~290 days) + COLLECTIVE_PENALTY_CALCULATION_PERIOD* = 2^20 # slots (~2.4 months) + SLASHING_WHISTLEBLOWER_REWARD_DENOMINATOR* = 2^9 # ? + BASE_REWARD_QUOTIENT* = 2^15 # per-slot interest rate assuming all validators are + # participating, assuming total deposits of 1 ETH. It + # corresponds to ~3.88% annual interest assuming 10 + # million participating ETH. + MAX_VALIDATOR_CHURN_QUOTIENT* = 2^5 # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the + # validators can change during each validator set + # change. + POW_HASH_VOTING_PERIOD* = 2^10 # ? + POW_CONTRACT_MERKLE_TREE_DEPTH* = 2^5 # + INITIAL_FORK_VERSION* = 0 # currently behaves like a constant + type # Alias BLSPublicKey* = VerKey @@ -98,8 +128,10 @@ type last_finalized_slot*: uint64 # Last finalized slot last_justified_slot*: uint64 # Last justified slot justified_streak*: uint64 # Number of consecutive justified slots - shard_and_committee_for_slots*: seq[ShardAndCommittee] # Committee members and their assigned shard, per slot - persistent_committees*: Uint24 # Persistent shard committees + shard_and_committee_for_slots*: array[2 * CYCLE_LENGTH, seq[ShardAndCommittee]] ## \ + ## Committee members and their assigned shard, per slot, covers 2 cycles + ## worth of assignments + persistent_committees*: seq[seq[ValidatorRecord]] # Persistent shard committees persistent_committee_reassignments*: seq[ShardReassignmentRecord] next_shuffling_seed*: Blake2_256_Digest # Randao seed used for next shuffling deposits_penalized_in_period*: uint32 # Total deposits penalized in the given withdrawal period @@ -127,6 +159,13 @@ type exit_slot*: uint64 # Slot when validator exited (or 0) exit_seq*: uint64 # Sequence number when validator exited (or 0) + InitialValidator* = object + pubkey*: BLSPublicKey + proof_of_possession*: seq[byte] + withdrawal_shard*: uint16 + withdrawal_address*: EthAddress + randao_commitment*: Blake2_256_Digest + ValidatorStatusCodes* {.pure.} = enum PendingActivation = 0 Active = 1 @@ -157,31 +196,3 @@ type # with room to spare. # # Also, IntSets uses machine int size while we require int64 even on 32-bit platform. - - -const - SHARD_COUNT* = 1024 # a constant referring to the number of shards - DEPOSIT_SIZE* = 2^5 # You need to deposit 32 ETH to be a validator in Casper - SLOT_DURATION* = 16 # seconds - CYCLE_LENGTH* = 64 # slots - MIN_COMMITTEE_SIZE* = 2^7 # validators; 2018-11-05 version of spec also says: - # See a recommended `MIN_COMMITTEE_SIZE` of 111 here - # https://vitalik.ca/files/Ithaca201807_Sharding.pdf). - SQRT_E_DROP_TIME* = 2^16 # slots (~12 days); amount of time it takes for the - # quadratic leak to cut deposits of non-participating - # validators by ~39.4% - BASE_REWARD_QUOTIENT* = 2^15 # per-slot interest rate assuming all validators are - # participating, assuming total deposits of 1 ETH. It - # corresponds to ~3.88% annual interest assuming 10 - # million participating ETH. - MIN_BALANCE* = 2^4 # ETH - MIN_ONLINE_DEPOSIT_SIZE* = 2^4 # ETH - GWEI_PER_ETH* = 10^9 # Gwei/ETH - MIN_VALIDATOR_SET_CHANGE_INTERVAL* = 2^8 # slots (~1.1 hours) - RANDAO_SLOTS_PER_LAYER* = 2^12 # slots (~18 hours) - WITHDRAWAL_PERIOD* = 2^19 # slots (~97 days) - SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^16 # slots (~12 days) - MAX_VALIDATOR_CHURN_QUOTIENT* = 2^5 # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the - # validators can change during each validator set - # change. - INITIAL_FORK_VERSION* = 0 # currently behaves like a constant diff --git a/beacon_chain/private/helpers.nim b/beacon_chain/private/helpers.nim index 95a271da9..c4b70b3ec 100644 --- a/beacon_chain/private/helpers.nim +++ b/beacon_chain/private/helpers.nim @@ -8,43 +8,42 @@ # Helper functions import ../datatypes, sequtils, nimcrypto, math -func get_active_validator_indices(validators: seq[ValidatorRecord]): seq[Uint24] = - ## Select the active validators - result = @[] - for idx, val in validators: - if val.status == ACTIVE: - result.add idx.Uint24 - -func shuffle(values: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit.}= +func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] = ## Returns the shuffled ``values`` with seed as entropy. ## TODO: this calls out for tests, but I odn't particularly trust spec ## right now. let values_count = values.len - # Entropy is consumed from the seed in 3-byte (24 bit) chunks - const rand_bytes = 3 - let rand_max = 2^(rand_bytes * 8) - 1 + const + # Entropy is consumed from the seed in 3-byte (24 bit) chunks. + rand_bytes = 3 + # The highest possible result of the RNG. + rand_max = 2^(rand_bytes * 8) - 1 # The range of the RNG places an upper-bound on the size of the list that # may be shuffled. It is a logic error to supply an oversized list. assert values_count < rand_max - deepCopy(result, values) - var source = seed - - var i = 0 - while i < values.len - 1: - # Re-hash the `source` to obtain a new pattern of bytes + result = values + var + source = seed + index = 0 + while index < values_count - 1: + # Re-hash the `source` to obtain a new pattern of bytes. source = blake2_256.digest source.data - # Iterate through the `source` bytes in 3-byte chunks + + # Iterate through the `source` bytes in 3-byte chunks. for pos in countup(0, 29, 3): - let remaining = values_count - i + let remaining = values_count - index if remaining == 1: break # Read 3-bytes of `source` as a 24-bit big-endian integer. - let sample_from_source = source.data[pos].Uint24 shl 16 or source.data[pos+1].Uint24 shl 8 or source.data[pos+2].Uint24 + let sample_from_source = + source.data[pos].Uint24 shl 16 or + source.data[pos+1].Uint24 shl 8 or + source.data[pos+2].Uint24 # Sample values greater than or equal to `sample_max` will cause # modulo bias when mapped into the `remaining` range. @@ -52,58 +51,27 @@ func shuffle(values: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit # Perform a swap if the consumed entropy will not cause modulo bias. if sample_from_source < sample_max: - let replacement_position = sample_from_source mod remaining + i - swap result[i], result[replacement_position] - inc i + # Select a replacement index for the current index. + let replacement_position = sample_from_source mod remaining + index + swap result[index], result[replacement_position] + inc index -func split[T](lst: seq[T], N: Positive): seq[seq[T]] = +func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] = + ## split lst in N pieces, with each piece having `len(lst) div N` or + ## `len(lst) div N + 1` pieces # TODO: implement as an iterator result = newSeq[seq[T]](N) for i in 0 ..< N: result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray -func get_new_shuffling*(seed: Blake2_256_Digest, validators: seq[ValidatorRecord], - dynasty: int64, crosslinking_start_shard: int16): seq[seq[ShardAndCommittee]] {.noInit.} = - ## Split up validators into groups at the start of every epoch, - ## determining at what height they can make attestations and what shard they are making crosslinks for - ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png - - let avs = get_active_validator_indices(validators) - var committees_per_slot, slots_per_committee: uint16 - - if avs.len >= CYCLE_LENGTH * MIN_COMMITTEE_SIZE: - committees_per_slot = uint16 avs.len div CYCLE_LENGTH div (MIN_COMMITTEE_SIZE * 2) + 1 - slots_per_committee = 1 - else: - committees_per_slot = 1 - slots_per_committee = 1 - while avs.len.uint16 * slots_per_committee < CYCLE_LENGTH * MIN_COMMITTEE_SIZE and - slots_per_committee < CYCLE_LENGTH: - slots_per_committee *= 2 - - result = @[] - for slot, slot_indices in shuffle(avs, seed).split(CYCLE_LENGTH): - let shard_indices = slot_indices.split(committees_per_slot) - let shard_id_start = crosslinking_start_shard.uint16 + - slot.uint16 * committees_per_slot div slots_per_committee - - var committees = newSeq[ShardAndCommittee](shard_indices.len) - for j, indices in shard_indices: - committees[j].shard_id = (shard_id_start + j.uint16) mod SHARD_COUNT - committees[j].committee = indices - - result.add committees - func get_shards_and_committees_for_slot*(state: BeaconState, - slot: uint64): seq[ShardAndCommittee] = - # TODO: Spec why is active_state an argument? - # TODO: this returns a scalar, not vector, but its return type in spec is a seq/list? - + slot: uint64 + ): seq[ShardAndCommittee] = let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH assert earliest_slot_in_array <= slot assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2 - return @[state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]] + return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array] # TODO, slot is a uint64; will be an issue on int32 arch. # Clarify with EF if light clients will need the beacon chain @@ -114,11 +82,10 @@ func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): return state.recent_block_hashes[slot - earliest_slot_in_array] -func get_new_recent_block_hashes*( - old_block_hashes: seq[Blake2_256_Digest], - parent_slot, current_slot: int64, - parent_hash: Blake2_256_Digest - ): seq[Blake2_256_Digest] = +func get_new_recent_block_hashes*(old_block_hashes: seq[Blake2_256_Digest], + parent_slot, current_slot: int64, + parent_hash: Blake2_256_Digest + ): seq[Blake2_256_Digest] = # Should throw for `current_slot - CYCLE_LENGTH * 2 - 1` according to spec comment let d = current_slot - parent_slot @@ -126,3 +93,16 @@ func get_new_recent_block_hashes*( for _ in 0 ..< min(d, old_block_hashes.len): result.add parent_hash +func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord = + ## From Casper RPJ mini-spec: + ## When slot i begins, validator Vidx is expected + ## to create ("propose") a block, which contains a pointer to some parent block + ## that they perceive as the "head of the chain", + ## and includes all of the **attestations** that they know about + ## that have not yet been included into that chain. + ## + ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) + let + first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee + index = first_committee[(slot mod len(first_committee).uint64).int] + state.validators[index] diff --git a/beacon_chain/validator.nim b/beacon_chain/validator.nim new file mode 100644 index 000000000..1b5b13abc --- /dev/null +++ b/beacon_chain/validator.nim @@ -0,0 +1,94 @@ +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +# Helpers and functions pertaining to managing the validator set + +import + options, + eth_common, nimcrypto/blake2, + ./datatypes, ./private/helpers + +func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] = + for i, v in validators: + if v.status == WITHDRAWN and v.exit_slot + DELETION_PERIOD.uint64 <= current_slot: + return some(i) + +func add_validator*(validators: var seq[ValidatorRecord], + pubkey: BLSPublicKey, + proof_of_possession: seq[byte], + withdrawal_shard: uint16, + withdrawal_address: EthAddress, + randao_commitment: Blake2_256_Digest, + status: ValidatorStatusCodes, + current_slot: uint64 + ): int = + # Check that validator really did register + # let signed_message = as_bytes32(pubkey) + as_bytes2(withdrawal_shard) + withdrawal_address + randao_commitment + # assert BLSVerify(pub=pubkey, + # msg=hash(signed_message), + # sig=proof_of_possession) + + # Pubkey uniqueness + # assert pubkey not in [v.pubkey for v in validators] + let + rec = ValidatorRecord( + pubkey: pubkey, + withdrawal_shard: withdrawal_shard, + withdrawal_address: withdrawal_address, + randao_commitment: randao_commitment, + randao_last_change: current_slot, + balance: DEPOSIT_SIZE * GWEI_PER_ETH, + status: status, + exit_slot: 0, + exit_seq: 0 + ) + + let index = min_empty_validator(validators, current_slot) + if index.isNone: + validators.add(rec) + return len(validators) - 1 + else: + validators[index.get()] = rec + return index.get() + +func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] = + ## Select the active validators + result = @[] + for idx, val in validators: + if val.status == ACTIVE: + result.add idx.Uint24 + +func get_new_shuffling*(seed: Blake2_256_Digest, + validators: openArray[ValidatorRecord], + crosslinking_start_shard: int + ): array[CYCLE_LENGTH, seq[ShardAndCommittee]] = + ## Split up validators into groups at the start of every epoch, + ## determining at what height they can make attestations and what shard they are making crosslinks for + ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png + + let + active_validators = get_active_validator_indices(validators) + committees_per_slot = clamp( + len(active_validators) div CYCLE_LENGTH div TARGET_COMMITTEE_SIZE, + 1, SHARD_COUNT div CYCLE_LENGTH) + # Shuffle with seed + shuffled_active_validator_indices = shuffle(active_validators, seed) + # Split the shuffled list into cycle_length pieces + validators_per_slot = split(shuffled_active_validator_indices, CYCLE_LENGTH) + + assert validators_per_slot.len() == CYCLE_LENGTH # what split should do.. + + for slot, slot_indices in validators_per_slot: + let + shard_indices = split(slot_indices, committees_per_slot) + shard_id_start = crosslinking_start_shard + slot * committees_per_slot + + var committees = newSeq[ShardAndCommittee](shard_indices.len) + for shard_position, indices in shard_indices: + committees[shard_position].shard_id = (shard_id_start + shard_position).uint16 mod SHARD_COUNT + committees[shard_position].committee = indices + + result[slot] = committees diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 89bb553b3..497b1e3e4 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -6,5 +6,6 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import + ./test_block_processing, ./test_ssz, - ./test_block_processing + ./test_validator diff --git a/tests/test_validator.nim b/tests/test_validator.nim new file mode 100644 index 000000000..8d6eae6d3 --- /dev/null +++ b/tests/test_validator.nim @@ -0,0 +1,31 @@ +# Copyright (c) 2018 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + math, nimcrypto, unittest, sequtils, + ../beacon_chain/[datatypes, validator] + +func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int = + for x in v: + for y in x: + inc result, y.committee.len + +suite "Validators": + ## For now just test that we can compile and execute block processing with mock data. + ## https://github.com/status-im/nim-beacon-chain/issues/1 + ## https://github.com/sigp/lighthouse/blob/ba548e49a52687a655c61b443b6835d79c6d4236/beacon_chain/validator_shuffling/src/shuffle.rs + test "Smoke validator shuffling": + let + validators = repeat( + ValidatorRecord( + status: ACTIVE + ), 1024) + + # XXX the shuffling looks really odd, probably buggy + let s = get_new_shuffling(Blake2_256_Digest(), validators, 0) + check: + s.len == CYCLE_LENGTH + sumCommittees(s) == validators.len() # all validators accounted for