From ac600d398b93006ddc8e17e794d3a57217d64b6c Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 3 Dec 2018 15:41:24 -0600 Subject: [PATCH] spec updates * lots of renames * add some epoch processing --- beacon_chain/beacon_node.nim | 12 +- beacon_chain/fork_choice.nim | 2 +- beacon_chain/spec/beaconstate.nim | 109 ++++++------- beacon_chain/spec/datatypes.nim | 84 +++++----- beacon_chain/spec/helpers.nim | 50 ++++++ beacon_chain/spec/validator.nim | 76 ++++++++- beacon_chain/ssz.nim | 4 +- beacon_chain/state_transition.nim | 248 +++++++++++++++++++++++++++--- beacon_chain/sync_protocol.nim | 6 +- beacon_chain/time.nim | 2 +- beacon_chain/validator_pool.nim | 2 +- tests/test_beaconstate.nim | 2 +- 12 files changed, 461 insertions(+), 136 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index abd97dde9..8addf4dff 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -1,7 +1,7 @@ import os, net, asyncdispatch2, chronicles, confutils, eth_p2p, eth_keys, - spec/[beaconstate, datatypes], conf, time, fork_choice, + spec/[beaconstate, datatypes, helpers], conf, time, fork_choice, beacon_chain_db, validator_pool, mainchain_monitor, sync_protocol, gossipsub_protocol, trusted_state_snapshots @@ -52,9 +52,9 @@ proc sync*(node: BeaconNode): Future[bool] {.async.} = node.beaconState = persistedState[] var targetSlot = toSlot timeSinceGenesis(node.beaconState) - while node.beaconState.last_finalized_slot.int < targetSlot: + while node.beaconState.finalized_slot.int < targetSlot: var (peer, changeLog) = await node.network.getValidatorChangeLog( - node.beaconState.validator_set_delta_hash_chain) + node.beaconState.validator_registry_delta_chain_tip) if peer == nil: error "Failed to sync with any peer" @@ -79,7 +79,7 @@ proc addLocalValidators*(node: BeaconNode) = discard proc getAttachedValidator(node: BeaconNode, idx: int): AttachedValidator = - let validatorKey = node.beaconState.validators[idx].pubkey + let validatorKey = node.beaconState.validator_registry[idx].pubkey return node.attachedValidators.getValidator(validatorKey) proc makeAttestation(node: BeaconNode, @@ -129,7 +129,7 @@ proc proposeBlock(node: BeaconNode, proc scheduleCycleActions(node: BeaconNode) = ## This schedules the required block proposals and ## attestations from our attached validators. - let cycleStart = node.beaconState.last_state_recalculation_slot.int + let cycleStart = node.beaconState.latest_state_recalculation_slot.int for i in 0 ..< EPOCH_LENGTH: # Schedule block proposals @@ -148,7 +148,7 @@ proc scheduleCycleActions(node: BeaconNode) = # Schedule attestations let - committeesIdx = get_shards_and_committees_index(node.beaconState, slot.uint64) + committeesIdx = get_shard_and_committees_index(node.beaconState, slot.uint64) for shard in node.beaconState.shard_and_committee_for_slots[committees_idx]: for validatorIdx in shard.committee: diff --git a/beacon_chain/fork_choice.nim b/beacon_chain/fork_choice.nim index 02b6ce65c..ff593b625 100644 --- a/beacon_chain/fork_choice.nim +++ b/beacon_chain/fork_choice.nim @@ -5,7 +5,7 @@ import type Attestation* = object validator*: int - data*: AttestationSignedData + data*: AttestationData signature*: ValidatorSig AttestationPool* = object diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index c440cf875..67fc48936 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -10,11 +10,8 @@ import ../extras, ./datatypes, ./digest, ./helpers, ./validator -func mod_get[T](arr: openarray[T], pos: Natural): T = - arr[pos mod arr.len] - func on_startup*(initial_validator_entries: openArray[InitialValidator], - genesis_time: int, + genesis_time: uint64, processed_pow_receipt_root: Eth2Digest): BeaconState = ## BeaconState constructor ## @@ -40,7 +37,7 @@ func on_startup*(initial_validator_entries: openArray[InitialValidator], ForkData( pre_fork_version: INITIAL_FORK_VERSION, post_fork_version: INITIAL_FORK_VERSION, - fork_slot_number: 0xffffffffffffffff'u64 + fork_slot: INITIAL_SLOT_NUMBER ), v.pubkey, v.deposit_size, @@ -48,71 +45,65 @@ func on_startup*(initial_validator_entries: openArray[InitialValidator], v.withdrawal_credentials, v.randao_commitment, ACTIVE, - 0 + INITIAL_SLOT_NUMBER ).validators # Setup state let - x = get_new_shuffling(Eth2Digest(), validators, 0) + initial_shuffling = get_new_shuffling(Eth2Digest(), validators, 0) - # x + x in spec, but more ugly - var tmp: array[2 * EPOCH_LENGTH, seq[ShardAndCommittee]] - for i, n in x: - tmp[i] = n - tmp[EPOCH_LENGTH + i] = n + # initial_shuffling + initial_shuffling in spec, but more ugly + var shard_and_committee_for_slots: array[2 * EPOCH_LENGTH, seq[ShardAndCommittee]] + for i, n in initial_shuffling: + shard_and_committee_for_slots[i] = n + shard_and_committee_for_slots[EPOCH_LENGTH + i] = n # The spec says to use validators, but it's actually indices.. let validator_indices = get_active_validator_indices(validators) + let persistent_committees = split(shuffle( + validator_indices, ZERO_HASH), SHARD_COUNT) + BeaconState( - validators: validators, - shard_and_committee_for_slots: tmp, - persistent_committees: split( - shuffle(validator_indices, Eth2Digest()), SHARD_COUNT), - fork_data: ForkData( + validator_registry: validators, + validator_registry_latest_change_slot: INITIAL_SLOT_NUMBER, + validator_registry_exit_count: 0, + validator_registry_delta_chain_tip: ZERO_HASH, + + # Randomness and committees + randao_mix: ZERO_HASH, + next_seed: ZERO_HASH, + shard_and_committee_for_slots: shard_and_committee_for_slots, + persistent_committees: persistent_committees, + + # Finality + previous_justified_slot: INITIAL_SLOT_NUMBER, + justified_slot: INITIAL_SLOT_NUMBER, + finalized_slot: INITIAL_SLOT_NUMBER, + + # Recent state + latest_state_recalculation_slot: INITIAL_SLOT_NUMBER, + latest_block_hashes: repeat(ZERO_HASH, EPOCH_LENGTH * 2), + + # PoW receipt root + processed_pow_receipt_root: processed_pow_receipt_root, + # Misc + genesis_time: genesis_time, + fork_data: ForkData( pre_fork_version: INITIAL_FORK_VERSION, - post_fork_version: INITIAL_FORK_VERSION - ) + post_fork_version: INITIAL_FORK_VERSION, + fork_slot: INITIAL_SLOT_NUMBER, + ), ) -func get_shards_and_committees_index*(state: BeaconState, slot: uint64): uint64 = - # TODO spec unsigned-unsafe here - let earliest_slot_in_array = - if state.last_state_recalculation_slot > EPOCH_LENGTH.uint64: - state.last_state_recalculation_slot - EPOCH_LENGTH - else: - 0 - - doAssert earliest_slot_in_array <= slot and - slot < earliest_slot_in_array + EPOCH_LENGTH * 2 - slot - earliest_slot_in_array - -proc get_shards_and_committees_for_slot*( - state: BeaconState, slot: uint64): seq[ShardAndCommittee] = - let index = state.get_shards_and_committees_index(slot) - state.shard_and_committee_for_slots[index] - -func get_beacon_proposer_index*(state: BeaconState, slot: uint64): uint64 = - ## From Casper RPJ mini-spec: - ## When slot i begins, validator Vidx is expected - ## to create ("propose") a block, which contains a pointer to some parent block - ## that they perceive as the "head of the chain", - ## and includes all of the **attestations** that they know about - ## that have not yet been included into that chain. - ## - ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) - - let idx = get_shards_and_committees_index(state, slot) - state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot) - func get_block_hash*(state: BeaconState, current_block: BeaconBlock, - slot: int): Eth2Digest = + slot: uint64): Eth2Digest = let earliest_slot_in_array = - current_block.slot.int - state.recent_block_hashes.len - assert earliest_slot_in_array <= slot - assert slot < current_block.slot.int + current_block.slot.int - state.latest_block_hashes.len + assert earliest_slot_in_array <= slot.int + assert slot < current_block.slot - state.recent_block_hashes[slot - earliest_slot_in_array] + state.latest_block_hashes[slot.int - earliest_slot_in_array] func append_to_recent_block_hashes*(old_block_hashes: seq[Eth2Digest], parent_slot, current_slot: uint64, @@ -122,8 +113,8 @@ func append_to_recent_block_hashes*(old_block_hashes: seq[Eth2Digest], result.add repeat(parent_hash, d) proc get_attestation_participants*(state: BeaconState, - attestation_data: AttestationSignedData, - attester_bitfield: seq[byte]): seq[int] = + attestation_data: AttestationData, + participation_bitfield: seq[byte]): seq[int] = ## Attestation participants in the attestation data are called out in a ## bit field that corresponds to the committee of the shard at the time - this ## function converts it to list of indices in to BeaconState.validators @@ -132,7 +123,7 @@ proc get_attestation_participants*(state: BeaconState, # TODO bitfield type needed, once bit order settles down # TODO iterator candidate let - sncs_for_slot = get_shards_and_committees_for_slot( + sncs_for_slot = get_shard_and_committees_for_slot( state, attestation_data.slot) for snc in sncs_for_slot: @@ -140,10 +131,10 @@ proc get_attestation_participants*(state: BeaconState, continue # TODO investigate functional library / approach to help avoid loop bugs - assert len(attester_bitfield) == ceil_div8(len(snc.committee)) + assert len(participation_bitfield) == ceil_div8(len(snc.committee)) for i, vindex in snc.committee: let - bit = (attester_bitfield[i div 8] shr (7 - (i mod 8))) mod 2 + bit = (participation_bitfield[i div 8] shr (7 - (i mod 8))) mod 2 if bit == 1: result.add(vindex) return # found the shard, we're done diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index d724d72af..12fe13ea8 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -37,9 +37,8 @@ const INITIAL_FORK_VERSION* = 0 # INITIAL_SLOT_NUMBER* = 0 # GWEI_PER_ETH* = 10^9 # Gwei/ETH + ZERO_HASH* = Eth2Digest() BEACON_CHAIN_SHARD_NUMBER* = not 0'u64 - WITHDRAWALS_PER_CYCLE* = 2^2 # validators (5.2m ETH in ~6 months) - MIN_WITHDRAWAL_PERIOD* = 2^13 # slots (~14 hours) # Time constants SLOT_DURATION* = 6 # seconds @@ -82,18 +81,18 @@ type proposer_signature*: ValidatorSig # Proposer signature AttestationRecord* = object - data*: AttestationSignedData # - attester_bitfield*: seq[byte] # Attester participation bitfield - poc_bitfield*: seq[byte] # Proof of custody bitfield + data*: AttestationData + participation_bitfield*: seq[byte] # Attester participation bitfield + custody_bitfield*: seq[byte] # Proof of custody bitfield aggregate_sig*: ValidatorSig # BLS aggregate signature - AttestationSignedData* = object + AttestationData* = object slot*: uint64 # Slot number shard*: uint64 # Shard number - block_hash*: Eth2Digest # Hash of the block we're signing - cycle_boundary_hash*: Eth2Digest # Hash of the ancestor at the cycle boundary + beacon_block_hash*: Eth2Digest # Hash of the block we're signing + epoch_boundary_hash*: Eth2Digest # Hash of the ancestor at the cycle boundary shard_block_hash*: Eth2Digest # Shard block hash being attested to - last_crosslink_hash*: Eth2Digest # Last crosslink hash + latest_crosslink_hash*: Eth2Digest # Last crosslink hash justified_slot*: uint64 # Slot of last justified beacon block justified_block_hash*: Eth2Digest # Hash of last justified beacon block @@ -107,31 +106,42 @@ type data*: seq[byte] # Data BeaconState* = object - validator_set_change_slot*: uint64 # Slot of last validator set change - validators*: seq[ValidatorRecord] # List of validators - crosslinks*: array[SHARD_COUNT, CrosslinkRecord] # Most recent crosslink for each shard - last_state_recalculation_slot*: uint64 # Last cycle-boundary state recalculation - last_finalized_slot*: uint64 # Last finalized slot - justification_source*: uint64 # Justification source - prev_cycle_justification_source*: uint64 # - justified_slot_bitfield*: uint64 # Recent justified slot bitmask + # Validator registry + validator_registry*: seq[ValidatorRecord] + validator_registry_latest_change_slot*: uint64 + validator_registry_exit_count*: uint64 + validator_registry_delta_chain_tip*: Eth2Digest ##\ + ## For light clients to easily track delta + + # Randomness and committees + randao_mix*: Eth2Digest # RANDAO state + next_seed*: Eth2Digest # Randao seed used for next shuffling shard_and_committee_for_slots*: array[2 * EPOCH_LENGTH, seq[ShardAndCommittee]] ## \ ## Committee members and their assigned shard, per slot, covers 2 cycles ## worth of assignments persistent_committees*: seq[seq[Uint24]] # Persistent shard committees persistent_committee_reassignments*: seq[ShardReassignmentRecord] - next_shuffling_seed*: Eth2Digest # Randao seed used for next shuffling - deposits_penalized_in_period*: uint32 # Total deposits penalized in the given withdrawal period - validator_set_delta_hash_chain*: Eth2Digest # Hash chain of validator set changes (for light clients to easily track deltas) - current_exit_seq*: uint64 # Current sequence number for withdrawals - genesis_time*: uint64 # Genesis time - candidate_pow_receipt_root*: Eth2Digest # PoW receipt root - candidate_pow_receipt_roots*: seq[CandidatePoWReceiptRootRecord] # - fork_data*: ForkData # Parameters relevant to hard forks / versioning. - # Should be updated only by hard forks. - pending_attestations*: seq[ProcessedAttestation] # Attestations not yet processed - recent_block_hashes*: seq[Eth2Digest] # recent beacon block hashes needed to process attestations, older to newer - randao_mix*: Eth2Digest # RANDAO state + + # Finality + previous_justified_slot*: uint64 + justified_slot*: uint64 + justified_slot_bitfield*: uint64 + finalized_slot*: uint64 + + latest_crosslinks*: array[SHARD_COUNT, CrosslinkRecord] + latest_state_recalculation_slot*: uint64 + latest_block_hashes*: seq[Eth2Digest] ##\ + ## Needed to process attestations, older to newer + latest_penalized_exit_balances*: seq[uint64] ##\ + ## Balances penalized in the current withdrawal period + latest_attestations*: seq[PendingAttestationRecord] + + processed_pow_receipt_root*: Eth2Digest + candidate_pow_receipt_roots*: seq[CandidatePoWReceiptRootRecord] + + genesis_time*: uint64 + fork_data*: ForkData ##\ + ## For versioning hard forks ValidatorRecord* = object pubkey*: ValidatorPubKey # Public key @@ -140,12 +150,12 @@ type randao_skips*: uint64 # Slot the proposer has skipped (ie. layers of RANDAO expected) balance*: uint64 # Balance in Gwei status*: ValidatorStatusCodes # Status code - last_status_change_slot*: uint64 # Slot when validator last changed status (or 0) - exit_seq*: uint64 # Sequence number when validator exited (or 0) + latest_status_change_slot*: uint64 # Slot when validator last changed status (or 0) + exit_count*: uint64 # Exit counter when validator exited (or 0) CrosslinkRecord* = object slot*: uint64 # Slot number - hash*: Eth2Digest # Shard chain block hash + shard_block_hash*: Eth2Digest # Shard chain block hash ShardAndCommittee* = object shard*: uint64 # Shard number @@ -163,12 +173,12 @@ type ForkData* = object pre_fork_version*: uint64 # Previous fork version post_fork_version*: uint64 # Post fork version - fork_slot_number*: uint64 # Fork slot number + fork_slot*: uint64 # Fork slot number - ProcessedAttestation* = object - data*: AttestationSignedData # Signed data - attester_bitfield*: seq[byte] # Attester participation bitfield (2 bits per attester) - poc_bitfield*: seq[byte] # Proof of custody bitfield + PendingAttestationRecord* = object + data*: AttestationData # Signed data + participation_bitfield*: seq[byte] # Attester participation bitfield + custody_bitfield*: seq[byte] # Proof of custody bitfield slot_included*: uint64 # Slot in which it was included ValidatorStatusCodes* {.pure.} = enum diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index dbd361239..a8f39d4ee 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -9,6 +9,9 @@ import ./datatypes, ./digest, sequtils, math +func mod_get[T](arr: openarray[T], pos: Natural): T = + arr[pos mod arr.len] + func shuffle*[T](values: seq[T], seed: Eth2Digest): seq[T] = ## Returns the shuffled ``values`` with seed as entropy. ## TODO: this calls out for tests, but I odn't particularly trust spec @@ -83,3 +86,50 @@ func repeat_hash*(v: Eth2Digest, n: SomeInteger): Eth2Digest = v else: repeat_hash(eth2hash(v.data), n - 1) + +func get_shard_and_committees_index*(state: BeaconState, slot: uint64): uint64 = + # TODO spec unsigned-unsafe here + let earliest_slot_in_array = + if state.latest_state_recalculation_slot > EPOCH_LENGTH.uint64: + state.latest_state_recalculation_slot - EPOCH_LENGTH + else: + 0 + + doAssert earliest_slot_in_array <= slot and + slot < earliest_slot_in_array + EPOCH_LENGTH * 2 + slot - earliest_slot_in_array + +proc get_shard_and_committees_for_slot*( + state: BeaconState, slot: uint64): seq[ShardAndCommittee] = + let index = state.get_shard_and_committees_index(slot) + state.shard_and_committee_for_slots[index] + +func get_beacon_proposer_index*(state: BeaconState, slot: uint64): uint64 = + ## From Casper RPJ mini-spec: + ## When slot i begins, validator Vidx is expected + ## to create ("propose") a block, which contains a pointer to some parent block + ## that they perceive as the "head of the chain", + ## and includes all of the **attestations** that they know about + ## that have not yet been included into that chain. + ## + ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee) + + let idx = get_shard_and_committees_index(state, slot) + state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot) + +func int_sqrt*(n: SomeInteger): SomeInteger = + var + x = n + y = (x + 1) div 2 + while y < x: + x = y + y = (x + n div x) div 2 + x + +func get_fork_version*(fork_data: ForkData, slot: uint64): uint64 = + if slot < fork_data.fork_slot: fork_data.pre_fork_version + else: fork_data.post_fork_version + +func get_domain*(fork_data: ForkData, slot: uint64, domain_type: uint64): uint64 = + # TODO Slot overflow? Or is slot 32 bits for all intents and purposes? + (get_fork_version(fork_data, slot) shl 32) + domain_type diff --git a/beacon_chain/spec/validator.nim b/beacon_chain/spec/validator.nim index 17f5b20d5..53eea0385 100644 --- a/beacon_chain/spec/validator.nim +++ b/beacon_chain/spec/validator.nim @@ -7,14 +7,16 @@ # Helpers and functions pertaining to managing the validator set import - options, + options, nimcrypto, eth_common, + ../ssz, ./crypto, ./datatypes, ./digest, ./helpers func min_empty_validator_index(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] = for i, v in validators: - if v.balance == 0 and v.last_status_change_slot + DELETION_PERIOD.uint64 <= current_slot: - return some(i) + if v.balance == 0 and + v.latest_status_change_slot + DELETION_PERIOD.uint64 <= current_slot: + return some(i) func get_new_validators*(current_validators: seq[ValidatorRecord], fork_data: ForkData, @@ -59,8 +61,8 @@ func get_new_validators*(current_validators: seq[ValidatorRecord], randao_skips: 0, balance: deposit, status: status, - last_status_change_slot: current_slot, - exit_seq: 0 + latest_status_change_slot: current_slot, + exit_count: 0 ) let index = min_empty_validator_index(new_validators, current_slot) @@ -74,7 +76,7 @@ func get_new_validators*(current_validators: seq[ValidatorRecord], func get_active_validator_indices*(validators: openArray[ValidatorRecord]): seq[Uint24] = ## Select the active validators for idx, val in validators: - if val.status == ACTIVE or val.status == PENDING_EXIT: + if val.status in {ACTIVE, PENDING_EXIT}: result.add idx.Uint24 func get_new_shuffling*(seed: Eth2Digest, @@ -109,3 +111,65 @@ func get_new_shuffling*(seed: Eth2Digest, committees[shard_position].committee = indices result[slot] = committees + +func get_new_validator_registry_delta_chain_tip( + current_validator_registry_delta_chain_tip: Eth2Digest, + index: Uint24, + pubkey: ValidatorPubKey, + flag: ValidatorSetDeltaFlags): Eth2Digest = + ## Compute the next hash in the validator registry delta hash chain. + + withEth2Hash: + h.update hashSSZ(current_validator_registry_delta_chain_tip) + h.update hashSSZ(flag.uint8) + h.update hashSSZ(index) + # TODO h.update hashSSZ(pubkey) + +func get_effective_balance*(validator: ValidatorRecord): uint64 = + min(validator.balance, MAX_DEPOSIT.uint64) + +func exit_validator*(index: Uint24, + state: var BeaconState, + penalize: bool, + current_slot: uint64) = + ## Remove the validator with the given `index` from `state`. + ## Note that this function mutates `state`. + + state.validator_registry_exit_count.inc() + + var + validator = state.validator_registry[index] + + validator.latest_status_change_slot = current_slot + validator.exit_count = state.validator_registry_exit_count + + # Remove validator from persistent committees + for committee in state.persistent_committees.mitems(): + for i, validator_index in committee: + if validator_index == index: + committee.delete(i) + break + + if penalize: + validator.status = EXITED_WITH_PENALTY + state.latest_penalized_exit_balances[ + (current_slot div COLLECTIVE_PENALTY_CALCULATION_PERIOD.uint64).int].inc( + get_effective_balance(validator).int) + + var + whistleblower = + state.validator_registry[get_beacon_proposer_index(state, current_slot).int] + whistleblower_reward = + validator.balance div WHISTLEBLOWER_REWARD_QUOTIENT.uint64 + whistleblower.balance.inc(whistleblower_reward.int) + validator.balance.dec(whistleblower_reward.int) + else: + validator.status = PENDING_EXIT + + state.validator_registry_delta_chain_tip = + get_new_validator_registry_delta_chain_tip( + state.validator_registry_delta_chain_tip, + index, + validator.pubkey, + EXIT, + ) diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index c9887ce02..f54501a2f 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -194,8 +194,8 @@ func hashSSZ*(x: ValidatorRecord): array[32, byte] = h.update hashSSZ(x.randao_skips) h.update hashSSZ(x.balance) # h.update hashSSZ(x.status) # TODO it's an enum, deal with it - h.update hashSSZ(x.last_status_change_slot) - h.update hashSSZ(x.exit_seq) + h.update hashSSZ(x.latest_status_change_slot) + h.update hashSSZ(x.exit_count) func hashSSZ*(x: ShardAndCommittee): array[32, byte] = withHash: diff --git a/beacon_chain/state_transition.nim b/beacon_chain/state_transition.nim index ee5c62774..c371b62fb 100644 --- a/beacon_chain/state_transition.nim +++ b/beacon_chain/state_transition.nim @@ -12,9 +12,9 @@ # missing pieces - needs testing throughout import - options, + math, options, sequtils, ./extras, - ./spec/[beaconstate, crypto, datatypes, digest, helpers], + ./spec/[beaconstate, crypto, datatypes, digest, helpers, validator], ./ssz, milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master @@ -24,30 +24,45 @@ import func checkAttestations(state: BeaconState, blck: BeaconBlock, - parent_slot: uint64): Option[seq[ProcessedAttestation]] = + parent_slot: uint64): Option[seq[PendingAttestationRecord]] = # TODO perf improvement potential.. if blck.attestations.len > MAX_ATTESTATIONS_PER_BLOCK: return - var res: seq[ProcessedAttestation] + var res: seq[PendingAttestationRecord] for attestation in blck.attestations: if attestation.data.slot <= blck.slot - MIN_ATTESTATION_INCLUSION_DELAY: return - if attestation.data.slot >= max(parent_slot - EPOCH_LENGTH + 1, 0): + # TODO unsigned undeflow in spec + if attestation.data.slot >= max(parent_slot.int - EPOCH_LENGTH + 1, 0).uint64: + return + + let expected_justified_slot = + if attestation.data.slot >= state.latest_state_recalculation_slot: + state.justified_slot + else: + state.previous_justified_slot + if attestation.data.justified_slot != expected_justified_slot: + return + + let expected_justified_block_hash = + get_block_hash(state, blck, attestation.data.justified_slot) + if attestation.data.justified_block_hash != expected_justified_block_hash: + return + + if state.latest_crosslinks[attestation.data.shard].shard_block_hash notin [ + attestation.data.latest_crosslink_hash, attestation.data.shard_block_hash]: return - #doAssert attestation.data.justified_slot == justification_source if attestation.data.slot >= state.last_state_recalculation_slot else prev_cycle_justification_source - # doAssert attestation.data.justified_block_hash == get_block_hash(state, block, attestation.data.justified_slot). - # doAssert either attestation.data.last_crosslink_hash or attestation.data.shard_block_hash equals state.crosslinks[shard].shard_block_hash. let attestation_participants = get_attestation_participants( - state, attestation.data, attestation.attester_bitfield) + state, attestation.data, attestation.participation_bitfield) var agg_pubkey: ValidatorPubKey empty = true for attester_idx in attestation_participants: - let validator = state.validators[attester_idx] + let validator = state.validator_registry[attester_idx] if empty: agg_pubkey = validator.pubkey empty = false @@ -62,10 +77,10 @@ func checkAttestations(state: BeaconState, debugEcho "Aggregate sig verify message: ", attestation.aggregate_sig.verifyMessage(msg, agg_pubkey) - res.add ProcessedAttestation( + res.add PendingAttestationRecord( data: attestation.data, - attester_bitfield: attestation.attester_bitfield, - poc_bitfield: attestation.poc_bitfield, + participation_bitfield: attestation.participation_bitfield, + custody_bitfield: attestation.custody_bitfield, slot_included: blck.slot ) @@ -84,7 +99,7 @@ func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool = verifyMessage( blck.proposer_signature, proposal_hash, - state.validators[get_beacon_proposer_index(state, blck.slot).int].pubkey) + state.validator_registry[get_beacon_proposer_index(state, blck.slot).int].pubkey) func processRandaoReveal(state: var BeaconState, blck: BeaconBlock, @@ -92,11 +107,11 @@ func processRandaoReveal(state: var BeaconState, # Update randao skips for slot in parentslot + 1 ..< blck.slot: let proposer_index = get_beacon_proposer_index(state, slot) - state.validators[proposer_index.int].randao_skips.inc() + state.validator_registry[proposer_index.int].randao_skips.inc() var proposer_index = get_beacon_proposer_index(state, blck.slot) - proposer = state.validators[proposer_index.int] + proposer = state.validator_registry[proposer_index.int] # Check that proposer commit and reveal match if repeat_hash(blck.randao_reveal, proposer.randao_skips + 1) != @@ -112,6 +127,22 @@ func processRandaoReveal(state: var BeaconState, true +func processPoWReceiptRoot(state: var BeaconState, blck: BeaconBlock): bool = + for x in state.candidate_pow_receipt_roots.mitems(): + if blck.candidate_pow_receipt_root == x.candidate_pow_receipt_root: + x.votes.inc + return true + + state.candidate_pow_receipt_roots.add CandidatePoWReceiptRootRecord( + candidate_pow_receipt_root: blck.candidate_pow_receipt_root, + votes: 1 + ) + return true + +func processSpecials(state: var BeaconState, blck: BeaconBlock): bool = + # TODO incoming spec changes here.. + true + func process_block*(state: BeaconState, blck: BeaconBlock): Option[BeaconState] = ## When a new block is received, all participants must verify that the block ## makes sense and update their state accordingly. This function will return @@ -127,15 +158,15 @@ func process_block*(state: BeaconState, blck: BeaconBlock): Option[BeaconState] # TODO actually get parent block, which means fixing up BeaconState refs above; # there's no distinction between active/crystallized state anymore, etc. - state.recent_block_hashes = - append_to_recent_block_hashes(state.recent_block_hashes, parent_slot, slot, + state.latest_block_hashes = + append_to_recent_block_hashes(state.latest_block_hashes, parent_slot, slot, parent_hash) let processed_attestations = checkAttestations(state, blck, parent_slot) if processed_attestations.isNone: return - state.pending_attestations.add processed_attestations.get() + state.latest_attestations.add processed_attestations.get() if not verifyProposerSignature(state, blck): return @@ -143,4 +174,183 @@ func process_block*(state: BeaconState, blck: BeaconBlock): Option[BeaconState] if not processRandaoReveal(state, blck, parent_slot): return + if not processPoWReceiptRoot(state, blck): + return + + if not processSpecials(state, blck): + return + some(state) # Looks ok - move on with the updated state + +func flatten[T](v: openArray[seq[T]]): seq[T] = + for x in v: result.add x + +func get_epoch_boundary_attesters( + state: BeaconState, + attestations: openArray[PendingAttestationRecord]): seq[int] = + deduplicate(flatten(mapIt(attestations, + get_attestation_participants(state, it.data, it.participation_bitfield)))) + +func adjust_for_inclusion_distance[T](magnitude: T, dist: T): T = + magnitude div 2 + (magnitude div 2) * MIN_ATTESTATION_INCLUSION_DELAY div dist + +func processEpoch*(state: BeaconState, blck: BeaconBlock): Option[BeaconState] = + ## Epoch processing happens every time we've passed EPOCH_LENGTH blocks. + ## Because some slots may be skipped, it may happen that we go through the + ## loop more than once - each time the latest_state_recalculation_slot will be + ## increased by EPOCH_LENGTH. + + # TODO: simplistic way to be able to rollback state + var state = state + + # Precomputation + + while blck.slot >= EPOCH_LENGTH.uint64 + state.latest_state_recalculation_slot: + let s = state.latest_state_recalculation_slot + + let + active_validators = + mapIt(get_active_validator_indices(state.validator_registry), + state.validator_registry[it]) + + total_balance = sum(mapIt(active_validators, get_effective_balance(it))) + + total_balance_in_eth = total_balance.int div GWEI_PER_ETH + + # The per-slot maximum interest rate is `2/reward_quotient`.) + reward_quotient = BASE_REWARD_QUOTIENT * int_sqrt(total_balance_in_eth) + + proc base_reward(v: ValidatorRecord): uint64 = + get_effective_balance(v) div reward_quotient.uint64 + + # TODO doing this with iterators failed: + # https://github.com/nim-lang/Nim/issues/9827 + let + this_epoch_attestations = filterIt(state.latest_attestations, + s <= it.data.slot and it.data.slot < s + EPOCH_LENGTH) + + this_epoch_boundary_attestations = filterIt(this_epoch_attestations, + it.data.epoch_boundary_hash == get_block_hash(state, blck, s) and + it.data.justified_slot == state.justified_slot) + + this_epoch_boundary_attesters = + get_epoch_boundary_attesters(state, this_epoch_attestations) + + this_epoch_boundary_attesting_balance = sum( + mapIt(this_epoch_boundary_attesters, + get_effective_balance(state.validator_registry[it])) + ) + + let + previous_epoch_attestations = filterIt(state.latest_attestations, + s <= it.data.slot + EPOCH_LENGTH and it.data.slot < s) + previous_epoch_boundary_attestations = filterIt(previous_epoch_attestations, + it.data.epoch_boundary_hash == get_block_hash(state, blck, s) and + it.data.justified_slot == state.justified_slot) + previous_epoch_boundary_attesters = + get_epoch_boundary_attesters(state, previous_epoch_boundary_attestations) + previous_epoch_boundary_attesting_balance = sum( + mapIt(previous_epoch_boundary_attesters, + get_effective_balance(state.validator_registry[it])) + ) + + # TODO gets pretty hairy here + func attesting_validators( + obj: ShardAndCommittee, shard_block_hash: Eth2Digest): seq[int] = + flatten( + mapIt( + filterIt(concat(this_epoch_attestations, previous_epoch_attestations), + it.data.shard == obj.shard and + it.data.shard_block_hash == shard_block_hash), + get_attestation_participants(state, it.data, it.participation_bitfield))) + + # TODO which shard_block_hash:es? + # * Let `attesting_validators(obj)` be equal to `attesting_validators(obj, shard_block_hash)` for the value of `shard_block_hash` such that `sum([get_effective_balance(v) for v in attesting_validators(obj, shard_block_hash)])` is maximized (ties broken by favoring lower `shard_block_hash` values). + # * Let `total_attesting_balance(obj)` be the sum of the balances-at-stake of `attesting_validators(obj)`. + # * Let `winning_hash(obj)` be the winning `shard_block_hash` value. + # * Let `total_balance(obj) = sum([get_effective_balance(v) for v in obj.committee])`. + + # Let `inclusion_slot(v)` equal `a.slot_included` for the attestation `a` where `v` is in `get_attestation_participants(state, a.data, a.participation_bitfield)`, and `inclusion_distance(v) = a.slot_included - a.data.slot` for the same attestation. We define a function `adjust_for_inclusion_distance(magnitude, distance)` which adjusts the reward of an attestation based on how long it took to get included (the longer, the lower the reward). Returns a value between 0 and `magnitude`. + + # Adjust justified slots and crosslink status + + var new_justified_slot: Option[uint64] + # overflow intentional! + state.justified_slot_bitfield = state.justified_slot_bitfield * 2 + + if 3'u64 * previous_epoch_boundary_attesting_balance >= 2'u64 * total_balance: + # TODO spec says "flip the second lowest bit to 1" and does "AND", wrong? + state.justified_slot_bitfield = state.justified_slot_bitfield or 2 + new_justified_slot = some(s - EPOCH_LENGTH) + + if 3'u64 * this_epoch_boundary_attesting_balance >= 2'u64 * total_balance: + # TODO spec says "flip the second lowest bit to 1" and does "AND", wrong? + state.justified_slot_bitfield = state.justified_slot_bitfield or 1 + new_justified_slot = some(s) + + if state.justified_slot == s - EPOCH_LENGTH and + state.justified_slot_bitfield mod 4 == 3: + state.finalized_slot = state.justified_slot + if state.justified_slot == s - EPOCH_LENGTH - EPOCH_LENGTH and + state.justified_slot_bitfield mod 8 == 7: + state.finalized_slot = state.justified_slot + + if state.justified_slot == s - EPOCH_LENGTH - 2 * EPOCH_LENGTH and + state.justified_slot_bitfield mod 16 in [15'u64, 14]: + state.finalized_slot = state.justified_slot + + state.previous_justified_slot = state.justified_slot + + if new_justified_slot.isSome(): + state.justified_slot = new_justified_slot.get() + + # for obj in state.shard_and_committee_for_slots: + # 3 * total_attesting_balance(obj) >= 2 * total_balance(obj): + # state.crosslinks[shard] = CrosslinkRecord( + # slot: latest_state_recalculation_slot + EPOCH_LENGTH, + # hash: winning_hash(obj)) + + # Balance recalculations related to FFG rewards + let + # The portion lost by offline [validators](#dfn-validator) after `D` + # epochs is about `D*D/2/inactivity_penalty_quotient`. + inactivity_penalty_quotient = SQRT_E_DROP_TIME^2 + time_since_finality = blck.slot - state.finalized_slot + + if time_since_finality <= 4'u64 * EPOCH_LENGTH: + # for v in previous_epoch_boundary_attesters: + # state.validators[v].balance.inc(adjust_for_inclusion_distance( + # base_reward(state.validators[v]) * + # prev_cycle_boundary_attesting_balance div total_balance, + # inclusion_distance(v))) + + for v in get_active_validator_indices(state.validator_registry): + if v notin previous_epoch_boundary_attesters: + state.validator_registry[v].balance.dec( + base_reward(state.validator_registry[v]).int) + else: + # Any validator in `prev_cycle_boundary_attesters` sees their balance + # unchanged. + # Others might get penalized: + for vindex, v in state.validator_registry.mpairs(): + if (v.status == ACTIVE and vindex notin previous_epoch_boundary_attesters) or + v.status == EXITED_WITH_PENALTY: + v.balance.dec( + (base_reward(v) + get_effective_balance(v) * time_since_finality div + inactivity_penalty_quotient.uint64).int) + + # For each `v` in `prev_cycle_boundary_attesters`, we determine the proposer `proposer_index = get_beacon_proposer_index(state, inclusion_slot(v))` and set `state.validators[proposer_index].balance += base_reward(v) // INCLUDER_REWARD_SHARE_QUOTIENT`. + + # Balance recalculations related to crosslink rewards + + # Ethereum 1.0 chain related rules + + # Validator registry change + + # If a validator registry change does NOT happen + + # Proposer reshuffling + + # Finally... + + some(state) diff --git a/beacon_chain/sync_protocol.nim b/beacon_chain/sync_protocol.nim index 6d2c71847..5ca110012 100644 --- a/beacon_chain/sync_protocol.nim +++ b/beacon_chain/sync_protocol.nim @@ -85,9 +85,9 @@ proc applyValidatorChangeLog*(log: ChangeLog, # 4. Apply all changes to the validator set # - outBeaconState.last_finalized_slot = + outBeaconState.finalized_slot = log.signedBlock.slot div EPOCH_LENGTH - outBeaconState.validator_set_delta_hash_chain = - log.beaconState.validator_set_delta_hash_chain + outBeaconState.validator_registry_delta_chain_tip = + log.beaconState.validator_registry_delta_chain_tip diff --git a/beacon_chain/time.nim b/beacon_chain/time.nim index dbe4b7528..b480631ac 100644 --- a/beacon_chain/time.nim +++ b/beacon_chain/time.nim @@ -35,7 +35,7 @@ proc randomTimeInSlot*(s: BeaconState, proc slotDistanceFromNow*(s: BeaconState): int64 = ## Returns how many slots have passed since a particular BeaconState was finalized - int64(s.timeSinceGenesis() div (SLOT_DURATION * 1000)) - int64(s.last_finalized_slot) + int64(s.timeSinceGenesis() div (SLOT_DURATION * 1000)) - int64(s.finalized_slot) proc syncrhronizeClock*() {.async.} = ## This should determine the offset of the local clock against a global diff --git a/beacon_chain/validator_pool.nim b/beacon_chain/validator_pool.nim index 67454787c..f904e070a 100644 --- a/beacon_chain/validator_pool.nim +++ b/beacon_chain/validator_pool.nim @@ -53,7 +53,7 @@ proc signBlockProposal*(v: AttachedValidator, discard proc signAttestation*(v: AttachedValidator, - attestation: AttestationSignedData): Future[ValidatorSig] {.async.} = + attestation: AttestationData): Future[ValidatorSig] {.async.} = # TODO: implement this if v.kind == inProcess: await sleepAsync(1) diff --git a/tests/test_beaconstate.nim b/tests/test_beaconstate.nim index 5bb28c276..379e3e75d 100644 --- a/tests/test_beaconstate.nim +++ b/tests/test_beaconstate.nim @@ -16,4 +16,4 @@ suite "Beacon state": test "Smoke on_startup": let state = on_startup(makeInitialValidators(EPOCH_LENGTH), 0, Eth2Digest()) - check: state.validators.len == EPOCH_LENGTH + check: state.validator_registry.len == EPOCH_LENGTH