spec updates (#39)

* some minor renames
* fix some leftover block slot references (replaced by state slot)
* fix some more uint underflows
* update epoch calculations (reward et al)
* now works past 100 epochs at least with empty blocks (very slow)

beacon_chain/beacon_node.nim

@@ -168,7 +168,7 @@ proc scheduleCycleActions(node: BeaconNode) =
 
     # Schedule attestations
     let
-      committeesIdx = get_shard_and_committees_index(node.beaconState, slot.uint64)
+      committeesIdx = get_shard_committees_index(node.beaconState, slot.uint64)
 
     for shard in node.beaconState.shard_committees_at_slots[committees_idx]:
       for validatorIdx in shard.committee:

beacon_chain/spec/beaconstate.nim

@@ -188,7 +188,6 @@ func on_startup*(initial_validator_deposits: openArray[Deposit],
 
     # Recent state
     latest_state_recalculation_slot: INITIAL_SLOT_NUMBER,
-    latest_block_roots: repeat(ZERO_HASH, LATEST_BLOCK_ROOTS_COUNT),
 
      # PoW receipt root
     processed_pow_receipt_root: processed_pow_receipt_root,
@@ -228,18 +227,10 @@ func on_startup*(initial_validator_deposits: openArray[Deposit],
 
 func get_block_root*(state: BeaconState,
                      slot: uint64): Eth2Digest =
-  let earliest_slot_in_array =
-    state.slot - len(state.latest_block_roots).uint64
-  assert earliest_slot_in_array <= slot
-  assert slot < state.slot
-  state.latest_block_roots[(slot - earliest_slot_in_array).int]
-
-func append_to_recent_block_roots*(old_block_roots: seq[Eth2Digest],
-                                    parent_slot, current_slot: uint64,
-                                    parent_hash: Eth2Digest): seq[Eth2Digest] =
-  let d = current_slot - parent_slot
-  result = old_block_roots
-  result.add repeat(parent_hash, d)
+  doAssert slot + len(state.latest_block_roots).uint64 > state.slot
+  doAssert slot < state.slot
+  state.latest_block_roots[
+    (slot + len(state.latest_block_roots).uint64 - state.slot).int]
 
 func get_attestation_participants*(state: BeaconState,
                                    attestation_data: AttestationData,
@@ -252,7 +243,7 @@ func get_attestation_participants*(state: BeaconState,
   # TODO bitfield type needed, once bit order settles down
   # TODO iterator candidate
   let
-    sncs_for_slot = get_shard_and_committees_for_slot(
+    sncs_for_slot = get_shard_committees_at_slot(
       state, attestation_data.slot)
 
   for snc in sncs_for_slot:

beacon_chain/spec/datatypes.nim

@@ -277,7 +277,7 @@ type
 
     latest_crosslinks*: array[SHARD_COUNT, CrosslinkRecord]
     latest_state_recalculation_slot*: uint64
-    latest_block_roots*: seq[Eth2Digest] ##\
+    latest_block_roots*: array[LATEST_BLOCK_ROOTS_COUNT, Eth2Digest] ##\
     ## Needed to process attestations, older to newer
     latest_penalized_exit_balances*: seq[uint64] ##\
     ## Balances penalized in the current withdrawal period

beacon_chain/spec/helpers.nim

@@ -92,21 +92,14 @@ func repeat_hash*(v: Eth2Digest, n: SomeInteger): Eth2Digest =
     result = eth2hash(result.data)
     dec n
 
-func get_shard_and_committees_index*(state: BeaconState, slot: uint64): uint64 =
+func get_shard_committees_index*(state: BeaconState, slot: uint64): uint64 =
   # TODO spec unsigned-unsafe here
-  let earliest_slot_in_array =
-    if state.latest_state_recalculation_slot > EPOCH_LENGTH.uint64:
-      state.latest_state_recalculation_slot - EPOCH_LENGTH
-    else:
-      0
+  doAssert slot + (state.slot mod EPOCH_LENGTH) + EPOCH_LENGTH > state.slot
+  slot + (state.slot mod EPOCH_LENGTH) + EPOCH_LENGTH - state.slot
 
-  doAssert earliest_slot_in_array <= slot and
-           slot < earliest_slot_in_array + EPOCH_LENGTH * 2
-  slot - earliest_slot_in_array
-
-proc get_shard_and_committees_for_slot*(
+proc get_shard_committees_at_slot*(
     state: BeaconState, slot: uint64): seq[ShardCommittee] =
-  let index = state.get_shard_and_committees_index(slot)
+  let index = state.get_shard_committees_index(slot)
   state.shard_committees_at_slots[index]
 
 func get_beacon_proposer_index*(state: BeaconState, slot: uint64): Uint24 =
@@ -119,7 +112,7 @@ func get_beacon_proposer_index*(state: BeaconState, slot: uint64): Uint24 =
   ##
   ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
 
-  let idx = get_shard_and_committees_index(state, slot)
+  let idx = get_shard_committees_index(state, slot)
   state.shard_committees_at_slots[idx][0].committee.mod_get(slot)
 
 func int_sqrt*(n: SomeInteger): SomeInteger =

beacon_chain/state_transition.nim

@@ -354,7 +354,8 @@ func processSlot(state: var BeaconState, latest_block: BeaconBlock) =
     previous_block_root = Eth2Digest(data: hash_tree_root(latest_block))
   for i in 0 ..< state.latest_block_roots.len - 1:
     state.latest_block_roots[i] = state.latest_block_roots[i + 1]
-  state.latest_block_roots[^1] = previous_block_root
+  state.latest_block_roots[state.latest_block_roots.len - 1] =
+    previous_block_root
 
   if state.slot mod LATEST_BLOCK_ROOTS_COUNT == 0:
     state.batched_block_roots.add(merkle_root(state.latest_block_roots))
@@ -393,17 +394,11 @@ func lowerThan(candidate, current: Eth2Digest): bool =
     if v > candidate.data[i]: return true
   return false
 
-func processEpoch(state: var BeaconState, blck: BeaconBlock) =
-  ## Epoch processing happens every time we've passed EPOCH_LENGTH blocks.
-  ## Because some slots may be skipped, it may happen that we go through the
-  ## loop more than once - each time the latest_state_recalculation_slot will be
-  ## increased by EPOCH_LENGTH.
-  ##
+func processEpoch(state: var BeaconState) =
   ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#per-epoch-processing
 
-  while blck.slot >= EPOCH_LENGTH.uint64 + state.latest_state_recalculation_slot:
-    # Convenience shortcut, from spec
-    let s = state.latest_state_recalculation_slot
+  if state.slot mod EPOCH_LENGTH != 0:
+    return
 
   # Precomputation
   let
@@ -413,22 +408,28 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
     total_balance_in_eth = total_balance div GWEI_PER_ETH
 
     # The per-slot maximum interest rate is `2/reward_quotient`.)
-      reward_quotient = BASE_REWARD_QUOTIENT * int_sqrt(total_balance_in_eth)
+    base_reward_quotient = BASE_REWARD_QUOTIENT * int_sqrt(total_balance_in_eth)
 
-      # TODO not in spec, convenient
-      epoch_boundary_root = get_block_root(state, s)
+  func base_reward(v: ValidatorRecord): uint64 =
+    get_effective_balance(v) div base_reward_quotient.uint64 div 4
 
-    proc base_reward(v: ValidatorRecord): uint64 =
-      get_effective_balance(v) div reward_quotient.uint64
+  func inactivity_penalty(
+      v: ValidatorRecord, slots_since_finality: uint64): uint64 =
+    base_reward(v) +
+      get_effective_balance(v) *
+        slots_since_finality div INACTIVITY_PENALTY_QUOTIENT
 
   # TODO doing this with iterators failed:
   #      https://github.com/nim-lang/Nim/issues/9827
   let
-      this_epoch_attestations = filterIt(state.latest_attestations,
-        s <= it.data.slot and it.data.slot < s + EPOCH_LENGTH)
+    this_epoch_attestations =
+      filterIt(state.latest_attestations,
+        state.slot <= it.data.slot + EPOCH_LENGTH and
+        it.data.slot < state.slot)
 
     this_epoch_boundary_attestations =
-        boundary_attestations(state, epoch_boundary_root,
+      boundary_attestations(
+        state, get_block_root(state, state.slot-EPOCH_LENGTH),
         this_epoch_attestations)
 
     this_epoch_boundary_attesters =
@@ -438,29 +439,71 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
       sum_effective_balances(state, this_epoch_boundary_attesters)
 
   let
-      previous_epoch_attestations = filterIt(state.latest_attestations,
-        s <= it.data.slot + EPOCH_LENGTH and it.data.slot < s)
+    previous_epoch_attestations = filterIt(
+      state.latest_attestations,
+      state.slot <= it.data.slot + 2 * EPOCH_LENGTH and
+      it.data.slot + EPOCH_LENGTH < state.slot)
 
+    previous_epoch_attesters = flatten(mapIt(
+      previous_epoch_attestations,
+      get_attestation_participants(state, it.data, it.participation_bitfield)
+    ))
+
+  let # Previous epoch justified
+    previous_epoch_justified_attestations = filterIt(
+      concat(this_epoch_attestations, previous_epoch_attestations),
+        it.data.justified_slot == state.previous_justified_slot
+      )
+
+    previous_epoch_justified_attesters = flatten(mapIt(
+      previous_epoch_justified_attestations,
+      get_attestation_participants(state, it.data, it.participation_bitfield)
+    ))
+
+    previous_epoch_justified_attesting_balance =
+      sum_effective_balances(state, previous_epoch_justified_attesters)
+
+  let # Previous epoch boundary
+    # TODO check this with spec...
+    negative_uint_hack =
+      if state.slot < 2 * EPOCH_LENGTH: 0'u64 else: state.slot - 2 * EPOCH_LENGTH
     previous_epoch_boundary_attestations =
-        boundary_attestations(state, epoch_boundary_root,
+      boundary_attestations(
+        state, get_block_root(state, negative_uint_hack),
         previous_epoch_attestations)
 
-      previous_epoch_boundary_attesters =
-        get_epoch_boundary_attesters(state, previous_epoch_boundary_attestations)
+    previous_epoch_boundary_attesters = flatten(mapIt(
+      previous_epoch_boundary_attestations,
+      get_attestation_participants(state, it.data, it.participation_bitfield)
+    ))
 
     previous_epoch_boundary_attesting_balance =
-        sum_effective_balances(state, this_epoch_boundary_attesters)
+      sum_effective_balances(state, previous_epoch_boundary_attesters)
+
+  let # Previous epoch head
+    previous_epoch_head_attestations =
+      filterIt(
+        previous_epoch_attestations,
+        it.data.beacon_block_root == get_block_root(state, it.data.slot))
+
+    previous_epoch_head_attesters = flatten(mapIt(
+      previous_epoch_head_attestations,
+      get_attestation_participants(state, it.data, it.participation_bitfield)
+    ))
+
+    previous_epoch_head_attesting_balance =
+      sum_effective_balances(state, previous_epoch_head_attesters)
 
   # TODO this is really hairy - we cannot capture `state` directly, but we
   #      can capture a pointer to it - this is safe because we don't leak
   #      these closures outside this scope, but still..
   let statePtr = state.addr
   func attesting_validators(
-        obj: ShardCommittee, shard_block_root: Eth2Digest): seq[Uint24] =
+      shard_committee: ShardCommittee, shard_block_root: Eth2Digest): seq[Uint24] =
     flatten(
       mapIt(
         filterIt(concat(this_epoch_attestations, previous_epoch_attestations),
-            it.data.shard == obj.shard and
+          it.data.shard == shard_committee.shard and
             it.data.shard_block_root == shard_block_root),
         get_attestation_participants(statePtr[], it.data, it.participation_bitfield)))
 
@@ -474,6 +517,10 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
           it.data.shard == obj.shard),
         it.data.shard_block_root)
 
+    # TODO not covered by spec!
+    if candidates.len == 0:
+      return
+
     var max_hash = candidates[0]
     var max_val =
       sum_effective_balances(statePtr[], attesting_validators(obj, max_hash))
@@ -542,18 +589,31 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
 
   block: # Crosslinks
     for sac in state.shard_committees_at_slots:
-        for obj in sac:
-          if 3'u64 * total_attesting_balance(obj) >=
-              2'u64 * total_balance_sac(obj):
-            state.latest_crosslinks[obj.shard] = CrosslinkRecord(
+      for shard_committee in sac:
+        if 3'u64 * total_attesting_balance(shard_committee) >=
+            2'u64 * total_balance_sac(shard_committee):
+          state.latest_crosslinks[shard_committee.shard] = CrosslinkRecord(
             slot: state.latest_state_recalculation_slot + EPOCH_LENGTH,
-              shard_block_root: winning_hash(obj))
+            shard_block_root: winning_hash(shard_committee))
 
-    block: # Justification and finalization rewards and penalties
+  block: # Justification and finalization
     let
-        slots_since_finality = blck.slot - state.finalized_slot
+      slots_since_finality = state.slot - state.finalized_slot
 
     if slots_since_finality <= 4'u64 * EPOCH_LENGTH:
+      # Expected FFG source
+      for v in previous_epoch_justified_attesters:
+        state.validator_registry[v].balance += adjust_for_inclusion_distance(
+          base_reward(state.validator_registry[v]) *
+          previous_epoch_justified_attesting_balance div total_balance,
+          inclusion_distance(v))
+
+      for v in active_validator_indices:
+        if v notin previous_epoch_justified_attesters:
+          state.validator_registry[v].balance -=
+            base_reward(state.validator_registry[v])
+
+      # Expected FFG target:
       for v in previous_epoch_boundary_attesters:
         state.validator_registry[v].balance += adjust_for_inclusion_distance(
           base_reward(state.validator_registry[v]) *
@@ -564,26 +624,43 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
         if v notin previous_epoch_boundary_attesters:
           state.validator_registry[v].balance -=
             base_reward(state.validator_registry[v])
-      else:
-        # Any validator in `prev_cycle_boundary_attesters` sees their balance
-        # unchanged.
-        # Others might get penalized:
-        for vindex, v in state.validator_registry.mpairs():
-          if (v.status == ACTIVE and
-                vindex.Uint24 notin previous_epoch_boundary_attesters) or
-              v.status == EXITED_WITH_PENALTY:
-            v.balance -= base_reward(v) +
-              get_effective_balance(v) * slots_since_finality div
-                INACTIVITY_PENALTY_QUOTIENT
 
-        for v in previous_epoch_boundary_attesters:
+      # Expected beacon chain head:
+      for v in previous_epoch_head_attesters:
+        state.validator_registry[v].balance += adjust_for_inclusion_distance(
+          base_reward(state.validator_registry[v]) *
+          previous_epoch_head_attesting_balance div total_balance,
+          inclusion_distance(v))
+
+      for v in active_validator_indices:
+        if v notin previous_epoch_head_attesters:
+          state.validator_registry[v].balance -=
+            base_reward(state.validator_registry[v])
+
+    else:
+      for v in active_validator_indices:
+        let validator = addr state.validator_registry[v]
+        if v notin previous_epoch_justified_attesters:
+          validator[].balance -=
+            inactivity_penalty(validator[], slots_since_finality)
+        if v notin previous_epoch_boundary_attesters:
+          validator[].balance -=
+            inactivity_penalty(validator[], slots_since_finality)
+        if v notin previous_epoch_head_attesters:
+          validator[].balance -=
+            inactivity_penalty(validator[], slots_since_finality)
+        if validator[].status == EXITED_WITH_PENALTY:
+          validator[].balance -=
+            3'u64 * inactivity_penalty(validator[], slots_since_finality)
+
+  block: # Attestation inclusion
+    for v in previous_epoch_attesters:
       let proposer_index =
         get_beacon_proposer_index(state, inclusion_slot(v))
       state.validator_registry[proposer_index].balance +=
-            base_reward(state.validator_registry[v]) div
-              INCLUDER_REWARD_QUOTIENT
+        base_reward(state.validator_registry[v]) div INCLUDER_REWARD_QUOTIENT
 
-    block: # Crosslink rewards and penalties
+  block: # Crosslinks
     for sac in state.shard_committees_at_slots[0 ..< EPOCH_LENGTH]:
       for obj in sac:
         for vindex in obj.committee:
@@ -591,13 +668,13 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
 
           if vindex in attesting_validators(obj):
             v.balance += adjust_for_inclusion_distance(
-                base_reward(v[]) * total_attesting_balance(obj) div total_balance_sac(obj),
+              base_reward(v[]) * total_attesting_balance(obj) div
+                total_balance_sac(obj),
               inclusion_distance(vindex))
           else:
             v.balance -= base_reward(v[])
 
   block: # Validator registry
-
     if state.finalized_slot > state.validator_registry_latest_change_slot and
         allIt(state.shard_committees_at_slots,
           allIt(it,
@@ -636,24 +713,24 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
         # Note that `start_shard` is not changed from the last epoch.
 
   block: # Proposer reshuffling
-      let active_validator_indices = get_active_validator_indices(state.validator_registry)
     let num_validators_to_reshuffle =
       len(active_validator_indices) div
         SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD.int
     for i in 0..<num_validators_to_reshuffle:
       # Multiplying i to 2 to ensure we have different input to all the required hashes in the shuffling
       # and none of the hashes used for entropy in this loop will be the same
-        let validator_index = 0.Uint24 # active_validator_indices[hash(state.randao_mix + bytes8(i * 2)) mod len(active_validator_indices)]
-        let new_shard = 0'u64 # hash(state.randao_mix + bytes8(i * 2 + 1)) mod SHARD_COUNT
-        let shard_reassignment_record = ShardReassignmentRecord(
+      let
+        validator_index = 0.Uint24 # active_validator_indices[hash(state.randao_mix + bytes8(i * 2)) mod len(active_validator_indices)]
+        new_shard = 0'u64 # hash(state.randao_mix + bytes8(i * 2 + 1)) mod SHARD_COUNT
+        shard_reassignment_record = ShardReassignmentRecord(
           validator_index: validator_index,
           shard: new_shard,
-            slot: s + SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD
+          slot: state.slot + SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD
         )
       state.persistent_committee_reassignments.add(shard_reassignment_record)
 
     while len(state.persistent_committee_reassignments) > 0 and
-          state.persistent_committee_reassignments[0].slot <= s:
+        state.persistent_committee_reassignments[0].slot <= state.slot:
       let reassignment = state.persistent_committee_reassignments[0]
       state.persistent_committee_reassignments.delete(0)
       for committee in state.persistent_committees.mitems():
@@ -663,8 +740,9 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock) =
         reassignment.validator_index)
 
   block: # Final updates
-      # TODO Remove all attestation records older than slot `s`.
-      state.latest_block_roots = state.latest_block_roots[EPOCH_LENGTH..^1]
+    state.latest_attestations.keepItIf(
+      not (it.data.slot + EPOCH_LENGTH < state.slot)
+    )
 
 proc updateState*(state: BeaconState, latest_block: BeaconBlock,
     new_block: Option[BeaconBlock]):
@@ -716,7 +794,7 @@ proc updateState*(state: BeaconState, latest_block: BeaconBlock,
       true
 
   # Heavy updates that happen for every epoch - these never fail (or so we hope)
-  processEpoch(new_state, latest_block)
+  processEpoch(new_state)
 
   # State update never fails, but block validation might...
   (new_state, block_ok)

tests/testutil.nim

@@ -73,13 +73,14 @@ func makeGenesisBlock*(state: BeaconState): BeaconBlock =
 
 func makeBlock*(
     state: BeaconState, latest_block: BeaconBlock): BeaconBlock =
+  var next_state = state
+  next_state.slot += 1
   let
-    new_slot = state.slot + 1
     proposer = state.validator_registry[
-      get_beacon_proposer_index(state, new_slot)]
+      get_beacon_proposer_index(next_state, next_state.slot)]
 
   var new_block = BeaconBlock(
-      slot: new_slot,
+      slot: next_state.slot,
       state_root: Eth2Digest(data: hash_tree_root(state)),
       randao_reveal: hackReveal(proposer)
     )