Update helpers to latest sped, highlight spec bugs

beacon_chain/private/helpers.nim

@@ -6,74 +6,98 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 # Helper functions
-import ../datatypes, sequtils, nimcrypto
+import ../datatypes, sequtils, nimcrypto, math
 
-func getShuffling*(seed: Blake2_256_Digest, validatorCount: int): seq[int] {.noInit.}=
+func get_active_validator_indices(validators: seq[ValidatorRecord], dynasty: int64): seq[Uint24] =
+  ## Select the active validators
+  result = @[]
+  for idx, val in validators:
+    if  val.start_dynasty <= dynasty and
+        dynasty < val.end_dynasty:
+      result.add idx.Uint24
+
+func shuffle(validators: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit.}=
   ## Pseudorandomly shuffles the validator set based on some seed
 
-  assert validatorCount <= MaxValidatorCount
+  const UpperBound = 2^24 # 16777216
+  assert validators.len <= UpperBound
 
-  let randMax = MaxValidatorCount - MaxValidatorCount mod validatorCount
-  result = toSeq(0 ..< validatorCount)
+  deepCopy(result, validators)
   var source = seed
 
   var i = 0
-  while i < validatorCount:
+  while i < validators.len:
     source = blake2_256.digest source.data
     for pos in countup(0, 29, 3):
-      let remaining = validatorCount - i
+      let remaining = validators.len - i
       if remaining == 0:
         break
 
-      let m = source.data[pos].int shl 16 or source.data[pos+1].int shl 8 or source.data[pos+2].int
+      let m = source.data[pos].Uint24 shl 16 or source.data[pos+1].Uint24 shl 8 or source.data[pos+2].Uint24
+      let rand_max = Uint24 UpperBound - UpperBound mod remaining
 
-      if validatorCount < randMax:
+      if m < randMax:
         let replacementPos = m mod remaining + i
         swap result[i], result[replacementPos]
         inc i
 
-func getCutoffs*(validatorCount: int): tuple[height, shard: seq[int]] {.noInit.} =
+func split[T](lst: seq[T], N: Positive): seq[seq[T]] =
+  # TODO: implement as an iterator
+  result = newSeq[seq[T]](N)
+  for i in 0 ..< N:
+    result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray
+
+func get_new_shuffling*(seed: Blake2_256_Digest, validators: seq[ValidatorRecord],
+    dynasty: int64, crosslinking_start_shard: int16): seq[seq[ShardAndCommittee]] {.noInit.} =
   ## Split up validators into groups at the start of every epoch,
   ## determining at what height they can make attestations and what shard they are making crosslinks for
   ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png
-  ## TODO: It doens't work.
 
-  result.height = @[0]
-  let cofactor = 39 # EpochLength / phi
-  const StandardCommitteeSize = MaxValidatorCount div ShardCount
+  let avs = get_active_validator_indices(validators, dynasty)
+  var committees_per_slot, slots_per_committee: int16
 
-  var heightCount: int
-  var heights: seq[int]
-
-  if validatorCount < EpochLength * MinCommitteeSize:
-    # If there are not enough validators to fill a minimally
-    # sized committee at every height, skip some heights
-    heightCount = validatorCount div MinCommitteeSize or 1 # TODO div/or precedence ?
-    for i in 0 ..< heightCount:
-      heights.add (i * cofactor) mod EpochLength
+  if avs.len >= CYCLE_LENGTH * MIN_COMMITTEE_SIZE:
+    committees_per_slot = int16 avs.len div CYCLE_LENGTH div (MIN_COMMITTEE_SIZE * 2) + 1
+    slots_per_committee = 1
   else:
-    # If there are enough validators, fill all the heights
-    heightCount = EpochLength
-    heights = toSeq(0 ..< EpochLength)
+    committees_per_slot = 1
+    slots_per_committee = 1
+    while avs.len * slots_per_committee < CYCLE_LENGTH * MIN_COMMITTEE_SIZE and
+        slots_per_committee < CYCLE_LENGTH:
+      slots_per_committee *= 2
 
-  var filled = 0
-  for i in 0 ..< EpochLength - 1:
-    if i notin heights: # TODO, this will be slow for seq, use intsets instead?
-      result.height.add result.height[^1]
-    else:
-      inc filled
-      result.height.add filled * validatorCount div heightCount
-  result.height.add validatorCount
+  result = @[]
+  for slot, height_indices in shuffle(avs, seed).split(CYCLE_LENGTH):
+    let shard_indices = height_indices.split(committees_per_slot)
 
-  # For the validators assigned to each height, split them up
-  # into committees for different shards. Do not assign the
-  # last END_EPOCH_GRACE_PERIOD heights in an epoch to any shards.
+    var committees = newSeq[ShardAndCommittee](shard_indices.len)
+    for j, indices in shard_indices:
+      committees[j].shard_id = crosslinking_start_shard +
+                                slot.int16 * committees_per_slot div slots_per_committee + j.int16
+      committees[j].committee = indices
 
-  result.shard = @[0]
-  for i in 0 ..< EpochLength - EndEpochGracePeriod:
-    let
-      size = result.height[i+1] - result.height[i]
-      shards = (size + StandardCommitteeSize - 1) div StandardCommitteeSize
-      pre = result.shard[^1]
-    for j in 1 .. shards:
-      result.shard.add pre + size * j div shards
+    result.add committees
+
+func get_indices_for_slot(crystallized_state: CrystallizedState,
+        slot: int64): seq[ShardAndCommittee] {.noInit.}=
+  # TODO: Spec why is active_state an argument?
+
+  let lsr = crystallized_state.last_state_recalc
+  assert lsr <= slot
+  assert slot < lsr + CYCLE_LENGTH * 2
+
+  result = crystallized_state.indices_for_heights[int slot - lsr]
+  # TODO, slot is an int64 will be an issue on int32 arch.
+  #       Clarify with EF if light clients will need the beacon chain
+
+func get_block_hash(beacon_block: BeaconBlock,
+        active_state: ActiveState, slot: int64): Keccak256_Digest =
+
+  # TODO: Spec uses crystallized_state as arg and activ_state.slot_number
+  #       which doesn't exist
+
+  let sback = beacon_block.slot_number - CYCLE_LENGTH * 2
+  assert sback <= slot
+  assert slot < sback + CYCLE_LENGTH * 2
+
+  result = active_state.recent_block_hashes[int slot - sback]