Merge pull request #24 from status-im/beaconstate-updates

beacon state updates

beacon_chain/beacon_node.nim

@@ -135,8 +135,8 @@ proc scheduleCycleActions(node: BeaconNode) =
     # Schedule block proposals
     let
       slot = cycleStart + i
-      proposerIdx = get_beacon_proposer_idx(node.beaconState, slot)
+      proposerIdx = get_beacon_proposer_index(node.beaconState, slot.uint64)
-      attachedValidator = node.getAttachedValidator(proposerIdx)
+      attachedValidator = node.getAttachedValidator(proposerIdx.int)
 
     if attachedValidator != nil:
       # TODO:
@@ -148,7 +148,7 @@ proc scheduleCycleActions(node: BeaconNode) =
 
     # Schedule attestations
     let
-      committeesIdx = get_shard_and_committees_idx(node.beaconState, slot)
+      committeesIdx = get_shards_and_committees_index(node.beaconState, slot.uint64)
 
     for shard in node.beaconState.shard_and_committee_for_slots[committees_idx]:
       for validatorIdx in shard.committee:

beacon_chain/extras.nim

@@ -0,0 +1,24 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# Temporary dumping ground for extra types and helpers that could make it into
+# the spec potentially
+
+import
+  ./spec/[crypto, digest]
+
+const
+  BEACON_CHAIN_SHARD* = 0xffffffffffffffff'u64
+
+type
+  InitialValidator* = object
+    ## Eth1 validator registration contract output
+    pubkey*: ValidatorPubKey
+    deposit_size*: uint64
+    proof_of_possession*: seq[byte]
+    withdrawal_credentials*: Eth2Digest
+    randao_commitment*: Eth2Digest

beacon_chain/spec/beaconstate.nim

@@ -6,24 +6,92 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
+  math, sequtils,
+  ../extras,
   ./datatypes, ./digest, ./helpers, ./validator
 
 func mod_get[T](arr: openarray[T], pos: Natural): T =
   arr[pos mod arr.len]
 
-func get_shard_and_committees_idx*(state: BeaconState, slot: int): int =
+func on_startup*(initial_validator_entries: openArray[InitialValidator],
-  # This replaces `get_shards_and_committees_for_slot` from the spec
+                 genesis_time: int,
-  # since in Nim, it's not currently efficient to create read-only
+                 processed_pow_receipt_root: Eth2Digest): BeaconState =
-  # accessors to expensive-to-copy members (such as sequences).
+  ## BeaconState constructor
-  let earliest_slot_in_array = state.last_state_recalculation_slot.int - CYCLE_LENGTH
+  ##
+  ## Before the beacon chain starts, validators will register in the Eth1 chain
+  ## and deposit ETH. When enough many validators have registered, a
+  ## `ChainStart` log will be emitted and the beacon chain can start beaconing.
+  ##
+  ## Because the state root hash is part of the genesis block, the beacon state
+  ## must be calculated before creating the genesis block.
+  #
+  # Induct validators
+  # Not in spec: the system doesn't work unless there are at least CYCLE_LENGTH
+  # validators - there needs to be at least one member in each committee -
+  # good to know for testing, though arguably the system is not that useful at
+  # at that point :)
+  assert initial_validator_entries.len >= CYCLE_LENGTH
+
+  var validators: seq[ValidatorRecord]
+
+  for v in initial_validator_entries:
+    validators = get_new_validators(
+        validators,
+        ForkData(
+                pre_fork_version: INITIAL_FORK_VERSION,
+                post_fork_version: INITIAL_FORK_VERSION,
+                fork_slot_number: 0xffffffffffffffff'u64
+            ),
+        v.pubkey,
+        v.deposit_size,
+        v.proof_of_possession,
+        v.withdrawal_credentials,
+        v.randao_commitment,
+        ACTIVE,
+        0
+      ).validators
+  # Setup state
+  let
+    x = get_new_shuffling(Eth2Digest(), validators, 0)
+
+  # x + x in spec, but more ugly
+  var tmp: array[2 * CYCLE_LENGTH, seq[ShardAndCommittee]]
+  for i, n in x:
+    tmp[i] = n
+    tmp[CYCLE_LENGTH + i] = n
+
+  # The spec says to use validators, but it's actually indices..
+  let validator_indices = get_active_validator_indices(validators)
+
+  BeaconState(
+      validators: validators,
+      shard_and_committee_for_slots: tmp,
+      persistent_committees: split(
+        shuffle(validator_indices, Eth2Digest()), SHARD_COUNT),
+      fork_data: ForkData(
+        pre_fork_version: INITIAL_FORK_VERSION,
+        post_fork_version: INITIAL_FORK_VERSION
+      )
+  )
+
+func get_shards_and_committees_index*(state: BeaconState, slot: uint64): uint64 =
+  # TODO spec unsigned-unsafe here
+  let earliest_slot_in_array =
+    if state.last_state_recalculation_slot > CYCLE_LENGTH.uint64:
+      state.last_state_recalculation_slot - CYCLE_LENGTH
+    else:
+      0
+
   doAssert earliest_slot_in_array <= slot and
            slot < earliest_slot_in_array + CYCLE_LENGTH * 2
-  return int(slot - earliest_slot_in_array)
+  slot - earliest_slot_in_array
 
-proc get_shards_and_committees_for_slot*(state: BeaconState, slot: int): seq[ShardAndCommittee] =
+proc get_shards_and_committees_for_slot*(
-  return state.shard_and_committee_for_slots[state.get_shard_and_committees_idx(slot)]
+    state: BeaconState, slot: uint64): seq[ShardAndCommittee] =
+  let index = state.get_shards_and_committees_index(slot)
+  state.shard_and_committee_for_slots[index]
 
-func get_beacon_proposer_idx*(state: BeaconState, slot: int): int =
+func get_beacon_proposer_index*(state: BeaconState, slot: uint64): uint64 =
   ## From Casper RPJ mini-spec:
   ## When slot i begins, validator Vidx is expected
   ## to create ("propose") a block, which contains a pointer to some parent block
@@ -33,19 +101,49 @@ func get_beacon_proposer_idx*(state: BeaconState, slot: int): int =
   ##
   ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
 
-  # This replaces `get_beacon_proposer` from the spec since in Nim,
+  let idx = get_shards_and_committees_index(state, slot)
-  # it's not currently efficient to create read-only accessors to
+  state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot)
-  # expensive-to-copy members (such as ValidatorRecord).
-
-  let idx = get_shard_and_committees_idx(state, slot)
-  return state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot)
 
 func get_block_hash*(state: BeaconState,
                      current_block: BeaconBlock,
                      slot: int): Eth2Digest =
-  let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
+  let earliest_slot_in_array =
+    current_block.slot.int - state.recent_block_hashes.len
   assert earliest_slot_in_array <= slot
   assert slot < current_block.slot.int
 
-  return state.recent_block_hashes[slot - earliest_slot_in_array]
+  state.recent_block_hashes[slot - earliest_slot_in_array]
 
+func append_to_recent_block_hashes*(old_block_hashes: seq[Eth2Digest],
+                                    parent_slot, current_slot: uint64,
+                                    parent_hash: Eth2Digest): seq[Eth2Digest] =
+  let d = current_slot - parent_slot
+  result = old_block_hashes
+  result.add repeat(parent_hash, d)
+
+proc get_attestation_participants*(state: BeaconState,
+                                   attestation_data: AttestationSignedData,
+                                   attester_bitfield: seq[byte]): seq[int] =
+  ## Attestation participants in the attestation data are called out in a
+  ## bit field that corresponds to the committee of the shard at the time - this
+  ## function converts it to list of indices in to BeaconState.validators
+  ## Returns empty list if the shard is not found
+  # TODO Linear search through shard list? borderline ok, it's a small list
+  # TODO bitfield type needed, once bit order settles down
+  # TODO iterator candidate
+  let
+    sncs_for_slot = get_shards_and_committees_for_slot(
+      state, attestation_data.slot)
+
+  for snc in sncs_for_slot:
+    if snc.shard != attestation_data.shard:
+      continue
+
+    # TODO investigate functional library / approach to help avoid loop bugs
+    assert len(attester_bitfield) == ceil_div8(len(snc.committee))
+    for i, vindex in snc.committee:
+      let
+        bit = (attester_bitfield[i div 8] shr (7 - (i mod 8))) mod 2
+      if bit == 1:
+          result.add(vindex)
+    return # found the shard, we're done

beacon_chain/spec/datatypes.nim

@@ -17,6 +17,11 @@
 #
 # How wrong the code is:
 # https://github.com/ethereum/eth2.0-specs/compare/126a7abfa86448091a0e037f52966b6a9531a857...master
+#
+# These datatypes are used as specifications for serialization - thus should not
+# be altered outside of what the spec says. Likewise, they should not be made
+# `ref` - this can be achieved by wrapping them in higher-level
+# types / composition
 
 import
   intsets, eth_common, math,
@@ -99,7 +104,7 @@ type
   BeaconState* = object
     validator_set_change_slot*: uint64                     # Slot of last validator set change
     validators*: seq[ValidatorRecord]                      # List of validators
-    crosslinks*: seq[CrosslinkRecord]                      # Most recent crosslink for each shard
+    crosslinks*: array[SHARD_COUNT, CrosslinkRecord]       # Most recent crosslink for each shard
     last_state_recalculation_slot*: uint64                 # Last cycle-boundary state recalculation
     last_finalized_slot*: uint64                           # Last finalized slot
     justification_source*: uint64                          # Justification source
@@ -119,7 +124,7 @@ type
     candidate_pow_receipt_roots*: seq[CandidatePoWReceiptRootRecord] #
     fork_data*: ForkData                                   # Parameters relevant to hard forks / versioning.
                                                            # Should be updated only by hard forks.
-    pending_attestations*: seq[AttestationRecord]          # Attestations not yet processed
+    pending_attestations*: seq[ProcessedAttestation]       # Attestations not yet processed
     recent_block_hashes*: seq[Eth2Digest]                  # recent beacon block hashes needed to process attestations, older to newer
     randao_mix*: Eth2Digest                                # RANDAO state
 

beacon_chain/spec/helpers.nim

@@ -76,3 +76,10 @@ func get_new_recent_block_hashes*(old_block_hashes: seq[Eth2Digest],
   for _ in 0 ..< min(d, old_block_hashes.len):
     result.add parent_hash
 
+func ceil_div8*(v: int): int = (v + 7) div 8 # TODO use a proper bitarray!
+
+func repeat_hash*(v: Eth2Digest, n: SomeInteger): Eth2Digest =
+  if n == 0:
+    v
+  else:
+    repeat_hash(eth2hash(v.data), n - 1)

beacon_chain/spec/validator.nim

@@ -16,22 +16,41 @@ func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64)
       if v.status == WITHDRAWN and v.last_status_change_slot + DELETION_PERIOD.uint64 <= current_slot:
           return some(i)
 
-func add_validator*(validators: var seq[ValidatorRecord],
+func get_new_validators*(current_validators: seq[ValidatorRecord],
-                    pubkey: ValidatorPubKey,
+                         fork_data: ForkData,
-                    proof_of_possession: seq[byte],
+                         pubkey: ValidatorPubKey,
-                    withdrawal_credentials: Eth2Digest,
+                         deposit_size: uint64,
-                    randao_commitment: Eth2Digest,
+                         proof_of_possession: seq[byte],
-                    status: ValidatorStatusCodes,
+                         withdrawal_credentials: Eth2Digest,
-                    current_slot: uint64
+                         randao_commitment: Eth2Digest,
-                    ): int =
+                         status: ValidatorStatusCodes,
+                         current_slot: uint64
+                         ): tuple[validators: seq[ValidatorRecord], index: int] =
+  # TODO Spec candidate: inefficient API
+  #
   # Check that validator really did register
-  # let signed_message = as_bytes32(pubkey) + as_bytes2(withdrawal_shard) + withdrawal_address + randao_commitment
+  # let signed_message = signed_message = bytes32(pubkey) + withdrawal_credentials + randao_commitment
   # assert BLSVerify(pub=pubkey,
   #                  msg=hash(signed_message),
-  #                  sig=proof_of_possession)
+  #                  sig=proof_of_possession,
+  # domain=get_domain(
+  #     fork_data,
+  #     current_slot,
+  #     DOMAIN_DEPOSIT
+  # ))
 
-  # Pubkey uniqueness
+  var new_validators = current_validators
-  # assert pubkey not in [v.pubkey for v in validators]
+
+  for index, val in new_validators.mpairs():
+    if val.pubkey == pubkey:
+      # assert deposit_size >= MIN_TOPUP_SIZE
+      # assert val.status != WITHDRAWN
+      # assert val.withdrawal_credentials == withdrawal_credentials
+
+      val.balance.inc(deposit_size.int)
+      return (new_validators, index)
+
+  # new validator
   let
     rec = ValidatorRecord(
       pubkey: pubkey,
@@ -44,17 +63,16 @@ func add_validator*(validators: var seq[ValidatorRecord],
       exit_seq: 0
     )
 
-  let index = min_empty_validator(validators, current_slot)
+  let index = min_empty_validator(new_validators, current_slot)
   if index.isNone:
-      validators.add(rec)
+    new_validators.add(rec)
-      return len(validators) - 1
+    (new_validators, len(new_validators) - 1)
   else:
-      validators[index.get()] = rec
+    new_validators[index.get()] = rec
-      return index.get()
+    (new_validators, index.get())
 
-func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] =
+func get_active_validator_indices*(validators: openArray[ValidatorRecord]): seq[Uint24] =
   ## Select the active validators
-  result = @[]
   for idx, val in validators:
     if val.status == ACTIVE:
       result.add idx.Uint24
@@ -86,7 +104,8 @@ func get_new_shuffling*(seed: Eth2Digest,
 
     var committees = newSeq[ShardAndCommittee](shard_indices.len)
     for shard_position, indices in shard_indices:
-      committees[shard_position].shard = (shard_id_start + shard_position).uint16 mod SHARD_COUNT
+      committees[shard_position].shard =
+        uint64(shard_id_start + shard_position) mod SHARD_COUNT
       committees[shard_position].committee = indices
 
     result[slot] = committees

beacon_chain/ssz.nim

@@ -43,9 +43,9 @@ func toBytesSSZ(x: Eth2Digest): array[32, byte] = x.data
 func fromBytesSSZUnsafe(T: typedesc[SomeInteger], data: ptr byte): T =
   ## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
   ## All integers are serialized as **big endian**.
-  ## XXX: Assumes data points to a sufficiently large buffer
+  ## TODO: Assumes data points to a sufficiently large buffer
 
-  # XXX: any better way to get a suitably aligned buffer in nim???
+  # TODO: any better way to get a suitably aligned buffer in nim???
   # see also: https://github.com/nim-lang/Nim/issues/9206
   var tmp: uint64
   var alignedBuf = cast[ptr byte](tmp.addr)
@@ -115,7 +115,7 @@ func deserialize(data: ptr byte, pos: var int, len: int, typ: typedesc[object]):
 func deserialize*(
       data: seq[byte or uint8] or openarray[byte or uint8] or string,
       typ: typedesc[object]): auto {.inline.} =
-  # XXX: returns Option[typ]: https://github.com/nim-lang/Nim/issues/9195
+  # TODO: returns Option[typ]: https://github.com/nim-lang/Nim/issues/9195
   var pos = 0
   return deserialize((ptr byte)(data[0].unsafeAddr), pos, data.len, typ)
 
@@ -140,7 +140,7 @@ const CHUNK_SIZE = 128
 
 # ################### Hashing helpers ###################################
 
-# XXX varargs openarray, anyone?
+# TODO varargs openarray, anyone?
 template withHash(body: untyped): array[32, byte] =
   let tmp = withEth2Hash: body
   toBytesSSZ tmp
@@ -154,9 +154,10 @@ func hash(a, b: openArray[byte]): array[32, byte] =
     h.update(a)
     h.update(b)
 
-# XXX: er, how is this _actually_ done?
+# TODO: er, how is this _actually_ done?
+# Mandatory bug: https://github.com/nim-lang/Nim/issues/9825
 func empty(T: typedesc): T = discard
-const emptyChunk = @(empty(array[CHUNK_SIZE, byte]))
+const emptyChunk = empty(array[CHUNK_SIZE, byte])
 
 func merkleHash[T](lst: seq[T]): array[32, byte]
 
@@ -186,13 +187,13 @@ func hashSSZ*(x: openArray[byte]): array[32, byte] =
 
 func hashSSZ*(x: ValidatorRecord): array[32, byte] =
   ## Containers have their fields recursively hashed, concatenated and hashed
-  # XXX hash_ssz.py code contains special cases for some types, why?
+  # TODO hash_ssz.py code contains special cases for some types, why?
   withHash:
-    # tmp.add(x.pubkey) # XXX uncertain future of public key format
+    # tmp.add(x.pubkey) # TODO uncertain future of public key format
     h.update hashSSZ(x.withdrawal_credentials)
     h.update hashSSZ(x.randao_skips)
     h.update hashSSZ(x.balance)
-    # h.update hashSSZ(x.status) # XXX it's an enum, deal with it
+    # h.update hashSSZ(x.status) # TODO it's an enum, deal with it
     h.update hashSSZ(x.last_status_change_slot)
     h.update hashSSZ(x.exit_seq)
 
@@ -207,7 +208,7 @@ func hashSSZ*[T: not enum](x: T): array[32, byte] =
     merkleHash(x)
   else:
     ## Containers have their fields recursively hashed, concatenated and hashed
-    # XXX could probaby compile-time-macro-sort fields...
+    # TODO could probaby compile-time-macro-sort fields...
     var fields: seq[tuple[name: string, value: seq[byte]]]
     for name, field in x.fieldPairs:
       fields.add (name, @(hashSSZ(field)))
@@ -262,7 +263,7 @@ func hashSSZ*(x: BeaconBlock): array[32, byte] =
 func merkleHash[T](lst: seq[T]): array[32, byte] =
   ## Merkle tree hash of a list of homogenous, non-empty items
 
-  # XXX: the heap allocations here can be avoided by computing the merkle tree
+  # TODO: the heap allocations here can be avoided by computing the merkle tree
   #      recursively, but for now keep things simple and aligned with upstream
 
   # Store length of list (to compensate for non-bijectiveness of padding)
@@ -274,7 +275,7 @@ func merkleHash[T](lst: seq[T]): array[32, byte] =
   var chunkz: seq[seq[byte]]
 
   if len(lst) == 0:
-    chunkz.add emptyChunk
+    chunkz.add @emptyChunk
   elif sizeof(hashSSZ(lst[0])) < CHUNK_SIZE:
     # See how many items fit in a chunk
     let itemsPerChunk = CHUNK_SIZE div sizeof(hashSSZ(lst[0]))
@@ -284,6 +285,9 @@ func merkleHash[T](lst: seq[T]): array[32, byte] =
     # Build a list of chunks based on the number of items in the chunk
     for i in 0..<chunkz.len:
       for j in 0..<itemsPerChunk:
+        if i == chunkz.len - 1:
+          let idx = i * itemsPerChunk + j
+          if idx >= lst.len: break # Last chunk may be partial!
         chunkz[i].add hashSSZ(lst[i * itemsPerChunk + j])
   else:
     # Leave large items alone
@@ -293,7 +297,7 @@ func merkleHash[T](lst: seq[T]): array[32, byte] =
 
   while chunkz.len() > 1:
     if chunkz.len() mod 2 == 1:
-      chunkz.add emptyChunk
+      chunkz.add @emptyChunk
     for i in 0..<(chunkz.len div 2):
       # As tradition dictates - one feature, at least one nim bug:
       # https://github.com/nim-lang/Nim/issues/9684

beacon_chain/state_transition.nim

@@ -5,106 +5,142 @@
 #   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-
+# A imcomplete implementation of the state transition function, as described
-# Note: this is also inspired by https://github.com/ethereum/beacon_chain/blob/master/beacon_chain/state/state_transition.py
+# under "Per-block processing" in https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md
-# The official spec at https://notes.ethereum.org/SCIg8AH5SA-O4C1G1LYZHQ# is not fully
-# defining the state transitions.
 #
-# Note that the ethresearch impl is using "block_vote_cache" field, which is a dictionary mapping hashes
+# The code is here mainly to verify the data types and get an idea about
-# to the following sub-dictionary:
+# missing pieces - needs testing throughout
-#     {
-#       'voter_indices': set(),
-#       'total_voter_deposits': 0
-#     }
-# It should not be needed anymore with the new AttestationRecord type
-
-{.warning: "The official spec at https://notes.ethereum.org/SCIg8AH5SA-O4C1G1LYZHQ# is not fully defining state transitions.".}
 
 import
+  options,
+  ./extras,
   ./spec/[beaconstate, crypto, datatypes, digest, helpers],
-  intsets, endians, nimcrypto,
+  ./ssz,
   milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master
 
-func process_block*(active_state: BeaconState, crystallized_state: BeaconState, blck: BeaconBlock, slot: uint64) =
+# TODO there's an ugly mix of functional and procedural styles here that
-  # TODO: non-attestation verification parts of per-block processing
+#      is due to how the spec is mixed as well - once we're past the prototype
+#      stage, this will need clearing up and unification.
 
-  #let parent_hash = blck.ancestor_hashes[0]
+func checkAttestations(state: BeaconState,
-  # TODO actually get parent block, which means fixing up BeaconState refs above;
+                       blck: BeaconBlock,
-  # there's no distinction between active/crystallized state anymore, etc.
+                       parent_slot: uint64): Option[seq[ProcessedAttestation]] =
-  let parent_slot = 0'u64
+  # TODO perf improvement potential..
+  if blck.attestations.len > MAX_ATTESTATION_COUNT:
+    return
 
+  var res: seq[ProcessedAttestation]
   for attestation in blck.attestations:
-    doAssert attestation.data.slot <= blck.slot - MIN_ATTESTATION_INCLUSION_DELAY
+    if attestation.data.slot <= blck.slot - MIN_ATTESTATION_INCLUSION_DELAY:
-    doAssert attestation.data.slot >= max(parent_slot - CYCLE_LENGTH + 1, 0)
+      return
+    if attestation.data.slot >= max(parent_slot - CYCLE_LENGTH + 1, 0):
+      return
     #doAssert attestation.data.justified_slot == justification_source if attestation.data.slot >= state.last_state_recalculation_slot else prev_cycle_justification_source
     # doAssert attestation.data.justified_block_hash == get_block_hash(state, block, attestation.data.justified_slot).
     # doAssert either attestation.data.last_crosslink_hash or attestation.data.shard_block_hash equals state.crosslinks[shard].shard_block_hash.
 
-    # Let attestation_indices be get_shards_and_committees_for_slot(crystallized_state, slot)[x], choosing x so that attestation_indices.shard_id equals the shard_id value provided to find the set of validators that is creating this attestation record.
+    let attestation_participants = get_attestation_participants(
-    let attestation_indices = block:
+      state, attestation.data, attestation.attester_bitfield)
-      let shard_and_committees = get_shards_and_committees_for_slot(crystallized_state, slot.int)
-      var
-        x = 1
-        record_creator = shard_and_committees[0]
-      while record_creator.shard != attestation.data.shard:
-        record_creator = shard_and_committees[x]
-        inc x
-      record_creator
 
-    # TODO: Verify that len(attester_bitfield) == ceil_div8(len(attestation_indices)), where ceil_div8 = (x + 7) // 8. Verify that bits len(attestation_indices).... and higher, if present (i.e. len(attestation_indices) is not a multiple of 8), are all zero
+    var
+      agg_pubkey: ValidatorPubKey
+      empty = true
 
-    # Derive a group public key by adding the public keys of all of the attesters in attestation_indices for whom the corresponding bit in attester_bitfield (the ith bit is (attester_bitfield[i // 8] >> (7 - (i %8))) % 2) equals 1
+    for attester_idx in attestation_participants:
-    var agg_pubkey: ValidatorPubKey
+      let validator = state.validators[attester_idx]
-    var empty = true
+      if empty:
-    for attester_idx in attestation_indices.committee:
+        agg_pubkey = validator.pubkey
-      # TODO re-enable, but currently this whole function's a nonfunctional stub
+        empty = false
-      # because state's lacking.
+      else:
-      #if attester_idx in attestation.attester_bitfield:
+        agg_pubkey.combine(validator.pubkey)
-        let validator = crystallized_state.validators[attester_idx]
-        if empty:
-          agg_pubkey = validator.pubkey
-          empty = false
-        else:
-          agg_pubkey.combine(validator.pubkey)
 
-    # Verify that aggregate_sig verifies using the group pubkey generated and hash((slot % CYCLE_LENGTH).to_bytes(8, 'big') + parent_hashes + shard_id + shard_block_hash) as the message.
+    # Verify that aggregate_sig verifies using the group pubkey.
-    var msg: array[32, byte]
+    let msg = hashSSZ(attestation.data)
-
-    block:
-      var ctx: blake2_512 # Context for streaming blake2b computation
-      ctx.init()
-
-      var be_slot: array[8, byte]
-      bigEndian64(be_slot[0].addr, attestation.data.slot.unsafeAddr)
-      ctx.update be_slot
-
-      # TODO: re-enable these, but parent_hashes isn't in new spec, so
-      # this needs more substantial adjustment.
-      # let size_p_hashes = uint parent_hashes.len * sizeof(Eth2Digest)
-      # ctx.update(cast[ptr byte](parent_hashes[0].addr), size_p_hashes)
-
-      var be_shard_id: array[2, byte]           # Unsure, spec doesn't mention big-endian representation
-      bigEndian16(be_shard_id.addr, attestation.data.shard.unsafeAddr)
-      ctx.update be_shard_id
-
-      ctx.update attestation.data.shard_block_hash.data
-
-      var be_justified_slot: array[8, byte]
-      bigEndian64(be_justified_slot[0].addr, attestation.data.justified_slot.unsafeAddr)
-      ctx.update be_justified_slot
-
-      let h = ctx.finish()                      # Full hash (Blake2b-512)
-      msg[0 ..< 32] = h.data.toOpenArray(0, 32) # Keep only the first 32 bytes - https://github.com/ethereum/beacon_chain/issues/60
-
-      ctx.clear()                               # Cleanup context/memory
 
     # For now only check compilation
     # doAssert attestation.aggregate_sig.verifyMessage(msg, agg_pubkey)
-    debugEcho "Aggregate sig verify message: ", attestation.aggregate_sig.verifyMessage(msg, agg_pubkey)
+    debugEcho "Aggregate sig verify message: ",
+      attestation.aggregate_sig.verifyMessage(msg, agg_pubkey)
 
-  # Extend the list of AttestationRecord objects in the active_state, ordering the new additions in the same order as they came in the block.
+    res.add ProcessedAttestation(
-  # TODO
+      data: attestation.data,
+      attester_bitfield: attestation.attester_bitfield,
+      poc_bitfield: attestation.poc_bitfield,
+      slot_included: blck.slot
+    )
 
-  # Verify that the slot % len(get_indices_for_slot(crystallized_state, slot-1)[0])'th attester in get_indices_for_slot(crystallized_state, slot-1)[0]is part of at least one of the AttestationRecord objects; this attester can be considered to be the proposer of the block.
+  some(res)
-  # TODO
 
+func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
+  var blck_without_sig = blck
+  blck_without_sig.proposer_signature = ValidatorSig()
+
+  let
+    proposal_hash = hashSSZ(ProposalSignedData(
+      slot: blck.slot,
+      shard: BEACON_CHAIN_SHARD,
+      block_hash: Eth2Digest(data: hashSSZ(blck_without_sig))
+    ))
+
+  verifyMessage(
+    blck.proposer_signature, proposal_hash,
+    state.validators[get_beacon_proposer_index(state, blck.slot).int].pubkey)
+
+func processRandaoReveal(state: var BeaconState,
+                         blck: BeaconBlock,
+                         parent_slot: uint64): bool =
+  # Update randao skips
+  for slot in parentslot + 1 ..< blck.slot:
+    let proposer_index = get_beacon_proposer_index(state, slot)
+    state.validators[proposer_index.int].randao_skips.inc()
+
+  var
+    proposer_index = get_beacon_proposer_index(state, blck.slot)
+    proposer = state.validators[proposer_index.int]
+
+  # Check that proposer commit and reveal match
+  if repeat_hash(blck.randao_reveal, proposer.randao_skips + 1) !=
+      proposer.randao_commitment:
+    return
+
+  # Update state and proposer now that we're alright
+  for i, b in state.randao_mix.data:
+    state.randao_mix.data[i] = b xor blck.randao_reveal.data[i]
+
+  proposer.randao_commitment = blck.randao_reveal
+  proposer.randao_skips = 0
+
+  true
+
+func process_block*(state: BeaconState, blck: BeaconBlock): Option[BeaconState] =
+  ## When a new block is received, all participants must verify that the block
+  ## makes sense and update their state accordingly. This function will return
+  ## the new state, unless something breaks along the way
+
+  # TODO: simplistic way to be able to rollback state
+  var state = state
+
+  let
+    parent_hash = blck.ancestor_hashes[0]
+    slot = blck.slot
+    parent_slot = slot - 1 # TODO Not!! can skip slots...
+  # TODO actually get parent block, which means fixing up BeaconState refs above;
+  # there's no distinction between active/crystallized state anymore, etc.
+
+  state.recent_block_hashes =
+    append_to_recent_block_hashes(state.recent_block_hashes, parent_slot, slot,
+      parent_hash)
+
+  let processed_attestations = checkAttestations(state, blck, parent_slot)
+  if processed_attestations.isNone:
+    return
+
+  state.pending_attestations.add processed_attestations.get()
+
+  if not verifyProposerSignature(state, blck):
+    return
+
+  if not processRandaoReveal(state, blck, parent_slot):
+    return
+
+  some(state) # Looks ok - move on with the updated state

tests/all_tests.nim

@@ -6,6 +6,7 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
+  ./test_beaconstate,
   ./test_block_processing,
   ./test_ssz,
   ./test_validator

tests/test_beaconstate.nim

@@ -0,0 +1,19 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  sequtils, unittest,
+  ./testhelpers,
+  ../beacon_chain/extras,
+  ../beacon_chain/spec/[beaconstate, datatypes, digest]
+
+suite "Beacon state":
+  # Smoke test
+
+  test "Smoke on_startup":
+    let state = on_startup(makeInitialValidators(CYCLE_LENGTH), 0, Eth2Digest())
+    check: state.validators.len == CYCLE_LENGTH

tests/test_block_processing.nim

@@ -6,17 +6,21 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  unittest,
+  options, sequtils, unittest,
-  ../beacon_chain/spec/datatypes,
+  ./testhelpers,
-  ../beacon_chain/state_transition
+  ../beacon_chain/spec/[beaconstate, datatypes, digest],
+  ../beacon_chain/[extras, state_transition]
 
 suite "Block processing":
   ## For now just test that we can compile and execute block processing with mock data.
 
   test "Mock process_block":
-    let actState = BeaconState()
+    let
-    let crystState = BeaconState()
+      state = on_startup(makeInitialValidators(), 0, Eth2Digest())
-    let blck = BeaconBlock()
+      blck = BeaconBlock(
-    let slot = 10'u
+        slot: 1,
-
+        ancestor_hashes: @[Eth2Digest()]
-    actState.process_block(crystState, blck, slot)
+      )
+      newState = process_block(state, blck)
+    check:
+      newState.isNone() # Broken block, should fail processing

tests/test_ssz.nim

@@ -60,7 +60,7 @@ suite "Simple serialization":
       expected_ser[1..^1].deserialize(Foo).isNone()
 
 suite "Tree hashing":
-  # XXX Nothing but smoke tests for now..
+  # TODO Nothing but smoke tests for now..
 
   test "Hash ValidatorRecord":
     let vr = ValidatorRecord()

tests/test_validator.nim

@@ -22,10 +22,14 @@ suite "Validators":
       validators = repeat(
         ValidatorRecord(
           status: ACTIVE
-        ), 1024)
+        ), 32*1024)
 
-    # XXX the shuffling looks really odd, probably buggy
+    # TODO the shuffling looks really odd, probably buggy
     let s = get_new_shuffling(Eth2Digest(), validators, 0)
     check:
       s.len == CYCLE_LENGTH
+       # 32k validators means 2 shards validated per slot - the aim is to get
+       # TARGET_COMMITTEE_SIZE validators in each shard and there are
+       # CYCLE_LENGTH slots which each will crosslink a different shard
+      s[0].len == 32 * 1024 div (TARGET_COMMITTEE_SIZE * CYCLE_LENGTH)
       sumCommittees(s) == validators.len() # all validators accounted for

tests/testhelpers.nim

@@ -0,0 +1,19 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  ../beacon_chain/extras,
+  ../beacon_chain/spec/[crypto, datatypes]
+
+func makeValidatorPubKey(n: int): ValidatorPubKey =
+  result.point.x.a.g[0] = n
+
+func makeInitialValidators*(n = CYCLE_LENGTH): seq[InitialValidator] =
+  for i in 0..<n:
+    result.add InitialValidator(
+      pubkey: makeValidatorPubKey(i)
+    )