spec updates (#37)

* use repeat_hash from spec in randao
* add some logging to state processing
* export crypto string converters in spec insulation layer
* enable block signature verification
* ssz: add support for arrays, remove custom data type code
* correctly handle previous and new block
* add trivial block processing tests
* prefer iterative repeat_hash
* state transition mostly done
* handle most specials and slashings

beacon_chain/extras.nim

@@ -8,17 +8,4 @@
 # Temporary dumping ground for extra types and helpers that could make it into
 # the spec potentially
 
-import
+# TODO Woohoo, nothing for now!
-  ./spec/[crypto, digest]
-
-const
-  BEACON_CHAIN_SHARD* = 0xffffffffffffffff'u64
-
-type
-  InitialValidator* = object
-    ## Eth1 validator registration contract output
-    pubkey*: ValidatorPubKey
-    deposit_size*: uint64
-    proof_of_possession*: ValidatorSig
-    withdrawal_credentials*: Eth2Digest
-    randao_commitment*: Eth2Digest

beacon_chain/randao.nim

@@ -1,4 +1,4 @@
-import spec/digest
+import spec/[digest, helpers]
 
 type Randao* = object
   seed: Eth2Digest
@@ -15,13 +15,6 @@ proc initRandao*(bytes: openarray[byte]): Randao =
   s.data[0 .. ^1] = bytes
   initRandao(bytes)
 
-func repeatHash*(h: Eth2Digest, n: int): Eth2Digest =
-  result = h
-  var n = n
-  while n != 0:
-    result = eth2hash(result.data)
-    dec n
-
 proc initialCommitment*(r: Randao): Eth2Digest =
   repeatHash(r.seed, MaxRandaoLevels)
 

beacon_chain/spec/beaconstate.nim

@@ -6,11 +6,145 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  math, sequtils,
+  chronicles, math, options, sequtils,
   ../extras, ../ssz,
   ./crypto, ./datatypes, ./digest, ./helpers, ./validator
 
-func on_startup*(initial_validator_entries: openArray[InitialValidator],
+func process_deposit(state: var BeaconState,
+                     pubkey: ValidatorPubKey,
+                     deposit: uint64,
+                     proof_of_possession: ValidatorSig,
+                     withdrawal_credentials: Eth2Digest,
+                     randao_commitment: Eth2Digest): Uint24 =
+  ## Process a deposit from Ethereum 1.0.
+  let msg = hash_tree_root((pubkey, withdrawal_credentials, randao_commitment))
+  assert BLSVerify(
+    pubkey, msg, proof_of_possession,
+    get_domain(state.fork_data, state.slot, DOMAIN_DEPOSIT))
+
+  let validator_pubkeys = mapIt(state.validator_registry, it.pubkey)
+
+  if pubkey notin validator_pubkeys:
+    # Add new validator
+    let validator = ValidatorRecord(
+      pubkey: pubkey,
+      withdrawal_credentials: withdrawal_credentials,
+      randao_commitment: randao_commitment,
+      randao_layers: 0,
+      balance: deposit,
+      status: PENDING_ACTIVATION,
+      latest_status_change_slot: state.slot,
+      exit_count: 0
+    )
+
+    let index = min_empty_validator_index(state.validator_registry, state.slot)
+    if index.isNone():
+      state.validator_registry.add(validator)
+      (len(state.validator_registry) - 1).Uint24
+    else:
+      state.validator_registry[index.get()] = validator
+      index.get().Uint24
+  else:
+    # Increase balance by deposit
+    let index = validator_pubkeys.find(pubkey)
+    let validator = addr state.validator_registry[index]
+    assert validator.withdrawal_credentials == withdrawal_credentials
+
+    validator.balance += deposit
+    index.Uint24
+
+func activate_validator(state: var BeaconState,
+                        index: Uint24) =
+  ## Activate the validator with the given ``index``.
+  let validator = addr state.validator_registry[index]
+
+  if validator.status != PENDING_ACTIVATION:
+      return
+
+  validator.status = ACTIVE
+  validator.latest_status_change_slot = state.slot
+  state.validator_registry_delta_chain_tip =
+    get_new_validator_registry_delta_chain_tip(
+      state.validator_registry_delta_chain_tip,
+      index,
+      validator.pubkey,
+      ACTIVATION,
+    )
+
+func initiate_validator_exit(state: var BeaconState,
+                             index: Uint24) =
+  ## Initiate exit for the validator with the given ``index``.
+  let validator = addr state.validator_registry[index]
+  if validator.status != ACTIVE:
+    return
+
+  validator.status = ACTIVE_PENDING_EXIT
+  validator.latest_status_change_slot = state.slot
+
+func exit_validator(state: var BeaconState,
+                     index: Uint24,
+                     new_status: ValidatorStatusCodes) =
+  ## Exit the validator with the given ``index``.
+  ## Note that this function mutates ``state``.
+
+  let
+    validator = addr state.validator_registry[index]
+    prev_status = validator.status
+
+  if prev_status == EXITED_WITH_PENALTY:
+    return
+
+  validator.status = new_status
+  validator.latest_status_change_slot = state.slot
+
+  if new_status == EXITED_WITH_PENALTY:
+    state.latest_penalized_exit_balances[
+      (state.slot div COLLECTIVE_PENALTY_CALCULATION_PERIOD).int] +=
+        get_effective_balance(validator[])
+
+    let
+      whistleblower = addr state.validator_registry[
+        get_beacon_proposer_index(state, state.slot)]
+      whistleblower_reward =
+        validator.balance div WHISTLEBLOWER_REWARD_QUOTIENT
+
+    whistleblower.balance += whistleblower_reward
+    validator.balance -= whistleblower_reward
+
+  if prev_status == EXITED_WITHOUT_PENALTY:
+    return
+
+  # The following updates only occur if not previous exited
+  state.validator_registry_exit_count += 1
+  validator.exit_count = state.validator_registry_exit_count
+  state.validator_registry_delta_chain_tip =
+    get_new_validator_registry_delta_chain_tip(
+      state.validator_registry_delta_chain_tip,
+      index,
+      validator.pubkey,
+      ValidatorSetDeltaFlags.EXIT
+    )
+
+  # Remove validator from persistent committees
+  for committee in state.persistent_committees.mitems():
+    for i, validator_index in committee:
+      if validator_index == index:
+        committee.delete(i)
+        break
+
+func update_validator_status*(state: var BeaconState,
+                              index: Uint24,
+                              new_status: ValidatorStatusCodes) =
+  ##  Update the validator status with the given ``index`` to ``new_status``.
+  ## Handle other general accounting related to this status update.
+  if new_status == ACTIVE:
+      activate_validator(state, index)
+  if new_status == ACTIVE_PENDING_EXIT:
+      initiate_validator_exit(state, index)
+  if new_status in [EXITED_WITH_PENALTY, EXITED_WITHOUT_PENALTY]:
+      exit_validator(state, index, new_status)
+
+func on_startup*(initial_validator_deposits: openArray[Deposit],
                  genesis_time: uint64,
                  processed_pow_receipt_root: Eth2Digest): BeaconState =
   ## BeaconState constructor
@@ -21,50 +155,24 @@ func on_startup*(initial_validator_entries: openArray[InitialValidator],
   ##
   ## Because the state root hash is part of the genesis block, the beacon state
   ## must be calculated before creating the genesis block.
-  #
+
   # Induct validators
   # Not in spec: the system doesn't work unless there are at least EPOCH_LENGTH
   # validators - there needs to be at least one member in each committee -
   # good to know for testing, though arguably the system is not that useful at
   # at that point :)
-  assert initial_validator_entries.len >= EPOCH_LENGTH
+  assert initial_validator_deposits.len >= EPOCH_LENGTH
 
-  var validators: seq[ValidatorRecord]
+  var state = BeaconState(
+    # Misc
+    slot: INITIAL_SLOT_NUMBER,
+    genesis_time: genesis_time,
+    fork_data: ForkData(
+        pre_fork_version: INITIAL_FORK_VERSION,
+        post_fork_version: INITIAL_FORK_VERSION,
+        fork_slot: INITIAL_SLOT_NUMBER,
+    ),
 
-  for v in initial_validator_entries:
-    validators = get_new_validators(
-        validators,
-        ForkData(
-                pre_fork_version: INITIAL_FORK_VERSION,
-                post_fork_version: INITIAL_FORK_VERSION,
-                fork_slot: INITIAL_SLOT_NUMBER
-            ),
-        v.pubkey,
-        v.deposit_size,
-        v.proof_of_possession,
-        v.withdrawal_credentials,
-        v.randao_commitment,
-        ACTIVE,
-        INITIAL_SLOT_NUMBER
-      ).validators
-  # Setup state
-  let
-    initial_shuffling = get_new_shuffling(Eth2Digest(), validators, 0)
-
-  # initial_shuffling + initial_shuffling in spec, but more ugly
-  var shard_committees_at_slots: array[2 * EPOCH_LENGTH, seq[ShardCommittee]]
-  for i, n in initial_shuffling:
-    shard_committees_at_slots[i] = n
-    shard_committees_at_slots[EPOCH_LENGTH + i] = n
-
-  # TODO validators vs indices
-  let active_validator_indices = get_active_validator_indices(validators)
-
-  let persistent_committees = split(shuffle(
-    active_validator_indices, ZERO_HASH), SHARD_COUNT)
-
-  BeaconState(
-    validator_registry: validators,
     validator_registry_latest_change_slot: INITIAL_SLOT_NUMBER,
     validator_registry_exit_count: 0,
     validator_registry_delta_chain_tip: ZERO_HASH,
@@ -72,8 +180,6 @@ func on_startup*(initial_validator_entries: openArray[InitialValidator],
     # Randomness and committees
     randao_mix: ZERO_HASH,
     next_seed: ZERO_HASH,
-    shard_committees_at_slots: shard_committees_at_slots,
-    persistent_committees: persistent_committees,
 
     # Finality
     previous_justified_slot: INITIAL_SLOT_NUMBER,
@@ -82,19 +188,44 @@ func on_startup*(initial_validator_entries: openArray[InitialValidator],
 
     # Recent state
     latest_state_recalculation_slot: INITIAL_SLOT_NUMBER,
-    latest_block_roots: repeat(ZERO_HASH, EPOCH_LENGTH * 2),
+    latest_block_roots: repeat(ZERO_HASH, LATEST_BLOCK_ROOTS_COUNT),
 
      # PoW receipt root
     processed_pow_receipt_root: processed_pow_receipt_root,
-    # Misc
-    genesis_time: genesis_time,
-    fork_data: ForkData(
-        pre_fork_version: INITIAL_FORK_VERSION,
-        post_fork_version: INITIAL_FORK_VERSION,
-        fork_slot: INITIAL_SLOT_NUMBER,
-    ),
   )
 
+  # handle initial deposits and activations
+  for deposit in initial_validator_deposits:
+    let validator_index = process_deposit(
+      state,
+      deposit.deposit_data.deposit_parameters.pubkey,
+      deposit.deposit_data.value,
+      deposit.deposit_data.deposit_parameters.proof_of_possession,
+      deposit.deposit_data.deposit_parameters.withdrawal_credentials,
+      deposit.deposit_data.deposit_parameters.randao_commitment
+    )
+    if state.validator_registry[validator_index].balance >= MAX_DEPOSIT:
+      update_validator_status(state, validator_index, ACTIVE)
+
+  # set initial committee shuffling
+  let
+    initial_shuffling =
+      get_new_shuffling(Eth2Digest(), state.validator_registry, 0)
+
+  # initial_shuffling + initial_shuffling in spec, but more ugly
+  for i, n in initial_shuffling:
+    state.shard_committees_at_slots[i] = n
+    state.shard_committees_at_slots[EPOCH_LENGTH + i] = n
+
+  # set initial persistent shuffling
+  let active_validator_indices =
+    get_active_validator_indices(state.validator_registry)
+
+  state.persistent_committees = split(shuffle(
+    active_validator_indices, ZERO_HASH), SHARD_COUNT)
+
+  state
+
 func get_block_root*(state: BeaconState,
                      slot: uint64): Eth2Digest =
   let earliest_slot_in_array =
@@ -143,7 +274,7 @@ func process_ejections*(state: var BeaconState) =
 
   for i, v in state.validator_registry.mpairs():
     if is_active_validator(v) and v.balance < EJECTION_BALANCE:
-      exit_validator(i.Uint24, state, EXITED_WITHOUT_PENALTY)
+      exit_validator(state, i.Uint24, EXITED_WITHOUT_PENALTY)
 
 func update_validator_registry*(state: var BeaconState) =
   # Update validator registry.
@@ -159,16 +290,20 @@ func update_validator_registry*(state: var BeaconState) =
         state.slot
       )
 
-func checkAttestation*(state: BeaconState, attestation: Attestation): bool =
+proc checkAttestation*(state: BeaconState, attestation: Attestation): bool =
   ## Check that an attestation follows the rules of being included in the state
   ## at the current slot.
   ##
   ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestations-1
 
-  if attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot:
+  if attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY >= state.slot:
+    warn("Attestation too new",
+      attestation_slot = attestation.data.slot, state_slot = state.slot)
     return
 
-  if attestation.data.slot + EPOCH_LENGTH >= state.slot:
+  if attestation.data.slot + EPOCH_LENGTH <= state.slot:
+    warn("Attestation too old",
+      attestation_slot = attestation.data.slot, state_slot = state.slot)
     return
 
   let expected_justified_slot =
@@ -178,16 +313,23 @@ func checkAttestation*(state: BeaconState, attestation: Attestation): bool =
       state.previous_justified_slot
 
   if attestation.data.justified_slot != expected_justified_slot:
+    warn("Unexpected justified slot",
+      attestation_justified_slot = attestation.data.justified_slot,
+      expected_justified_slot)
     return
 
   let expected_justified_block_root =
     get_block_root(state, attestation.data.justified_slot)
   if attestation.data.justified_block_root != expected_justified_block_root:
+    warn("Unexpected justified block root",
+      attestation_justified_block_root = attestation.data.justified_block_root,
+      expected_justified_block_root)
     return
 
   if state.latest_crosslinks[attestation.data.shard].shard_block_root notin [
       attestation.data.latest_crosslink_root,
       attestation.data.shard_block_root]:
+    warn("Unexpected crosslink shard_block_root")
     return
 
   let
@@ -203,10 +345,12 @@ func checkAttestation*(state: BeaconState, attestation: Attestation): bool =
         group_public_key, @msg & @[0'u8], attestation.aggregate_signature,
         get_domain(state.fork_data, attestation.data.slot, DOMAIN_ATTESTATION)
       ):
+    warn("Invalid attestation group signature")
     return
 
   # To be removed in Phase1:
   if attestation.data.shard_block_root != ZERO_HASH:
+    warn("Invalid shard block root")
     return
 
   true

beacon_chain/spec/crypto.nim

@@ -11,6 +11,7 @@
 
 import
   milagro_crypto, hashes
+export milagro_crypto.`$`
 
 type
   ValidatorPubKey* = milagro_crypto.VerKey

beacon_chain/spec/datatypes.nim

@@ -16,7 +16,7 @@
 # https://github.com/ethereum/eth2.0-specs/blob/master/specs/beacon-chain.md
 #
 # How wrong the code is:
-# https://github.com/ethereum/eth2.0-specs/compare/f956135763cbb410a8c28b3a509f14f750ff287c...master
+# https://github.com/ethereum/eth2.0-specs/compare/8116562049ed80ad1823dd62e98a7483ddf1546c...master
 #
 # These datatypes are used as specifications for serialization - thus should not
 # be altered outside of what the spec says. Likewise, they should not be made
@@ -24,7 +24,7 @@
 # types / composition
 
 import
-  intsets, eth_common, math,
+  eth_common, math,
   ./crypto, ./digest
 
 # TODO Data types:
@@ -55,19 +55,23 @@ const
   ## Once the balance of a validator drops below this, it will be ejected from
   ## the validator pool
 
-  MAX_ATTESTATIONS_PER_BLOCK* = 2^7 # attestations
-
-  MIN_BALANCE* = 2'u64^4 ##\
-  ## Minimum balance in ETH before a validator is removed from the validator
-  ## pool
-
   MAX_BALANCE_CHURN_QUOTIENT* = 2^5 ##\
   ## At most `1/MAX_BALANCE_CHURN_QUOTIENT` of the validators can change during
   ## each validator registry change.
 
   GWEI_PER_ETH* = 10'u64^9 # Gwei/ETH
+
   BEACON_CHAIN_SHARD_NUMBER* = not 0'u64 # 2^64 - 1 in spec
 
+  BLS_WITHDRAWAL_PREFIX_BYTE* = 0'u8
+
+  MAX_CASPER_VOTES* = 2^10
+  LATEST_BLOCK_ROOTS_COUNT* = 2'u64^13
+
+  MIN_BALANCE* = 2'u64^4 ##\
+  ## Minimum balance in ETH before a validator is removed from the validator
+  ## pool
+
   DEPOSIT_CONTRACT_TREE_DEPTH* = 2^5
 
   MIN_DEPOSIT* = 2'u64^0 ##\
@@ -112,14 +116,20 @@ const
   ## slots (~291 days)
 
   # Quotients
-  BASE_REWARD_QUOTIENT* = 2'u64^11 ##\
+  BASE_REWARD_QUOTIENT* = 2'u64^10 ##\
-  ## per-cycle interest rate assuming all validators are participating, assuming
+  ## The `BASE_REWARD_QUOTIENT` parameter dictates the per-epoch reward. It
-  ## total deposits of 1 ETH. It corresponds to ~2.57% annual interest assuming
+  ## corresponds to ~2.54% annual interest assuming 10 million participating
-  ## 10 million participating ETH.
+  ## ETH in every epoch.
   WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9
   INCLUDER_REWARD_QUOTIENT* = 2'u64^3
   INACTIVITY_PENALTY_QUOTIENT* = 2'u64^34
 
+  MAX_PROPOSER_SLASHINGS* = 2^4
+  MAX_CASPER_SLASHINGS* = 2^4
+  MAX_ATTESTATIONS* = 2^7
+  MAX_DEPOSITS* = 2^4
+  MAX_EXITS* = 2^4
+
 type
   Uint24* = range[0'u32 .. 0xFFFFFF'u32] # TODO: wrap-around
 
@@ -198,7 +208,7 @@ type
     # Minimum slot for processing exit
     slot*: uint64
     # Index of the exiting validator
-    validator_index*: uint64
+    validator_index*: Uint24
     # Validator signature
     signature*: ValidatorSig
 
@@ -210,9 +220,7 @@ type
     ## is formed.
 
     slot*: uint64
-    ancestor_hashes*: seq[Eth2Digest] ##\
+    parent_root*: Eth2Digest
-    ## Skip list of previous beacon block hashes i'th item is most recent
-    ## ancestor whose slot is a multiple of 2**i for i == 0, ..., 31
 
     state_root*: Eth2Digest
 
@@ -227,9 +235,9 @@ type
     body*: BeaconBlockBody
 
   BeaconBlockBody* = object
-    attestations*: seq[Attestation]
     proposer_slashings*: seq[ProposerSlashing]
     casper_slashings*: seq[CasperSlashing]
+    attestations*: seq[Attestation]
     deposits*: seq[Deposit]
     exits*: seq[Exit]
 
@@ -274,19 +282,32 @@ type
     latest_penalized_exit_balances*: seq[uint64] ##\
     ## Balances penalized in the current withdrawal period
     latest_attestations*: seq[PendingAttestationRecord]
+    batched_block_roots*: seq[Eth2Digest]
 
     processed_pow_receipt_root*: Eth2Digest
     candidate_pow_receipt_roots*: seq[CandidatePoWReceiptRootRecord]
 
   ValidatorRecord* = object
-    pubkey*: ValidatorPubKey                      # Public key
+    pubkey*: ValidatorPubKey
-    withdrawal_credentials*: Eth2Digest           # Withdrawal credentials
+    withdrawal_credentials*: Eth2Digest
-    randao_commitment*: Eth2Digest                # RANDAO commitment
+    randao_commitment*: Eth2Digest ##\
-    randao_skips*: uint64                         # Slot the proposer has skipped (ie. layers of RANDAO expected)
+    ## RANDAO commitment created by repeatedly taking the hash of a secret value
-    balance*: uint64                              # Balance in Gwei
+    ## so as to create "onion layers" around it. For every block that a
-    status*: ValidatorStatusCodes                 # Status code
+    ## validator proposes, one level of the onion is peeled. See:
-    latest_status_change_slot*: uint64            # Slot when validator last changed status (or 0)
+    ## * https://ethresear.ch/t/rng-exploitability-analysis-assuming-pure-randao-based-main-chain/1825
-    exit_count*: uint64                           # Exit counter when validator exited (or 0)
+    ## * repeat_hash
+    ## * processRandaoReveal
+
+    randao_layers*: uint64 ##\
+    ## Slot the proposer has skipped (ie. layers of RANDAO expected)
+
+    balance*: uint64 # Balance in Gwei
+    status*: ValidatorStatusCodes
+    latest_status_change_slot*: uint64 ##\
+    ## Slot when validator last changed status (or 0)
+
+    exit_count*: uint64 ##\
+    ## Exit counter when validator exited (or 0)
 
   CrosslinkRecord* = object
     slot*: uint64                                 # Slot number
@@ -340,20 +361,6 @@ type
     DOMAIN_PROPOSAL = 2
     DOMAIN_EXIT = 3
 
-    # Note:
-    # We use IntSet from Nim Standard library which are efficient sparse bitsets.
-    # See: https://nim-lang.org/docs/intsets.html
-    #
-    # Future:
-    #   IntSets stores the first 34 elements in an array[34, int] instead of a bitfield
-    #   to avoid heap allocation in profiled common cases.
-    #
-    #   In Ethereum we probably always have over 34 attesters given the goal of decentralization.
-    #   Allocating 8 * 34 = 272 bytes on the stack is wasteful, when this can be packed in just 8 bytes
-    #   with room to spare.
-    #
-    #   Also, IntSets uses machine int size while we require int64 even on 32-bit platform.
-
 when true:
   # TODO: Remove these once RLP serialization is no longer used
   import nimcrypto, rlp

beacon_chain/spec/digest.nim

@@ -22,6 +22,8 @@
 
 import nimcrypto/[blake2, hash]
 
+export hash.`$`
+
 type
   Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
   Eth2Hash* = blake2_512 ## Context for hash function

beacon_chain/spec/helpers.nim

@@ -82,10 +82,15 @@ func get_new_recent_block_roots*(old_block_roots: seq[Eth2Digest],
 func ceil_div8*(v: int): int = (v + 7) div 8 # TODO use a proper bitarray!
 
 func repeat_hash*(v: Eth2Digest, n: SomeInteger): Eth2Digest =
-  if n == 0:
+  # Spec version:
-    v
+  # if n == 0: v
-  else:
+  # else: repeat_hash(eth2hash(v.data), n - 1)
-    repeat_hash(eth2hash(v.data), n - 1)
+  # Nim is pretty bad at recursion though (max 2k levels / no tco), so:
+  result = v
+  var n = n
+  while n != 0:
+    result = eth2hash(result.data)
+    dec n
 
 func get_shard_and_committees_index*(state: BeaconState, slot: uint64): uint64 =
   # TODO spec unsigned-unsafe here
@@ -137,10 +142,10 @@ func get_domain*(
 
 func is_power_of_2*(v: uint64): bool = (v and (v-1)) == 0
 
-func get_updated_ancestor_hashes*(latest_block: BeaconBlock,
+func merkle_root*(values: openArray[Eth2Digest]): Eth2Digest =
-                                  latest_hash: Eth2Digest): seq[Eth2Digest] =
+  # o = [0] * len(values) + values
-    var new_ancestor_hashes = latest_block.ancestor_hashes
+  # for i in range(len(values)-1, 0, -1):
-    for i in 0..<32:
+  #     o[i] = hash(o[i*2] + o[i*2+1])
-      if latest_block.slot mod 2'u64^i == 0:
+  # return o[1]
-        new_ancestor_hashes[i] = latest_hash
+  # TODO
-    new_ancestor_hashes
+  discard

beacon_chain/spec/validator.nim

@@ -15,62 +15,13 @@ import
 func is_active_validator*(validator: ValidatorRecord): bool =
   validator.status in {ACTIVE, ACTIVE_PENDING_EXIT}
 
-func min_empty_validator_index(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =
+func min_empty_validator_index*(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =
   for i, v in validators:
     if v.balance == 0 and
         v.latest_status_change_slot +
           ZERO_BALANCE_VALIDATOR_TTL.uint64 <= current_slot:
       return some(i)
 
-func get_new_validators*(current_validators: seq[ValidatorRecord],
-                         fork_data: ForkData,
-                         pubkey: ValidatorPubKey,
-                         deposit: uint64,
-                         proof_of_possession: ValidatorSig,
-                         withdrawal_credentials: Eth2Digest,
-                         randao_commitment: Eth2Digest,
-                         status: ValidatorStatusCodes,
-                         current_slot: uint64
-                         ): tuple[validators: seq[ValidatorRecord], index: int] =
-  # TODO Spec candidate: inefficient API
-  #
-  # Check that validator really did register
-  # TODO fix tests and enable (nightmare)
-  # let msg = hash_tree_root((pubkey, withdrawal_credentials, randao_commitment))
-  # assert BLSVerify(
-  #   pubkey, msg, proof_of_possession,
-  #   get_domain(fork_data, current_slot, DOMAIN_DEPOSIT))
-
-  var validators_copy = current_validators
-
-  for index, validator in validators_copy.mpairs():
-    if validator.pubkey == pubkey:
-      assert validator.withdrawal_credentials == withdrawal_credentials
-
-      validator.balance += deposit
-      return (validators_copy, index)
-
-  # new validator
-  let
-    rec = ValidatorRecord(
-      pubkey: pubkey,
-      withdrawal_credentials: withdrawal_credentials,
-      randao_commitment: randao_commitment,
-      randao_skips: 0,
-      balance: deposit,
-      status: status,
-      latest_status_change_slot: current_slot,
-      exit_count: 0
-    )
-
-  let index = min_empty_validator_index(validators_copy, current_slot)
-  if index.isNone:
-    validators_copy.add(rec)
-    (validators_copy, len(validators_copy) - 1)
-  else:
-    validators_copy[index.get()] = rec
-    (validators_copy, index.get())
-
 func get_active_validator_indices*(validators: openArray[ValidatorRecord]): seq[Uint24] =
   ## Select the active validators
   for idx, val in validators:
@@ -113,7 +64,7 @@ func get_new_shuffling*(seed: Eth2Digest,
 
     result[slot] = committees
 
-func get_new_validator_registry_delta_chain_tip(
+func get_new_validator_registry_delta_chain_tip*(
     current_validator_registry_delta_chain_tip: Eth2Digest,
     index: Uint24,
     pubkey: ValidatorPubKey,
@@ -127,48 +78,7 @@ func get_new_validator_registry_delta_chain_tip(
     h.update hash_tree_root(pubkey)
 
 func get_effective_balance*(validator: ValidatorRecord): uint64 =
-    min(validator.balance, MAX_DEPOSIT)
+    min(validator.balance, MAX_DEPOSIT * GWEI_PER_ETH)
-
-func exit_validator*(index: Uint24,
-                     state: var BeaconState,
-                     new_status: ValidatorStatusCodes) =
-  ## Remove the validator with the given `index` from `state`.
-  ## Note that this function mutates `state`.
-
-  state.validator_registry_exit_count += 1
-
-  var
-    validator = addr state.validator_registry[index]
-
-  validator.status = new_status
-  validator.latest_status_change_slot = state.slot
-  validator.exit_count = state.validator_registry_exit_count
-
-  # Remove validator from persistent committees
-  for committee in state.persistent_committees.mitems():
-    for i, validator_index in committee:
-      if validator_index == index:
-        committee.delete(i)
-        break
-
-  if new_status == EXITED_WITH_PENALTY:
-    state.latest_penalized_exit_balances[
-      (state.slot div COLLECTIVE_PENALTY_CALCULATION_PERIOD).int] +=
-        get_effective_balance(validator[])
-
-    var
-      whistleblower = addr state.validator_registry[
-        get_beacon_proposer_index(state, state.slot)]
-      whistleblower_reward =
-        validator.balance div WHISTLEBLOWER_REWARD_QUOTIENT
-
-    whistleblower.balance += whistleblower_reward
-    validator.balance -= whistleblower_reward
-
-  state.validator_registry_delta_chain_tip =
-    get_new_validator_registry_delta_chain_tip(
-      state.validator_registry_delta_chain_tip, index, validator.pubkey,
-      ValidatorSetDeltaFlags.EXIT)
 
 func get_updated_validator_registry*(
     validator_registry: seq[ValidatorRecord],
@@ -205,7 +115,7 @@ func get_updated_validator_registry*(
   var validator_registry_delta_chain_tip = validator_registry_delta_chain_tip
   for i in 0..<len(validator_registry):
     if validator_registry[i].status == PENDING_ACTIVATION and
-        validator_registry[i].balance >= MAX_DEPOSIT:
+        validator_registry[i].balance >= MAX_DEPOSIT * GWEI_PER_ETH:
       # Check the balance churn would be within the allowance
       balance_churn += get_effective_balance(validator_registry[i])
       if balance_churn > max_balance_churn:

beacon_chain/ssz.nim

@@ -159,7 +159,7 @@ func hash(a, b: openArray[byte]): array[32, byte] =
 func empty(T: typedesc): T = discard
 const emptyChunk = empty(array[CHUNK_SIZE, byte])
 
-func merkleHash[T](lst: seq[T]): array[32, byte]
+func merkleHash[T](lst: openArray[T]): array[32, byte]
 
 # ################### Hashing interface ###################################
 
@@ -185,25 +185,8 @@ func hash_tree_root*(x: openArray[byte]): array[32, byte] =
   ## Blobs are hashed
   hash(x)
 
-func hash_tree_root*(x: ValidatorRecord): array[32, byte] =
-  ## Containers have their fields recursively hashed, concatenated and hashed
-  # TODO hash_ssz.py code contains special cases for some types, why?
-  withHash:
-    # tmp.add(x.pubkey) # TODO uncertain future of public key format
-    h.update hash_tree_root(x.withdrawal_credentials)
-    h.update hash_tree_root(x.randao_skips)
-    h.update hash_tree_root(x.balance)
-    # h.update hash_tree_root(x.status) # TODO it's an enum, deal with it
-    h.update hash_tree_root(x.latest_status_change_slot)
-    h.update hash_tree_root(x.exit_count)
-
-func hash_tree_root*(x: ShardCommittee): array[32, byte] =
-  withHash:
-    h.update hash_tree_root(x.shard)
-    h.update merkleHash(x.committee)
-
 func hash_tree_root*[T: not enum](x: T): array[32, byte] =
-  when T is seq:
+  when T is seq or T is array:
     ## Sequences are tree-hashed
     merkleHash(x)
   else:
@@ -242,7 +225,7 @@ func hash_tree_root*(x: ValidatorSig): array[32, byte] =
 
 # ################### Tree hash ###################################
 
-func merkleHash[T](lst: seq[T]): array[32, byte] =
+func merkleHash[T](lst: openArray[T]): array[32, byte] =
   ## Merkle tree hash of a list of homogenous, non-empty items
 
   # TODO: the heap allocations here can be avoided by computing the merkle tree

beacon_chain/state_transition.nim

@@ -10,7 +10,7 @@
 #
 # The purpose of this code right is primarily educational, to help piece
 # together the mechanics of the beacon state and to discover potential problem
-# areas.
+# areas. The entry point is `updateState` which is at the bottom of the file!
 #
 # General notes about the code (TODO):
 # * It's inefficient - we quadratically copy, allocate and iterate when there
@@ -34,36 +34,14 @@
 # now.
 
 import
-  math, options, sequtils,
+  chronicles, math, options, sequtils,
   ./extras, ./ssz,
   ./spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
   milagro_crypto
 
-func processAttestations(state: var BeaconState,
+func flatten[T](v: openArray[seq[T]]): seq[T] =
-                         blck: BeaconBlock): bool =
+  # TODO not in nim - doh.
-  # Each block includes a number of attestations that the proposer chose. Each
+  for x in v: result.add x
-  # attestation represents an update to a specific shard and is signed by a
-  # committee of validators.
-  # Here we make sanity checks for each attestation and it to the state - most
-  # updates will happen at the epoch boundary where state updates happen in
-  # bulk.
-  if blck.body.attestations.len > MAX_ATTESTATIONS_PER_BLOCK:
-    return
-
-  if not allIt(blck.body.attestations, checkAttestation(state, it)):
-    return
-
-  # All checks passed - update state
-  state.latest_attestations.add mapIt(blck.body.attestations,
-    PendingAttestationRecord(
-      data: it.data,
-      participation_bitfield: it.participation_bitfield,
-      custody_bitfield: it.custody_bitfield,
-      slot_included: state.slot
-    )
-  )
-
-  true
 
 func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
   ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposer-signature
@@ -72,98 +50,314 @@ func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
   blck_without_sig.signature = ValidatorSig()
 
   let
-    proposal_hash = hash_tree_root(ProposalSignedData(
+    signed_data = ProposalSignedData(
       slot: state.slot,
-      shard: BEACON_CHAIN_SHARD,
+      shard: BEACON_CHAIN_SHARD_NUMBER,
       block_root: Eth2Digest(data: hash_tree_root(blck_without_sig))
-    ))
+    )
+    proposal_hash = hash_tree_root(signed_data)
+
+  let proposer_index = get_beacon_proposer_index(state, state.slot)
 
-  let validator_idx = get_beacon_proposer_index(state, state.slot)
   BLSVerify(
-    state.validator_registry[validator_idx].pubkey,
+    state.validator_registry[proposer_index].pubkey,
     proposal_hash, blck.signature,
     get_domain(state.fork_data, state.slot, DOMAIN_PROPOSAL))
 
-func processRandaoReveal(state: var BeaconState,
+func processRandao(state: var BeaconState, blck: BeaconBlock): bool =
-                         blck: BeaconBlock): bool =
+  ## When a validator signs up, they will commit an hash to the block,
+  ## the randao_commitment - this hash is the result of a secret value
+  ## hashed n times.
+  ## The first time the proposer proposes a block, they will hash their secret
+  ## value n-1 times, and provide that as "reveal" - now everyone else can
+  ## verify the reveal by hashing once.
+  ## The next time the proposer proposes, they will reveal the secret value
+  ## hashed n-2 times and so on, and everyone will verify that it matches n-1.
+  ##
+  ## Effectively, the block proposer can only reveal n - 1 times, so better pick
+  ## a large N!
+  ##
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#randao
   let
     proposer_index = get_beacon_proposer_index(state, state.slot)
     proposer = addr state.validator_registry[proposer_index]
 
   # Check that proposer commit and reveal match
-  if repeat_hash(blck.randao_reveal, proposer.randao_skips + 1) !=
+  if repeat_hash(blck.randao_reveal, proposer.randao_layers) !=
       proposer.randao_commitment:
-    return
+    return false
 
   # Update state and proposer now that we're alright
   for i, b in state.randao_mix.data:
     state.randao_mix.data[i] = b xor blck.randao_reveal.data[i]
 
   proposer.randao_commitment = blck.randao_reveal
-  proposer.randao_skips = 0
+  proposer.randao_layers = 0
 
-  true
+  return true
+
+func processPoWReceiptRoot(state: var BeaconState, blck: BeaconBlock) =
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#pow-receipt-root
 
-func processPoWReceiptRoot(state: var BeaconState, blck: BeaconBlock): bool =
   for x in state.candidate_pow_receipt_roots.mitems():
     if blck.candidate_pow_receipt_root == x.candidate_pow_receipt_root:
       x.votes += 1
-      return true
+      return
 
   state.candidate_pow_receipt_roots.add CandidatePoWReceiptRootRecord(
     candidate_pow_receipt_root: blck.candidate_pow_receipt_root,
     votes: 1
   )
+
+proc processProposerSlashings(state: var BeaconState, blck: BeaconBlock): bool =
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposer-slashings-1
+
+  if len(blck.body.proposer_slashings) > MAX_PROPOSER_SLASHINGS:
+    warn("PropSlash: too many!",
+      proposer_slashings = len(blck.body.proposer_slashings))
+    return false
+
+  for proposer_slashing in blck.body.proposer_slashings:
+    let proposer = addr state.validator_registry[proposer_slashing.proposer_index]
+    if not BLSVerify(
+        proposer.pubkey,
+        hash_tree_root(proposer_slashing.proposal_data_1),
+        proposer_slashing.proposal_signature_1,
+        get_domain(
+          state.fork_data, proposer_slashing.proposal_data_1.slot,
+          DOMAIN_PROPOSAL)):
+      warn("PropSlash: invalid signature 1")
+      return false
+    if not BLSVerify(
+        proposer.pubkey,
+        hash_tree_root(proposer_slashing.proposal_data_2),
+        proposer_slashing.proposal_signature_2,
+        get_domain(
+          state.fork_data, proposer_slashing.proposal_data_2.slot,
+          DOMAIN_PROPOSAL)):
+      warn("PropSlash: invalid signature 2")
+      return false
+
+    if not (proposer_slashing.proposal_data_1.slot ==
+        proposer_slashing.proposal_data_2.slot):
+      warn("PropSlash: slot mismatch")
+      return false
+
+    if not (proposer_slashing.proposal_data_1.shard ==
+        proposer_slashing.proposal_data_2.shard):
+      warn("PropSlash: shard mismatch")
+      return false
+
+    if not (proposer_slashing.proposal_data_1.block_root ==
+        proposer_slashing.proposal_data_2.block_root):
+      warn("PropSlash: block root mismatch")
+      return false
+
+    if not (proposer.status != EXITED_WITH_PENALTY):
+      warn("PropSlash: wrong status")
+      return false
+
+    update_validator_status(
+      state, proposer_slashing.proposer_index, EXITED_WITH_PENALTY)
+
   return true
 
+func verify_casper_votes(state: BeaconState, votes: SlashableVoteData): bool =
+  if len(votes.aggregate_signature_poc_0_indices) +
+      len(votes.aggregate_signature_poc_1_indices) > MAX_CASPER_VOTES:
+    return false
+
+  # TODO
+  # let pubs = [
+  #   aggregate_pubkey(mapIt(votes.aggregate_signature_poc_0_indices,
+  #     state.validators[it].pubkey)),
+  #   aggregate_pubkey(mapIt(votes.aggregate_signature_poc_1_indices,
+  #     state.validators[it].pubkey))]
+
+  # return bls_verify_multiple(pubs, [hash_tree_root(votes)+bytes1(0), hash_tree_root(votes)+bytes1(1), signature=aggregate_signature)
+
+  return true
+
+proc indices(vote: SlashableVoteData): seq[Uint24] =
+  vote.aggregate_signature_poc_0_indices &
+    vote.aggregate_signature_poc_1_indices
+
+proc processCasperSlashings(state: var BeaconState, blck: BeaconBlock): bool =
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#casper-slashings-1
+  if len(blck.body.casper_slashings) > MAX_CASPER_SLASHINGS:
+    warn("CaspSlash: too many!")
+    return false
+
+  for casper_slashing in blck.body.casper_slashings:
+    if not verify_casper_votes(state, casper_slashing.votes_1):
+      warn("CaspSlash: invalid votes 1")
+      return false
+    if not verify_casper_votes(state, casper_slashing.votes_2):
+      warn("CaspSlash: invalid votes 2")
+      return false
+    if not (casper_slashing.votes_1.data != casper_slashing.votes_2.data):
+      warn("CaspSlash: invalid data")
+      return false
+
+    let intersection = filterIt(
+      indices(casper_slashing.votes_1), it in indices(casper_slashing.votes_2))
+
+    if len(intersection) < 1:
+      warn("CaspSlash: no intersection")
+      return false
+
+    if not (
+        ((casper_slashing.votes_1.data.justified_slot + 1 <
+          casper_slashing.votes_2.data.justified_slot + 1) ==
+        (casper_slashing.votes_2.data.slot < casper_slashing.votes_1.data.slot)) or
+        (casper_slashing.votes_1.data.slot == casper_slashing.votes_2.data.slot)):
+      warn("CaspSlash: some weird long condition failed")
+      return false
+
+    for i in intersection:
+      if state.validator_registry[i].status != EXITED_WITH_PENALTY:
+        update_validator_status(state, i, EXITED_WITH_PENALTY)
+
+  return true
+
+proc processAttestations(state: var BeaconState, blck: BeaconBlock): bool =
+  ## Each block includes a number of attestations that the proposer chose. Each
+  ## attestation represents an update to a specific shard and is signed by a
+  ## committee of validators.
+  ## Here we make sanity checks for each attestation and it to the state - most
+  ## updates will happen at the epoch boundary where state updates happen in
+  ## bulk.
+  ##
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestations-1
+  if blck.body.attestations.len > MAX_ATTESTATIONS:
+    warn("Attestation: too many!", attestations = blck.body.attestations.len)
+    return false
+
+  if not allIt(blck.body.attestations, checkAttestation(state, it)):
+    return false
+
+  # All checks passed - update state
+  state.latest_attestations.add mapIt(blck.body.attestations,
+    PendingAttestationRecord(
+      data: it.data,
+      participation_bitfield: it.participation_bitfield,
+      custody_bitfield: it.custody_bitfield,
+      slot_included: state.slot,
+    )
+  )
+
+  return true
+
+proc processDeposits(state: var BeaconState, blck: BeaconBlock): bool =
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#deposits-1
+  # TODO! Spec writing in progress
+  true
+
+proc processExits(state: var BeaconState, blck: BeaconBlock): bool =
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#exits-1
+  if len(blck.body.exits) > MAX_EXITS:
+    warn("Exit: too many!")
+    return false
+
+  for exit in blck.body.exits:
+    let validator = state.validator_registry[exit.validator_index]
+
+    if not BLSVerify(
+        validator.pubkey, ZERO_HASH.data, exit.signature,
+        get_domain(state.fork_data, exit.slot, DOMAIN_EXIT)):
+      warn("Exit: invalid signature")
+      return false
+
+    if not (validator.status == ACTIVE):
+      warn("Exit: validator not active")
+      return false
+
+    if not (state.slot >= exit.slot):
+      warn("Exit: bad slot")
+      return false
+
+    if not (state.slot >=
+        validator.latest_status_change_slot +
+          SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD):
+      warn("Exit: not within committee change period")
+
+    update_validator_status(state, exit.validator_index, ACTIVE_PENDING_EXIT)
+
+  return true
+
+proc process_ejections(state: var BeaconState) =
+  ## Iterate through the validator registry and eject active validators with
+  ## balance below ``EJECTION_BALANCE``
+  ##
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#ejections
+
+  for index, validator in state.validator_registry:
+    if is_active_validator(validator) and validator.balance < EJECTION_BALANCE:
+        update_validator_status(state, index.Uint24, EXITED_WITHOUT_PENALTY)
+
+proc processBlock(state: var BeaconState, latest_block, blck: BeaconBlock): bool =
+  ## When there's a new block, we need to verify that the block is sane and
+  ## update the state accordingly
+
+  # TODO when there's a failure, we should reset the state!
+  # TODO probably better to do all verification first, then apply state changes
+
+  if blck.slot != state.slot:
+    warn("Unexpected block slot number")
+    return false
+
+  if not verifyProposerSignature(state, blck):
+    warn("Proposer signature not valid")
+    return false
+
+  if not processRandao(state, blck):
+    warn("Randao reveal failed")
+    return false
+
+  processPoWReceiptRoot(state, blck)
+
+  if not processProposerSlashings(state, blck):
+    return false
+
+  if not processCasperSlashings(state, blck):
+    return false
 
-func processBlock(state: var BeaconState, blck: BeaconBlock): bool =
   if not processAttestations(state, blck):
-    false
+    return false
-  elif not verifyProposerSignature(state, blck):
-    false
-  elif not processRandaoReveal(state, blck):
-    false
-  elif not processPoWReceiptRoot(state, blck):
-    false
-  else:
-    true
 
-func processSlot(state: var BeaconState, latest_block: BeaconBlock): bool =
+  if not processDeposits(state, blck):
+    return false
+
+  if not processExits(state, blck):
+    return false
+
+  process_ejections(state)
+
+  return true
+
+func processSlot(state: var BeaconState, latest_block: BeaconBlock) =
   ## Time on the beacon chain moves in slots. Every time we make it to a new
   ## slot, a proposer cleates a block to represent the state of the beacon
   ## chain at that time. In case the proposer is missing, it may happen that
   ## the no block is produced during the slot.
   ##
   ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#per-slot-processing
-  # TODO state not rolled back in case of failure
 
   let
     latest_hash = Eth2Digest(data: hash_tree_root(latest_block))
 
   state.slot += 1
-  state.latest_block_roots.add latest_hash
+  state.validator_registry[
+    get_beacon_proposer_index(state, state.slot)].randao_layers += 1
 
-  if state.latest_block_roots.len < 2 or
+  let
-      state.latest_block_roots[^2] != state.latest_block_roots[^1]:
+    previous_block_root = Eth2Digest(data: hash_tree_root(latest_block))
-    # TODO a bit late for the following checks?
+  for i in 0 ..< state.latest_block_roots.len - 1:
-    # https://github.com/ethereum/eth2.0-specs/issues/284
+    state.latest_block_roots[i] = state.latest_block_roots[i + 1]
-    if latest_block.slot != state.slot:
+  state.latest_block_roots[^1] = previous_block_root
-      false
-    elif latest_block.ancestor_hashes !=
-        get_updated_ancestor_hashes(latest_block, latest_hash):
-      false
-    else:
-      processBlock(state, latest_block)
-  else:
-    state.validator_registry[get_beacon_proposer_index(state, state.slot)].randao_skips += 1
-    # Skip all other per-slot processing. Move directly to epoch processing
-    # prison. Do not do any slot updates when passing go.
-    true
 
-func flatten[T](v: openArray[seq[T]]): seq[T] =
+  if state.slot mod LATEST_BLOCK_ROOTS_COUNT == 0:
-  # TODO not in nim - doh.
+    state.batched_block_roots.add(merkle_root(state.latest_block_roots))
-  for x in v: result.add x
 
 func get_epoch_boundary_attesters(
     state: BeaconState,
@@ -199,11 +393,13 @@ func lowerThan(candidate, current: Eth2Digest): bool =
     if v > candidate.data[i]: return true
   return false
 
-func processEpoch(state: var BeaconState, blck: BeaconBlock): bool =
+func processEpoch(state: var BeaconState, blck: BeaconBlock) =
   ## Epoch processing happens every time we've passed EPOCH_LENGTH blocks.
   ## Because some slots may be skipped, it may happen that we go through the
   ## loop more than once - each time the latest_state_recalculation_slot will be
   ## increased by EPOCH_LENGTH.
+  ##
+  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#per-epoch-processing
 
   while blck.slot >= EPOCH_LENGTH.uint64 + state.latest_state_recalculation_slot:
     # Convenience shortcut, from spec
@@ -470,28 +666,57 @@ func processEpoch(state: var BeaconState, blck: BeaconBlock): bool =
       # TODO Remove all attestation records older than slot `s`.
       state.latest_block_roots = state.latest_block_roots[EPOCH_LENGTH..^1]
 
-  true
+proc updateState*(state: BeaconState, latest_block: BeaconBlock,
-
+    new_block: Option[BeaconBlock]):
-func updateState*(state: BeaconState, latest_block: BeaconBlock): Option[BeaconState] =
+      tuple[state: BeaconState, block_ok: bool] =
-  ## Adjust `state` according to the information in `blck`.
+  ## Time in the beacon chain moves by slots. Every time (haha.) that happens,
-  ## Returns the new state, or `none` if the block is invalid.
+  ## we will update the beacon state. Normally, the state updates will be driven
-
+  ## by the contents of a new block, but it may happen that the block goes
+  ## missing - the state updates happen regardless.
+  ## Each call to this function will advance the state by one slot - new_block,
+  ## if present, must match that slot.
+  #
+  # TODO this function can be written with a loop inside to handle all empty
+  #      slots up to the slot of the new_block - but then again, why not eagerly
+  #      update the state as time passes? Something to ponder...
   # TODO check to which extent this copy can be avoided (considering forks etc),
   #      for now, it serves as a reminder that we need to handle invalid blocks
   #      somewhere..
   # TODO many functions will mutate `state` partially without rolling back
   #      the changes in case of failure (look out for `var BeaconState` and
   #      bool return values...)
-  var state = state
+  # TODO There's a discussion about what this function should do, and when:
+  #      https://github.com/ethereum/eth2.0-specs/issues/284
+  var new_state = state
 
-  # Slot processing is split up into two phases - lightweight updates done
+  # Per-slot updates - these happen regardless if there is a block or not
-  # for each slot, and bigger updates done for each epoch.
+  processSlot(new_state, latest_block)
 
-  # Lightweight updates that happen for every slot
+  let block_ok =
-  if not processSlot(state, latest_block): return
+    if new_block.isSome():
+      # Block updates - these happen when there's a new block being suggested
+      # by the block proposer. Every actor in the network will update its state
+      # according to the contents of this block - but first they will validate
+      # that the block is sane.
+      # TODO what should happen if block processing fails?
+      #      https://github.com/ethereum/eth2.0-specs/issues/293
+      var block_state = new_state
+      if processBlock(block_state, latest_block, new_block.get()):
+        # processBlock will mutate the state! only apply if it worked..
+        # TODO yeah, this too is inefficient
+        new_state = block_state
+        true
+      else:
+        false
+    else:
+      let proposer_index = get_beacon_proposer_index(new_state, new_state.slot)
+      new_state.validator_registry[proposer_index].randao_layers += 1
+      # Skip all other per-slot processing. Move directly to epoch processing
+      # prison. Do not do any slot updates when passing go.
+      true
 
-  # Heavy updates that happen for every epoch
+  # Heavy updates that happen for every epoch - these never fail (or so we hope)
-  if not processEpoch(state, latest_block): return
+  processEpoch(new_state, latest_block)
 
-  # All good, we can return the new state
+  # State update never fails, but block validation might...
-  some(state)
+  (new_state, block_ok)

tests/all_tests.nim

@@ -7,7 +7,7 @@
 
 import
   ./test_beaconstate,
-  ./test_block_processing,
+  ./test_state_transition,
   ./test_helpers,
   ./test_ssz,
   ./test_validator,

tests/test_beaconstate.nim

@@ -12,8 +12,6 @@ import
   ../beacon_chain/spec/[beaconstate, datatypes, digest]
 
 suite "Beacon state":
-  # Smoke test
+  test "Smoke test on_startup":
-
+    let state = on_startup(makeInitialDeposits(EPOCH_LENGTH), 0, Eth2Digest())
-  test "Smoke on_startup":
-    let state = on_startup(makeInitialValidators(EPOCH_LENGTH), 0, Eth2Digest())
     check: state.validator_registry.len == EPOCH_LENGTH

tests/test_block_processing.nim

@@ -1,26 +0,0 @@
-# beacon_chain
-# Copyright (c) 2018 Status Research & Development GmbH
-# Licensed and distributed under either of
-#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
-#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
-# at your option. This file may not be copied, modified, or distributed except according to those terms.
-
-import
-  options, sequtils, unittest,
-  ./testutil,
-  ../beacon_chain/spec/[beaconstate, datatypes, digest],
-  ../beacon_chain/[extras, state_transition]
-
-suite "Block processing":
-  ## For now just test that we can compile and execute block processing with mock data.
-
-  test "Mock state update":
-    let
-      state = on_startup(makeInitialValidators(), 0, Eth2Digest())
-      blck = BeaconBlock(
-        slot: 1,
-        ancestor_hashes: @[Eth2Digest()]
-      )
-      newState = updateState(state, blck)
-    check:
-      newState.isNone() # Broken block, should fail processing

tests/test_state_transition.nim

@@ -0,0 +1,68 @@
+# beacon_chain
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  options, sequtils, unittest,
+  ./testutil,
+  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers],
+  ../beacon_chain/[extras, state_transition]
+
+suite "Block processing":
+  ## For now just test that we can compile and execute block processing with
+  ## mock data.
+
+  test "Passes from genesis state, no block":
+    let
+      state = on_startup(makeInitialDeposits(), 0, Eth2Digest())
+      latest_block = makeGenesisBlock(state)
+      new_state = updateState(state, latest_block, none(BeaconBlock))
+    check:
+      new_state.state.slot == latest_block.slot + 1
+      new_state.block_ok
+
+  test "Passes from genesis state, empty block":
+    let
+      state = on_startup(makeInitialDeposits(), 0, Eth2Digest())
+      latest_block = makeGenesisBlock(state)
+      new_block = makeBlock(state, latest_block)
+      new_state = updateState(state, latest_block, some(new_block))
+
+    check:
+      new_state.state.slot == latest_block.slot + 1
+      new_state.block_ok
+
+  test "Passes through epoch update, no block":
+    var
+      state = on_startup(makeInitialDeposits(), 0, Eth2Digest())
+      latest_block = makeGenesisBlock(state)
+
+    for i in 1..EPOCH_LENGTH.int:
+      let new_state = updateState(state, latest_block, none(BeaconBlock))
+      check:
+        new_state.block_ok
+      state = new_state.state
+
+    check:
+      state.slot == latest_block.slot + EPOCH_LENGTH
+
+  test "Passes through epoch update, empty block":
+    var
+      state = on_startup(makeInitialDeposits(), 0, Eth2Digest())
+      latest_block = makeGenesisBlock(state)
+
+    for i in 1..EPOCH_LENGTH.int:
+      var new_block = makeBlock(state, latest_block)
+
+      let new_state = updateState(state, latest_block, some(new_block))
+
+      check:
+        new_state.block_ok
+      state = new_state.state
+      latest_block = new_block
+
+    check:
+      state.slot == latest_block.slot

tests/testutil.nim

@@ -7,15 +7,99 @@
 
 import
   milagro_crypto,
-  ../beacon_chain/extras,
+  ../beacon_chain/[extras, ssz],
-  ../beacon_chain/spec/[crypto, datatypes]
+  ../beacon_chain/spec/[crypto, datatypes, digest, helpers]
 
-func makeValidatorPrivKey(n: int): ValidatorPrivKey =
+const
-  result.x[0] = n
+  randaoRounds = 100
 
-func makeInitialValidators*(n = EPOCH_LENGTH): seq[InitialValidator] =
+func makeValidatorPrivKey(i: int): ValidatorPrivKey =
-  for i in 0..<n.int:
+  var i = i + 1 # 0 does not work, as private key...
-    let key = makeValidatorPrivKey(i)
+  copyMem(result.x[0].addr, i.addr, min(sizeof(result.x), sizeof(i)))
-    result.add InitialValidator(
+
-      pubkey: key.fromSigKey()
+func makeFakeHash(i: int): Eth2Digest =
+  copyMem(result.data[0].addr, i.unsafeAddr, min(sizeof(result.data), sizeof(i)))
+
+func hackPrivKey(v: ValidatorRecord): ValidatorPrivKey =
+  ## Extract private key, per above hack
+  var i: int
+  copyMem(
+    i.addr, v.withdrawal_credentials.data[0].unsafeAddr,
+    min(sizeof(v.withdrawal_credentials.data), sizeof(i)))
+  makeValidatorPrivKey(i)
+
+func hackReveal(v: ValidatorRecord): Eth2Digest =
+  result = v.withdrawal_credentials
+  for i in 0..randaoRounds:
+    let tmp = repeat_hash(result, 1)
+    if tmp == v.randao_commitment:
+      return
+    result = tmp
+  raise newException(Exception, "can't find randao hack value")
+
+func makeDeposit(i: int): Deposit =
+  ## Ugly hack for now: we stick the private key in withdrawal_credentials
+  ## which means we can repro private key and randao reveal from this data,
+  ## for testing :)
+  let
+    privkey = makeValidatorPrivKey(i)
+    pubkey = privkey.fromSigKey()
+    withdrawal_credentials = makeFakeHash(i)
+    randao_commitment = repeat_hash(withdrawal_credentials, randaoRounds)
+    pop = signMessage(privkey, hash_tree_root(
+      (pubkey, withdrawal_credentials, randao_commitment)))
+
+  Deposit(
+    deposit_data: DepositData(
+      deposit_parameters: DepositParameters(
+        pubkey: pubkey,
+        proof_of_possession: pop,
+        withdrawal_credentials: withdrawal_credentials,
+        randao_commitment: randao_commitment
+      ),
+      value: MAX_DEPOSIT * GWEI_PER_ETH,
     )
+  )
+
+func makeInitialDeposits*(n = EPOCH_LENGTH): seq[Deposit] =
+  for i in 0..<n.int:
+    result.add makeDeposit(i + 1)
+
+func makeGenesisBlock*(state: BeaconState): BeaconBlock =
+  BeaconBlock(
+    slot: INITIAL_SLOT_NUMBER,
+    state_root: Eth2Digest(data: hash_tree_root(state))
+  )
+
+func makeBlock*(
+    state: BeaconState, latest_block: BeaconBlock): BeaconBlock =
+  let
+    new_slot = state.slot + 1
+    proposer = state.validator_registry[
+      get_beacon_proposer_index(state, new_slot)]
+
+  var new_block = BeaconBlock(
+      slot: new_slot,
+      state_root: Eth2Digest(data: hash_tree_root(state)),
+      randao_reveal: hackReveal(proposer)
+    )
+
+  let
+    signed_data = ProposalSignedData(
+      slot: new_block.slot,
+      shard: BEACON_CHAIN_SHARD_NUMBER,
+      block_root: Eth2Digest(data: hash_tree_root(new_block))
+    )
+    proposal_hash = hash_tree_root(signed_data)
+
+    proposerPrivkey = hackPrivKey(proposer)
+
+  assert proposerPrivkey.fromSigKey() == proposer.pubkey
+
+  new_block.signature =
+    signMessage(proposerPrivkey, proposal_hash)
+
+  assert verifyMessage(
+    new_block.signature, proposal_hash, proposerPrivkey.fromSigKey())
+
+  new_block