0.8.0, continued (#315)

* rename get_genesis_beacon_state(...) to initialize_beacon_state_from_eth1(...); remove unused vestiges get_temporary_block_header(...) and get_empty_block(...); partly align initialize_beacon_state_from_eth1(...) with 0.8.0 version; implement CompactCommittee object type; update process_final_updates(...) to 0.8.0

* mark get_shard_delta(...) as 0.8.0

* mark get_total_active_balance(...) and epoch transition helper functions as 0.8.0

* move FORK_CHOICE_BALANCE_INCREMENT, not in spec anymore, to sole user in fork_choice

beacon_chain/beacon_node.nim

@@ -737,7 +737,7 @@ when isMainModule:
         stderr.write "Please regenerate the deposit files by running validator_keygen again\n"
         quit 1
 
-    let initialState = get_genesis_beacon_state(
+    let initialState = initialize_beacon_state_from_eth1(
       deposits,
       uint64(times.toUnix(times.getTime()) + config.genesisOffset),
       Eth1Data(), {skipValidation})

beacon_chain/fork_choice.nim

@@ -1,5 +1,5 @@
 import
-  deques, options, sequtils, tables,
+  deques, math, options, sequtils, tables,
   chronicles,
   ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], extras,
   ./attestation_pool, ./beacon_node_types, ./beacon_chain_db, ./ssz
@@ -22,6 +22,8 @@ proc lmdGhost*(
   #       Nim implementation for cumulative frequencies at
   #       https://github.com/numforge/laser/blob/990e59fffe50779cdef33aa0b8f22da19e1eb328/benchmarks/random_sampling/fenwicktree.nim
 
+  const FORK_CHOICE_BALANCE_INCREMENT = 2'u64^0 * 10'u64^9
+
   let
     active_validator_indices =
       get_active_validator_indices(

beacon_chain/spec/beaconstate.nim

@@ -174,23 +174,35 @@ func slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex,
   increase_balance(
     state, whistleblower_index, whistleblowing_reward - proposer_reward)
 
-func get_temporary_block_header(blck: BeaconBlock): BeaconBlockHeader =
-  ## Return the block header corresponding to a block with ``state_root`` set
-  ## to ``ZERO_HASH``.
-  BeaconBlockHeader(
-    slot: blck.slot,
-    parent_root: blck.parent_root,
-    state_root: ZERO_HASH,
-    body_root: hash_tree_root(blck.body),
-    # signing_root(block) is used for block id purposes so signature is a stub
-    signature: ValidatorSig(),
-  )
+# https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#get_compact_committees_root
+func get_compact_committees_root*(state: BeaconState, epoch: Epoch): Eth2Digest =
+  # Return the compact committee root at ``epoch``.
 
-func get_empty_block*(): BeaconBlock =
-  # Nim default values fill this in mostly correctly.
-  BeaconBlock(slot: GENESIS_SLOT)
+  # TODO if profiling shows this as expensive, plumb through properly
+  var cache = get_empty_per_epoch_cache()
 
-func get_genesis_beacon_state*(
+  var committees : array[SHARD_COUNT, CompactCommittee]
+  let start_shard = get_start_shard(state, epoch)
+  for committee_number in 0'u64 ..< get_committee_count(state, epoch):
+    let shard = (start_shard + committee_number) mod SHARD_COUNT
+    for index in get_crosslink_committee(state, epoch, shard, cache):
+      let validator = state.validators[index]
+      committees[shard.int].pubkeys.add(validator.pubkey)
+      let
+        compact_balance =
+          validator.effective_balance div EFFECTIVE_BALANCE_INCREMENT
+
+        # `index` (top 6 bytes) + `slashed` (16th bit) + `compact_balance`
+        # (bottom 15 bits)
+        compact_validator =
+          uint64((index.uint64 shl 16) + (validator.slashed.uint64 shl 15) +
+            compact_balance)
+      committees[shard.int].compact_validators.add(compact_validator)
+
+  hash_tree_root(committees)
+
+# https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#genesis
+func initialize_beacon_state_from_eth1*(
     genesis_validator_deposits: openArray[Deposit],
     genesis_time: uint64,
     genesis_eth1_data: Eth1Data,
@@ -212,26 +224,17 @@ func get_genesis_beacon_state*(
   doAssert genesis_validator_deposits.len >= SLOTS_PER_EPOCH
 
   var state = BeaconState(
-    # Misc
     genesis_time: genesis_time,
-    fork: Fork(
-        previous_version: GENESIS_FORK_VERSION,
-        current_version: GENESIS_FORK_VERSION,
-        epoch: GENESIS_EPOCH,
-    ),
-
-    latest_block_header: get_temporary_block_header(get_empty_block()),
-
-    # Ethereum 1.0 chain data
-    # eth1_data_votes automatically initialized
+    latest_block_header:
+      BeaconBlockHeader(body_root: hash_tree_root(BeaconBlockBody())),
     eth1_data: genesis_eth1_data,
   )
 
-  # Process genesis deposits
+  # Process deposits
   for deposit in genesis_validator_deposits:
     discard process_deposit(state, deposit, flags)
 
-  # Process genesis activations
+  # Process activations
   for validator_index in 0 ..< state.validators.len:
     let validator = addr state.validators[validator_index]
     if validator.effective_balance >= MAX_EFFECTIVE_BALANCE:

beacon_chain/spec/datatypes.nim

@@ -57,9 +57,6 @@ const
   ## Spec version we're aiming to be compatible with, right now
   ## TODO: improve this scheme once we can negotiate versions in protocol
 
-  # TODO remove erstwhile blob/v0.7.0
-  FORK_CHOICE_BALANCE_INCREMENT* = 2'u64^0 * 10'u64^9
-
   # Initial values
   # ---------------------------------------------------------------
   # https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#initial-values
@@ -221,6 +218,11 @@ type
     signature*: ValidatorSig ##\
     ## Proposer signature
 
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#compactcommittee
+  CompactCommittee* = object
+    pubkeys*: seq[ValidatorPubKey]
+    compact_validators*: seq[uint64]
+
   # https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#beaconblockheader
   BeaconBlockHeader* = object
     slot*: Slot

beacon_chain/spec/state_transition_epoch.nim

@@ -37,15 +37,16 @@ import # TODO - cleanup imports
   ../extras, ../ssz, ../beacon_node_types,
   beaconstate, bitfield, crypto, datatypes, digest, helpers, validator
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.6.3/specs/core/0_beacon-chain.md#helper-functions-1
+# https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#get_total_active_balance
 func get_total_active_balance(state: BeaconState): Gwei =
   return get_total_balance(
     state,
     get_active_validator_indices(state, get_current_epoch(state)))
 
+# https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#helper-functions-1
 func get_matching_source_attestations(state: BeaconState, epoch: Epoch):
     seq[PendingAttestation] =
-  doAssert epoch in @[get_current_epoch(state), get_previous_epoch(state)]
+  doAssert epoch in [get_current_epoch(state), get_previous_epoch(state)]
   if epoch == get_current_epoch(state):
     state.current_epoch_attestations
   else:
@@ -145,7 +146,7 @@ func get_winning_crosslink_and_attesting_indices(
     winning_crosslink_balance = 0.Gwei
 
   for candidate_crosslink in crosslinks:
-    ## TODO when confidence that this exactly reproduces the spec version,
+    ## TODO when confident this exactly reproduces the spec version,
     ## remove the when false'd scaffolding.
     when false:
       let crosslink_balance_uncached =
@@ -427,7 +428,7 @@ func process_slashings(state: var BeaconState) =
           min(sum(state.slashings) * 3, total_balance) div total_balance
       decrease_balance(state, index.ValidatorIndex, penalty)
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.6.3/specs/core/0_beacon-chain.md#final-updates
+# https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#final-updates
 func process_final_updates(state: var BeaconState) =
   let
     current_epoch = get_current_epoch(state)
@@ -453,10 +454,20 @@ func process_final_updates(state: var BeaconState) =
     (state.start_shard + get_shard_delta(state, current_epoch)) mod
       SHARD_COUNT
 
-  # Set total slashed balances
-  state.slashings[next_epoch mod EPOCHS_PER_SLASHINGS_VECTOR] = (
-    state.slashings[current_epoch mod EPOCHS_PER_SLASHINGS_VECTOR]
-  )
+  # Set active index root
+  let
+    index_epoch = next_epoch + ACTIVATION_EXIT_DELAY
+    index_root_position = index_epoch mod EPOCHS_PER_HISTORICAL_VECTOR
+    indices_list = get_active_validator_indices(state, index_epoch)
+  state.active_index_roots[index_root_position] = hash_tree_root(indices_list)
+
+  # Set committees root
+  let committee_root_position = next_epoch mod EPOCHS_PER_HISTORICAL_VECTOR
+  state.compact_committees_roots[committee_root_position] =
+    get_compact_committees_root(state, next_epoch)
+
+  # Reset slashings
+  state.slashings[next_epoch mod EPOCHS_PER_SLASHINGS_VECTOR] = 0.Gwei
 
   # Set randao mix
   state.randao_mixes[next_epoch mod LATEST_RANDAO_MIXES_LENGTH] =

beacon_chain/spec/validator.nim

@@ -88,7 +88,7 @@ func get_previous_epoch*(state: BeaconState): Epoch =
   else:
     current_epoch - 1
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.7.1/specs/core/0_beacon-chain.md#get_shard_delta
+# https://github.com/ethereum/eth2.0-specs/blob/v0.8.0/specs/core/0_beacon-chain.md#get_shard_delta
 func get_shard_delta*(state: BeaconState, epoch: Epoch): uint64 =
   ## Return the number of shards to increment ``state.start_shard``
   ## during ``epoch``.

research/serialized_sizes.nim

@@ -5,7 +5,7 @@ import
   ../tests/testutil
 
 proc stateSize(deposits: int, maxContent = false) =
-  var state = get_genesis_beacon_state(
+  var state = initialize_beacon_state_from_eth1(
     makeInitialDeposits(
       deposits, {skipValidation}), 0, Eth1Data(), {skipValidation})
 

research/state_sim.nim

@@ -53,7 +53,7 @@ cli do(slots = 448,
        validate = true):
   let
     flags = if validate: {} else: {skipValidation}
-    genesisState = get_genesis_beacon_state(
+    genesisState = initialize_beacon_state_from_eth1(
       makeInitialDeposits(validators, flags), 0, Eth1Data(), flags)
     genesisBlock = get_initial_beacon_block(genesisState)
 

tests/official/fixtures_utils.nim

@@ -26,7 +26,6 @@ type
     DEPOSIT_CONTRACT_TREE_DEPTH*: int
     MIN_DEPOSIT_AMOUNT*: uint64
     MAX_EFFECTIVE_BALANCE*: uint64
-    FORK_CHOICE_BALANCE_INCREMENT*: uint64
     EJECTION_BALANCE*: uint64
     GENESIS_FORK_VERSION*: uint32
     GENESIS_SLOT*: Slot

tests/test_attestation_pool.nim

@@ -29,7 +29,7 @@ suite "Attestation pool processing" & preset():
 
   # Genesis state that results in 2 members per committee
   let
-    genState = get_genesis_beacon_state(
+    genState = initialize_beacon_state_from_eth1(
       makeInitialDeposits(SLOTS_PER_EPOCH * 2, {skipValidation}), 0, Eth1Data(),
         {skipValidation})
     genBlock = get_initial_beacon_block(genState)

tests/test_beaconstate.nim

@@ -12,7 +12,7 @@ import
   ../beacon_chain/spec/[beaconstate, datatypes, digest]
 
 suite "Beacon state" & preset():
-  test "Smoke test get_genesis_beacon_state" & preset():
-    let state = get_genesis_beacon_state(
+  test "Smoke test initialize_beacon_state_from_eth1" & preset():
+    let state = initialize_beacon_state_from_eth1(
       makeInitialDeposits(SLOTS_PER_EPOCH, {}), 0, Eth1Data(), {})
     check: state.validators.len == SLOTS_PER_EPOCH

tests/test_block_pool.nim

@@ -13,7 +13,7 @@ import
 
 suite "Block pool processing" & preset():
   let
-    genState = get_genesis_beacon_state(
+    genState = initialize_beacon_state_from_eth1(
       makeInitialDeposits(flags = {skipValidation}), 0, Eth1Data(),
         {skipValidation})
     genBlock = get_initial_beacon_block(genState)

tests/test_state_transition.nim

@@ -18,7 +18,7 @@ suite "Block processing" & preset():
   let
     # Genesis state with minimal number of deposits
     # TODO bls verification is a bit of a bottleneck here
-    genesisState = get_genesis_beacon_state(
+    genesisState = initialize_beacon_state_from_eth1(
       makeInitialDeposits(), 0, Eth1Data(), {})
     genesisBlock = get_initial_beacon_block(genesisState)
     genesisRoot = signing_root(genesisBlock)