convert some asserts to doAsserts to keep them in release mode builds; rename get_initial_beacon_state to get_genesis_beacon_state to track spec; switch target spec version to 0.3.0; switch references to penalize_validator to slash_validator/slashValidator to track spec; make some function returns safer by omitting 'return' (#132)

* convert some asserts to doAsserts to keep them in release mode builds; rename get_initial_beacon_state to get_genesis_beacon_state to track spec; switch target spec version to 0.3.0; switch references to penalize_validator to slash_validator/slashValidator to track spec; make some function returns safer by omitting 'return'

* 2x shuffling speedup by hoisting pivot calculations per https://github.com/protolambda/eth2-shuffle

beacon_chain/spec/beaconstate.nim

@@ -132,7 +132,7 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
   ## Note that this function mutates ``state``.
 
   let validator = addr state.validator_registry[index]
-  assert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
+  doAssert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
   ## [TO BE REMOVED IN PHASE 2]
 
   exit_validator(state, index)
@@ -153,12 +153,13 @@ func slash_validator*(state: var BeaconState, index: ValidatorIndex) =
   validator.withdrawable_epoch = get_current_epoch(state) +
     LATEST_SLASHED_EXIT_LENGTH
 
-func get_initial_beacon_state*(
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#on-genesis
+func get_genesis_beacon_state*(
     initial_validator_deposits: openArray[Deposit],
     genesis_time: uint64,
     latest_eth1_data: Eth1Data,
     flags: UpdateFlags = {}): BeaconState =
-  ## Get the initial ``BeaconState``.
+  ## Get the genesis ``BeaconState``.
   ##
   ## Before the beacon chain starts, validators will register in the Eth1 chain
   ## and deposit ETH. When enough many validators have registered, a
@@ -186,10 +187,12 @@ func get_initial_beacon_state*(
 
     validator_registry_update_epoch: GENESIS_EPOCH,
 
+    # validator_registry and validator_balances automatically initalized
     # TODO remove or conditionally compile; not in spec anymore
     validator_registry_delta_chain_tip: ZERO_HASH,
 
     # Randomness and committees
+    # latest_randao_mixes automatically initialized
     previous_shuffling_start_shard: GENESIS_START_SHARD,
     current_shuffling_start_shard: GENESIS_START_SHARD,
     previous_shuffling_epoch: GENESIS_EPOCH,
@@ -205,6 +208,11 @@ func get_initial_beacon_state*(
 
     # Deposit root
     latest_eth1_data: latest_eth1_data,
+
+    # Recent state
+    # TODO properly initialize latest_crosslinks
+    # latest_block_roots, latest_active_index_roots, latest_slashed_balances,
+    # latest_attestations, and batched_block_roots automatically initialized.
   )
 
   # Process initial deposits
@@ -223,6 +231,12 @@ func get_initial_beacon_state*(
     if get_effective_balance(state, vi) >= MAX_DEPOSIT_AMOUNT:
       activate_validator(state, vi, true)
 
+  let genesis_active_index_root = Eth2Digest(data: hash_tree_root(
+    get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)))
+  for index in 0 ..< LATEST_ACTIVE_INDEX_ROOTS_LENGTH:
+    state.latest_active_index_roots[index] = genesis_active_index_root
+  state.current_shuffling_seed = generate_seed(state, GENESIS_EPOCH)
+
   state
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_block_root

beacon_chain/spec/datatypes.nim

@@ -40,7 +40,7 @@ import
 # TODO Many of these constants should go into a config object that can be used
 #      to run.. well.. a chain with different constants!
 const
-  SPEC_VERSION* = "0.2.0" ## \
+  SPEC_VERSION* = "0.3.0" ## \
   ## Spec version we're aiming to be compatible with, right now
   ## TODO: improve this scheme once we can negotiate versions in protocol
 
@@ -146,7 +146,7 @@ const
   # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#state-list-lengths
   LATEST_BLOCK_ROOTS_LENGTH* = 2'u64^13
   LATEST_RANDAO_MIXES_LENGTH* = 2'u64^13
-  LATEST_ACTIVE_INDEX_ROOTS_LENGTH* = 2'u64^13
+  LATEST_ACTIVE_INDEX_ROOTS_LENGTH* = 8192 # 2'u64^13, epochs
   LATEST_SLASHED_EXIT_LENGTH* = 8192 # epochs
 
   # Reward and penalty quotients

beacon_chain/spec/helpers.nim

@@ -88,12 +88,13 @@ func is_power_of_2*(v: uint64): bool = (v and (v-1)) == 0
 func merkle_root*(values: openArray[Eth2Digest]): Eth2Digest =
   ## Merkleize ``values`` (where ``len(values)`` is a power of two) and return
   ## the Merkle root.
+  ## https://crypto.stackexchange.com/questions/43430/what-is-the-reason-to-separate-domains-in-the-internal-hash-algorithm-of-a-merkl
   let num_values = len(values)
 
   # Simplifies boundary conditions
-  assert is_power_of_two(num_values)
+  doAssert is_power_of_two(num_values)
-  assert num_values >= 2
+  doAssert num_values >= 2
-  assert num_values mod 2 == 0
+  doAssert num_values mod 2 == 0
 
   # TODO reverse ``o`` order and use newSeqWith to avoid pointless zero-filling.
   var o = repeat(ZERO_HASH, len(values))

beacon_chain/spec/validator.nim

@@ -12,21 +12,15 @@ import
   ../ssz,
   ./crypto, ./datatypes, ./digest, ./helpers
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#get_permuted_index
+# TODO remove once there are test vectors to check with directly
-func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest): uint64 =
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_permuted_index
+func get_permuted_index_spec(index: uint64, list_size: uint64, seed: Eth2Digest): uint64 =
   ## Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1`
   ## with ``seed`` as entropy.
   ##
   ## Utilizes 'swap or not' shuffling found in
   ## https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
   ## See the 'generalized domain' algorithm on page 3.
-  ##
-  ## This is very slow. Lots of pointless memory allocations. Some methods
-  ## can use updating-with-incremental-chunks-of-preallocated source data,
-  ## while others might have caller-allocated buffers, etc. What this does
-  ## is egregious though. It is allocating tens of thousands of identical,
-  ## small buffers, etc. There's nothing fundamentally inefficient about a
-  ## per-index approach, but naive implementations require optimization.
   result = index
   var pivot_buffer: array[(32+1), byte]
   var source_buffer: array[(32+1+4), byte]
@@ -55,21 +49,14 @@ func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest): uin
     if bit != 0:
       result = flip
 
+# TODO remove once there are test vectors to check with directly
 # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
-func get_shuffling*(seed: Eth2Digest,
+func get_shuffling_spec*(seed: Eth2Digest, validators: openArray[Validator],
-                    validators: openArray[Validator],
+                         epoch: Epoch): seq[seq[ValidatorIndex]] =
-                    epoch: Epoch
-                    ): seq[seq[ValidatorIndex]] =
   ## Shuffles ``validators`` into crosslink committees seeded by ``seed`` and
   ## ``slot``.
   ## Returns a list of ``SLOTS_PER_EPOCH * committees_per_slot`` committees where
   ## each committee is itself a list of validator indices.
-  ##
-  ## TODO: write unit tests for if this produces an "interesting" permutation
-  ## i.e. every number in input exists once and >95%, say, in a different
-  ## place than it began (there are more rigorous approaches, but something)
-  ## By design, get_permuted_index is somewhat difficult to test on its own;
-  ## get_shuffling is first layer where that's straightforward.
 
   let
     active_validator_indices = get_active_validator_indices(validators, epoch)
@@ -79,13 +66,93 @@ func get_shuffling*(seed: Eth2Digest,
 
     shuffled_active_validator_indices = mapIt(
       active_validator_indices,
-      active_validator_indices[get_permuted_index(
+      active_validator_indices[get_permuted_index_spec(
         it, len(active_validator_indices).uint64, seed).int])
 
   # Split the shuffled list into committees_per_epoch pieces
   result = split(shuffled_active_validator_indices, committees_per_epoch)
   assert result.len() == committees_per_epoch # what split should do..
 
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_permuted_index
+func get_permuted_index(index: uint64, list_size: uint64, seed: Eth2Digest,
+                        pivots: seq[uint64]): uint64 =
+  ## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go
+  ## Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1`
+  ## with ``seed`` as entropy.
+  ##
+  ## Utilizes 'swap or not' shuffling found in
+  ## https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+  ## See the 'generalized domain' algorithm on page 3.
+  result = index
+  var source_buffer: array[(32+1+4), byte]
+
+  doAssert len(pivots) == SHUFFLE_ROUND_COUNT
+
+  for round in 0 ..< SHUFFLE_ROUND_COUNT:
+    let
+      pivot = pivots[round]
+      flip = (pivot - index) mod list_size
+      position = max(index, flip)
+      round_bytes1 = int_to_bytes1(round)[0]
+
+    ## Tradeoff between slicing (if reusing one larger buffer) and additional
+    ## copies here of seed and `int_to_bytes1(round)`.
+    source_buffer[0..31] = seed.data
+    source_buffer[32] = round_bytes1
+    source_buffer[33..36] = int_to_bytes4(position div 256)
+
+    let
+      source = eth2hash(source_buffer).data
+      byte_value = source[(position mod 256) div 8]
+      bit = (byte_value shr (position mod 8)) mod 2
+
+    if bit != 0:
+      result = flip
+
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#get_shuffling
+func get_shuffling*(seed: Eth2Digest,
+                    validators: openArray[Validator],
+                    epoch: Epoch
+                    ): seq[seq[ValidatorIndex]] =
+  ## Via https://github.com/protolambda/eth2-shuffle/blob/master/shuffle.go
+  ## Shuffles ``validators`` into crosslink committees seeded by ``seed`` and
+  ## ``slot``.
+  ## Returns a list of ``SLOTS_PER_EPOCH * committees_per_slot`` committees where
+  ## each committee is itself a list of validator indices.
+
+  ## The pivot's a function of seed and round only, so precalculate all
+  ## SHUFFLE_ROUND_COUNT pivots, using one buffer.
+  let
+    active_validator_indices = get_active_validator_indices(validators, epoch)
+    list_size = active_validator_indices.len.uint64
+  var
+    pivot_buffer: array[(32+1), byte]
+
+    # Allow Nim stdlib to preallocate the correct seq size.
+    pivots = repeat(0'u64, SHUFFLE_ROUND_COUNT)
+
+  # This doesn't change across rounds.
+  pivot_buffer[0..31] = seed.data
+
+  for round in 0 ..< SHUFFLE_ROUND_COUNT:
+    let round_bytes1 = int_to_bytes1(round)[0]
+    pivot_buffer[32] = round_bytes1
+    let pivot = bytes_to_int(eth2hash(pivot_buffer).data[0..7]) mod list_size
+    pivots[round] = pivot
+
+  let
+    committees_per_epoch = get_epoch_committee_count(
+      len(active_validator_indices)).int
+
+    shuffled_active_validator_indices = mapIt(
+      active_validator_indices,
+      active_validator_indices[get_permuted_index(
+        it, len(active_validator_indices).uint64, seed, pivots).int])
+
+  # Split the shuffled list into committees_per_epoch pieces
+  result = split(shuffled_active_validator_indices, committees_per_epoch)
+  assert result.len() == committees_per_epoch # what split should do..
+
 func get_new_validator_registry_delta_chain_tip*(
     current_validator_registry_delta_chain_tip: Eth2Digest,
     index: ValidatorIndex,

beacon_chain/state_transition.nim

@@ -39,7 +39,7 @@ func flatten[T](v: openArray[seq[T]]): seq[T] =
   # TODO not in nim - doh.
   for x in v: result.add x
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#proposer-signature
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#proposer-signature
 func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
   ## When creating a block, the proposer will sign a version of the block that
   ## doesn't contain the data (chicken and egg), then add the signature to that
@@ -62,7 +62,7 @@ func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
     proposal_root.data, blck.signature,
     get_domain(state.fork, get_current_epoch(state), DOMAIN_PROPOSAL))
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#randao
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#randao
 proc processRandao(
     state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
   let
@@ -93,9 +93,9 @@ proc processRandao(
 
   true
 
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#eth1-data
 func processDepositRoot(state: var BeaconState, blck: BeaconBlock) =
-  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1-data
+  # TODO verify that there's at most one match
-
   for x in state.eth1_data_votes.mitems():
     if blck.eth1_data == x.eth1_data:
       x.vote_count += 1
@@ -106,28 +106,34 @@ func processDepositRoot(state: var BeaconState, blck: BeaconBlock) =
     vote_count: 1
   )
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#penalize_validator
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slashValidator
-func penalizeValidator(state: var BeaconState, index: ValidatorIndex) =
+func slashValidator(state: var BeaconState, index: ValidatorIndex) =
-  ## Penalize the validator of the given ``index``.
+  ## Slash the validator of the given ``index``.
   ## Note that this function mutates ``state``.
-  exit_validator(state, index)
   var validator = addr state.validator_registry[index]
+
+  doAssert state.slot < get_epoch_start_slot(validator.withdrawable_epoch) ##\
+  ## [TO BE REMOVED IN PHASE 2]
+
+  exit_validator(state, index)
   state.latest_slashed_balances[(get_current_epoch(state) mod
     LATEST_SLASHED_EXIT_LENGTH).int] += get_effective_balance(state,
       index.ValidatorIndex)
 
   let
     whistleblower_index = get_beacon_proposer_index(state, state.slot)
-    whistleblower_reward = get_effective_balance(state, index) div WHISTLEBLOWER_REWARD_QUOTIENT
+    whistleblower_reward = get_effective_balance(state, index) div
+      WHISTLEBLOWER_REWARD_QUOTIENT
   state.validator_balances[whistleblower_index] += whistleblower_reward
   state.validator_balances[index] -= whistleblower_reward
   validator.slashed_epoch = get_current_epoch(state)
+
+  # v0.3.0 spec bug, fixed later, involving renamed constants. Use v0.3.0 name.
   validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH
 
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#proposer-slashings-1
 proc processProposerSlashings(
     state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
-  ## https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposer-slashings-1
-
   if len(blck.body.proposer_slashings) > MAX_PROPOSER_SLASHINGS:
     notice "PropSlash: too many!",
       proposer_slashings = len(blck.body.proposer_slashings)
@@ -174,7 +180,7 @@ proc processProposerSlashings(
       notice "PropSlash: penalized slot"
       return false
 
-    penalizeValidator(state, proposer_slashing.proposer_index.ValidatorIndex)
+    slashValidator(state, proposer_slashing.proposer_index.ValidatorIndex)
 
   return true
 
@@ -227,8 +233,8 @@ func verify_slashable_attestation(state: BeaconState, slashable_attestation: Sla
     ),
   )
 
+# https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#attester-slashings-1
 proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock): bool =
-  ## https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attester-slashings-1
   if len(blck.body.attester_slashings) > MAX_ATTESTER_SLASHINGS:
     notice "CaspSlash: too many!"
     return false
@@ -267,9 +273,9 @@ proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock): bool =
 
     for index in slashable_indices:
       if state.validator_registry[index.int].slashed_epoch > get_current_epoch(state):
-        penalize_validator(state, index.ValidatorIndex)
+        slash_validator(state, index.ValidatorIndex)
 
-  return true
+  true
 
 proc processAttestations(
     state: var BeaconState, blck: BeaconBlock, flags: UpdateFlags): bool =
@@ -355,7 +361,11 @@ func processSlot(state: var BeaconState, previous_block_root: Eth2Digest) =
   ## slot, a proposer creates a block to represent the state of the beacon
   ## chain at that time. In case the proposer is missing, it may happen that
   ## the no block is produced during the slot.
+
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slot
   state.slot += 1
+
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#block-roots
   state.latest_block_roots[(state.slot - 1) mod LATEST_BLOCK_ROOTS_LENGTH] =
     previous_block_root
   if state.slot mod LATEST_BLOCK_ROOTS_LENGTH == 0:
@@ -369,7 +379,7 @@ proc processBlock(
   # TODO when there's a failure, we should reset the state!
   # TODO probably better to do all verification first, then apply state changes
 
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.2.0/specs/core/0_beacon-chain.md#slot-1
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.3.0/specs/core/0_beacon-chain.md#slot-1
   if not (blck.slot == state.slot):
     notice "Unexpected block slot number",
       blockSlot = blck.slot,

beacon_chain/trusted_state_snapshots.nim

@@ -31,7 +31,7 @@ proc obtainTrustedStateSnapshot*(db: BeaconChainDB): Future[BeaconState] {.async
 
 proc createStateSnapshot*(
     startup: ChainStartupData, genesisOffset: int, outFile: string) =
-  let initialState = get_initial_beacon_state(
+  let initialState = get_genesis_beacon_state(
     startup.validatorDeposits,
     uint64(int(fastEpochTime() div 1000) + genesisOffset),
     Eth1Data(), {})

research/serialized_sizes.nim

@@ -5,7 +5,7 @@ import
   ../tests/testutil
 
 proc stateSize(deposits: int, maxContent = false) =
-  var state = get_initial_beacon_state(
+  var state = get_genesis_beacon_state(
     makeInitialDeposits(deposits), 0, Eth1Data(), {skipValidation})
 
   if maxContent:

research/state_sim.nim

@@ -24,7 +24,7 @@ cli do(slots = 1945,
        validate = false):
   let
     flags = if validate: {} else: {skipValidation}
-    genesisState = get_initial_beacon_state(
+    genesisState = get_genesis_beacon_state(
       makeInitialDeposits(validators, flags), 0, Eth1Data(), flags)
     genesisBlock = makeGenesisBlock(genesisState)
 

tests/test_attestation_pool.nim

@@ -18,7 +18,7 @@ suite "Attestation pool processing":
   let
     # Genesis state with minimal number of deposits
     # TODO bls verification is a bit of a bottleneck here
-    genesisState = get_initial_beacon_state(
+    genesisState = get_genesis_beacon_state(
       makeInitialDeposits(), 0, Eth1Data(), {skipValidation})
     genesisBlock = makeGenesisBlock(genesisState)
     genesisRoot = hash_tree_root_final(genesisBlock)

tests/test_beaconstate.nim

@@ -12,7 +12,7 @@ import
   ../beacon_chain/spec/[beaconstate, datatypes, digest]
 
 suite "Beacon state":
-  test "Smoke test get_initial_beacon_state":
+  test "Smoke test get_genesis_beacon_state":
-    let state = get_initial_beacon_state(
+    let state = get_genesis_beacon_state(
       makeInitialDeposits(SLOTS_PER_EPOCH, {}), 0, Eth1Data(), {})
     check: state.validator_registry.len == SLOTS_PER_EPOCH

tests/test_state_transition.nim

@@ -18,7 +18,7 @@ suite "Block processing":
   let
     # Genesis state with minimal number of deposits
     # TODO bls verification is a bit of a bottleneck here
-    genesisState = get_initial_beacon_state(
+    genesisState = get_genesis_beacon_state(
       makeInitialDeposits(), 0, Eth1Data(), {})
     genesisBlock = makeGenesisBlock(genesisState)
 

tests/test_validator.nim

@@ -28,8 +28,12 @@ suite "Validators":
           exit_epoch: FAR_FUTURE_EPOCH
         ), num_validators)
       s = get_shuffling(Eth2Digest(), validators, 0)
+      #s_spec = get_shuffling_spec(Eth2Digest(), validators, 0)
       committees = get_epoch_committee_count(len(validators)).int
     check:
+      ## Enable checking equivalence of spec and optimized versions.
+      ## TODO enable checking against YAML test vectors
+      ## s == s_spec
       s.len == committees
        # 32k validators: SLOTS_PER_EPOCH slots * committee_count_per_slot =
        # get_epoch_committee_count committees.