flip targeted spec version and ssz test fixtures to 0.9.0; almost completely update BeaconState to 0.9.0; update minimal and mainnet preset constants to 0.9.0; update process_proposer_slashing(...), get_attestation_deltas(...), and get_matching_head_attestations(...) to 0.9.0; mark process_final_updates(...) as 0.9.0

beacon_chain/spec/datatypes.nim

@@ -53,7 +53,7 @@ else:
   {.fatal: "Preset \"" & const_preset ".nim\" is not supported.".}
 
 const
-  SPEC_VERSION* = "0.8.4" ## \
+  SPEC_VERSION* = "0.9.0" ## \
   ## Spec version we're aiming to be compatible with, right now
   ## TODO: improve this scheme once we can negotiate versions in protocol
 
@@ -265,7 +265,7 @@ type
     current_justified_checkpoint*: Checkpoint
     finalized_checkpoint*: Checkpoint
 
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#beaconstate
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#beaconstate
   BeaconState* = object
     # Versioning
     genesis_time*: uint64
@@ -301,15 +301,9 @@ type
     ## VALIDATOR_REGISTRY_LIMIT
 
     # Shuffling
-    start_shard*: Shard
+    start_shard* {.dontSerialize.}: Shard
     randao_mixes*: array[EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest]
 
-    active_index_roots*: array[EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest] ##\
-    ## Active index digests for light clients
-
-    compact_committees_roots*: array[EPOCHS_PER_HISTORICAL_VECTOR, Eth2Digest] ##\
-    ## Committee digests for light clients
-
     # Slashings
     slashings*: array[EPOCHS_PER_SLASHINGS_VECTOR, uint64] ##\
     ## Per-epoch sums of slashed effective balances
@@ -357,7 +351,10 @@ type
   PendingAttestation* = object
     aggregation_bits*: CommitteeValidatorsBits
     data*: AttestationData
+
+    # TODO this is a Slot
     inclusion_delay*: uint64
+
     proposer_index*: uint64
 
   # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#historicalbatch
@@ -392,6 +389,8 @@ type
     active_validator_indices_cache*:
       Table[Epoch, seq[ValidatorIndex]]
     start_shard_cache*: Table[Epoch, Shard]
+
+    # TODO still used?
     committee_count_cache*: Table[Epoch, uint64]
 
 when networkBackend == rlpxBackend:

beacon_chain/spec/helpers.nim

@@ -28,41 +28,6 @@ func integer_squareroot*(n: SomeInteger): SomeInteger =
     y = (x + n div x) div 2
   x
 
-# TODO reuse as necessary/useful for merkle proof building
-func merkle_root*(values: openArray[Eth2Digest]): Eth2Digest =
-  ## Merkleize ``values`` (where ``len(values)`` is a power of two) and return
-  ## the Merkle root.
-  ## https://crypto.stackexchange.com/questions/43430/what-is-the-reason-to-separate-domains-in-the-internal-hash-algorithm-of-a-merkl
-  let num_values = len(values)
-
-  # Simplifies boundary conditions
-  doAssert is_power_of_two(num_values)
-  doAssert num_values >= 2
-  doAssert num_values mod 2 == 0
-
-  # TODO reverse ``o`` order and use newSeqWith to avoid pointless zero-filling.
-  var o = repeat(ZERO_HASH, len(values))
-  var hash_buffer: array[2*32, byte]
-
-  # These ``o`` indices get filled from ``values``.
-  let highest_internally_filled_index = (num_values div 2) - 1
-  doAssert (highest_internally_filled_index + 1) * 2 >= num_values
-
-  for i in countdown(num_values-1, highest_internally_filled_index + 1):
-    hash_buffer[0..31] = values[i*2 - num_values].data
-    hash_buffer[32..63] = values[i*2+1 - num_values].data
-    o[i] = eth2hash(hash_buffer)
-
-  ## These ``o`` indices get filled from other ``o`` indices.
-  doAssert highest_internally_filled_index * 2 + 1 < num_values
-
-  for i in countdown(highest_internally_filled_index, 1):
-    hash_buffer[0..31] = o[i*2].data
-    hash_buffer[32..63] = o[i*2+1].data
-    o[i] = eth2hash(hash_buffer)
-
-  o[1]
-
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#compute_epoch_at_slot
 func compute_epoch_at_slot*(slot: Slot|uint64): Epoch =
   # Return the epoch number of the given ``slot``.
@@ -197,7 +162,5 @@ func get_seed*(state: BeaconState, epoch: Epoch): Eth2Digest =
   seed_input[0..31] =
     get_randao_mix(state,
       epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1).data
-  seed_input[32..63] =
-    state.active_index_roots[epoch mod EPOCHS_PER_HISTORICAL_VECTOR].data
   seed_input[64..95] = int_to_bytes32(epoch)
   eth2hash(seed_input)

beacon_chain/spec/presets/mainnet.nim

@@ -50,8 +50,8 @@ const
   SHUFFLE_ROUND_COUNT* = 90
 
   # Constants (TODO: not actually configurable)
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#constants
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#constants
-  BASE_REWARDS_PER_EPOCH* = 5
+  BASE_REWARDS_PER_EPOCH* = 4
 
   DEPOSIT_CONTRACT_TREE_DEPTH* = 32
 
@@ -98,7 +98,7 @@ const
   ## wait towards the end of the slot and still have time to publish the
   ## attestation.
 
-  SLOTS_PER_EPOCH* {.intdefine.} = 64 ##\
+  SLOTS_PER_EPOCH* {.intdefine.} = 32 ##\
   ## (~6.4 minutes)
   ## slots that make up an epoch, at the end of which more heavy
   ## processing is done
@@ -132,7 +132,7 @@ const
 
   # State vector lengths
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/configs/mainnet.yaml#L81
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/configs/mainnet.yaml#L81
   EPOCHS_PER_HISTORICAL_VECTOR* = 65536
   EPOCHS_PER_SLASHINGS_VECTOR* = 8192
   HISTORICAL_ROOTS_LIMIT* = 16777216
@@ -140,7 +140,7 @@ const
 
   # Reward and penalty quotients
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#rewards-and-penalties
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/configs/mainnet.yaml#L93
   BASE_REWARD_FACTOR* = 2'u64^6
   WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9
   PROPOSER_REWARD_QUOTIENT* = 2'u64^3

beacon_chain/spec/presets/minimal.nim

@@ -31,7 +31,6 @@ const
   MAX_VALIDATORS_PER_COMMITTEE* = 4096
   MIN_PER_EPOCH_CHURN_LIMIT* = 4
   CHURN_LIMIT_QUOTIENT* = 2^16
-  BASE_REWARDS_PER_EPOCH* = 5
 
   # Changed
   SHUFFLE_ROUND_COUNT* = 10
@@ -43,6 +42,8 @@ const
   # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#constants
   # TODO "The following values are (non-configurable) constants" ...
   # Unchanged
+  BASE_REWARDS_PER_EPOCH* = 4
+
   DEPOSIT_CONTRACT_TREE_DEPTH* = 32
 
   # Gwei values
@@ -57,7 +58,7 @@ const
 
   # Initial values
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/configs/minimal.yaml#L44
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/configs/minimal.yaml#L44
 
   # Unchanged
   GENESIS_SLOT* = 0.Slot
@@ -91,7 +92,7 @@ const
 
   # State vector lengths
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/configs/minimal.yaml#L79
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/configs/minimal.yaml#L83
 
   # Changed
   EPOCHS_PER_HISTORICAL_VECTOR* = 64
@@ -101,9 +102,8 @@ const
 
   # Reward and penalty quotients
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#rewards-and-penalties
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/configs/minimal.yaml#L95
 
-  # Unchanged
   BASE_REWARD_FACTOR* = 2'u64^6
   WHISTLEBLOWER_REWARD_QUOTIENT* = 2'u64^9
   PROPOSER_REWARD_QUOTIENT* = 2'u64^3

beacon_chain/spec/state_transition_block.nim

@@ -145,7 +145,7 @@ func is_slashable_validator(validator: Validator, epoch: Epoch): bool =
     (validator.activation_epoch <= epoch) and
     (epoch < validator.withdrawable_epoch)
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#proposer-slashings
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#proposer-slashings
 proc process_proposer_slashing*(
     state: var BeaconState, proposer_slashing: ProposerSlashing,
     flags: UpdateFlags, stateCache: var StateCache): bool =
@@ -155,10 +155,10 @@ proc process_proposer_slashing*(
 
   let proposer = state.validators[proposer_slashing.proposer_index.int]
 
-  # Verify that the epoch is the same
+  # Verify slots match
-  if not (compute_epoch_at_slot(proposer_slashing.header_1.slot) ==
+  if not (proposer_slashing.header_1.slot ==
-      compute_epoch_at_slot(proposer_slashing.header_2.slot)):
+      proposer_slashing.header_2.slot):
-    notice "Proposer slashing: epoch mismatch"
+    notice "Proposer slashing: slot mismatch"
     return false
 
   # But the headers are different
@@ -257,7 +257,7 @@ proc process_attester_slashing*(
       return false
     return true
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#attester-slashings
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#attester-slashings
 proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock,
     stateCache: var StateCache): bool =
   # Process ``AttesterSlashing`` operation.

beacon_chain/spec/state_transition_epoch.nim

@@ -62,7 +62,7 @@ func get_total_active_balance*(state: BeaconState): Gwei =
     state,
     get_active_validator_indices(state, get_current_epoch(state)))
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#helper-functions-1
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#helper-functions-1
 func get_matching_source_attestations(state: BeaconState, epoch: Epoch):
     seq[PendingAttestation] =
   doAssert epoch in [get_current_epoch(state), get_previous_epoch(state)]
@@ -83,7 +83,7 @@ func get_matching_head_attestations(state: BeaconState, epoch: Epoch):
   filterIt(
      get_matching_source_attestations(state, epoch),
      it.data.beacon_block_root ==
-       get_block_root_at_slot(state, get_attestation_data_slot(state, it.data))
+       get_block_root_at_slot(state, it.data.slot)
   )
 
 func get_attesting_balance(
@@ -232,7 +232,7 @@ func get_base_reward(state: BeaconState, index: ValidatorIndex): Gwei =
   effective_balance * BASE_REWARD_FACTOR div
     integer_squareroot(total_balance) div BASE_REWARDS_PER_EPOCH
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#rewards-and-penalties-1
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#rewards-and-penalties-1
 func get_attestation_deltas(state: BeaconState, stateCache: var StateCache):
     tuple[a: seq[Gwei], b: seq[Gwei]] =
   let
@@ -308,10 +308,7 @@ func get_attestation_deltas(state: BeaconState, stateCache: var StateCache):
     rewards[attestation.proposer_index.int] += proposer_reward
     let max_attester_reward = get_base_reward(state, index) - proposer_reward
 
-    rewards[index] +=
+    rewards[index] += max_attester_reward div attestation.inclusion_delay
-      ((max_attester_reward *
-       ((SLOTS_PER_EPOCH + MIN_ATTESTATION_INCLUSION_DELAY).uint64 -
-       attestation.inclusion_delay)) div SLOTS_PER_EPOCH).Gwei
 
   # Inactivity penalty
   let finality_delay = previous_epoch - state.finalized_checkpoint.epoch
@@ -358,7 +355,7 @@ func process_slashings*(state: var BeaconState) =
       let penalty = penalty_numerator div total_balance * increment
       decrease_balance(state, index.ValidatorIndex, penalty)
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#final-updates
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_beacon-chain.md#final-updates
 proc process_final_updates*(state: var BeaconState) =
   let
     current_epoch = get_current_epoch(state)
@@ -394,6 +391,8 @@ proc process_final_updates*(state: var BeaconState) =
     )
     state.historical_roots.add (hash_tree_root(historical_batch))
 
+  # TODO remove this when start_shard finally goes away, but doesn't
+  # interfere with 0.9.0. Gone after 0.8.4.
   # Update start shard
   state.start_shard =
     (state.start_shard + get_shard_delta(state, current_epoch)) mod

tests/official/fixtures_utils.nim

@@ -37,7 +37,7 @@ proc readValue*(r: var JsonReader, a: var seq[byte]) {.inline.} =
 const
   FixturesDir* = currentSourcePath.rsplit(DirSep, 1)[0] / "fixtures"
   JsonTestsDir* = FixturesDir/"json_tests_v0.8.3"
-  SszTestsDir* = FixturesDir/"tests-v0.8.3"
+  SszTestsDir* = FixturesDir/"tests-v0.9.0"
 
 proc parseTest*(path: string, Format: typedesc[Json or SSZ], T: typedesc): T =
   try:

tests/official/test_fixture_sanity_blocks.nim

@@ -44,7 +44,12 @@ template runValidTest(testName: string, identifier: untyped, num_blocks: int): u
 
       # Checks:
       # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-      reportDiff(stateRef, postRef)
+      var sr_pre, sr_post: ref BeaconStateNew
+      new sr_pre
+      new sr_post
+      sr_pre[] = GetNewBeaconState(stateRef[])
+      sr_post[] = GetNewBeaconState(postRef[])
+      reportDiff(sr_pre, sr_post)
 
   `testImpl _ blck _ identifier`()
 

tests/official/test_fixture_sanity_slots.nim

@@ -36,7 +36,13 @@ template runTest(testName: string, identifier: untyped, num_slots: uint64): unty
 
       process_slots(stateRef[], stateRef.slot + num_slots)
       # check: stateRef.hash_tree_root() == postRef.hash_tree_root()
-      reportDiff(stateRef, postRef)
+
+      var sr_pre, sr_post: ref BeaconStateNew
+      new sr_pre
+      new sr_post
+      sr_pre[] = GetNewBeaconState(stateRef[])
+      sr_post[] = GetNewBeaconState(postRef[])
+      reportDiff(sr_pre, sr_post)
 
   `testImpl _ slots _ identifier`()
 

tests/official/test_fixture_state_transition_epoch.nim

@@ -48,7 +48,12 @@ template runSuite(suiteDir, testName: string, transitionProc: untyped{ident}, us
           else:
             transitionProc(stateRef[])
 
-          reportDiff(stateRef, postRef)
+          var sr_pre, sr_post: ref BeaconStateNew
+          new sr_pre
+          new sr_post
+          sr_pre[] = GetNewBeaconState(stateRef[])
+          sr_post[] = GetNewBeaconState(postRef[])
+          reportDiff(sr_pre, sr_post)
 
   `suiteImpl _ transitionProc`()