fix network sim finalization; remove get_attestation_data_slot(...); remove 2 more get_crosslink_committee(...) calls

beacon_chain/attestation_pool.nim

@@ -164,7 +164,7 @@ proc add*(pool: var AttestationPool,
   # TODO inefficient data structures..
 
   let
-    attestationSlot = get_attestation_data_slot(state, attestation.data)
+    attestationSlot = attestation.data.slot
     idx = pool.slotIndex(state, attestationSlot)
     slotData = addr pool.slots[idx]
     validation = Validation(

beacon_chain/beacon_node.nim

@@ -340,6 +340,16 @@ proc proposeBlock(node: BeaconNode,
       cat = "fastforward"
     return head
 
+  if head.slot == 0 and slot == 0:
+    # TODO there's been a startup assertion, which sometimes (but not always
+    # evidently) crashes exactly one node on simulation startup, the one the
+    # beacon chain proposer index points to first for slot 0. it tries using
+    # slot 0 as required, notices head block's slot is also 0 (which, that's
+    # how it's created; it's never less), and promptly fails, with assertion
+    # occuring downstream via async code. This is most easily reproduced via
+    # make clean_eth2_network_simulation_files && make eth2_network_simulation
+    return head
+
   if head.slot == slot:
     # Weird, we should never see as head the same slot as we're proposing a
     # block for - did someone else steal our slot? why didn't we discard it?

beacon_chain/spec/beaconstate.nim

@@ -258,54 +258,6 @@ func get_initial_beacon_block*(state: BeaconState): BeaconBlock =
     # parent_root, randao_reveal, eth1_data, signature, and body automatically
     # initialized to default values.
 
-# TODO remove when shim layer isn't needed
-func get_epoch_and_shard(
-    state: BeaconState, slot: Slot, committee_index: uint64): auto =
-  # linearize, get compute_committee(...) index, then back-calculate
-  # what shard to use for the given epoch:
-  # (shard + SHARD_COUNT - get_start_shard(state, epoch)) mod SHARD_COUNT
-  let
-    committees_per_slot = get_committee_count_at_slot(state, slot)
-    gbc_index =
-      (slot mod SLOTS_PER_EPOCH) * committees_per_slot + committee_index
-    epoch = slot.compute_epoch_at_slot
-
-  # `index` is constructed as
-  # mod get_committee_count_at_slot(state, compute_start_slot_at_epoch(epoch))
-  # which is <= MAX_COMMITTEES_PER_SLOT (4 in minimal, 64 in mainnet).
-  # get_committee_count_at_slot(...) is constant across an epoch, so:
-  let
-    shard_offset_constant =
-      SHARD_COUNT.uint64 - get_start_shard(state, epoch).uint64
-    shard =
-      (gbc_index + SHARD_COUNT - shard_offset_constant) mod SHARD_COUNT
-
-  let (roundtrip_slot, roundtrip_index) = get_slot_and_index(
-    state, epoch, shard)
-  doAssert roundtrip_slot == slot
-  doAssert committee_index == roundtrip_index
-
-  (epoch, shard)
-
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#get_attestation_data_slot
-func get_attestation_data_slot*(state: BeaconState,
-    data: AttestationData, committee_count: uint64): Slot =
-  # Return the slot corresponding to the attestation ``data``.
-  let
-    (epoch, shard) = get_epoch_and_shard(state, data.slot, data.index)
-    offset = (shard + SHARD_COUNT -
-      get_start_shard(state, epoch)) mod SHARD_COUNT
-
-  compute_start_slot_at_epoch(epoch) + offset div
-    (committee_count div SLOTS_PER_EPOCH)
-
-# This is the slower (O(n)), spec-compatible signature.
-func get_attestation_data_slot*(state: BeaconState,
-    data: AttestationData): Slot =
-  get_attestation_data_slot(
-    state, data, get_committee_count_at_slot(
-      state, data.target.epoch.compute_start_slot_at_epoch) * SLOTS_PER_EPOCH)
-
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#get_block_root_at_slot
 func get_block_root_at_slot*(state: BeaconState,
                              slot: Slot): Eth2Digest =
@@ -568,7 +520,7 @@ proc process_attestation*(
   #      Result[void, cstring] instead of logging in check_attestation?)
   if check_attestation(state, attestation, flags, stateCache):
     let
-      attestation_slot = get_attestation_data_slot(state, attestation.data)
+      attestation_slot = attestation.data.slot
       pending_attestation = PendingAttestation(
         data: attestation.data,
         aggregation_bits: attestation.aggregation_bits,

beacon_chain/spec/datatypes.nim

@@ -665,6 +665,12 @@ chronicles.formatIt Epoch: it.shortLog
 chronicles.formatIt BeaconBlock: it.shortLog
 chronicles.formatIt AttestationData: it.shortLog
 
+# TODO remove
+const SHARD_COUNT* = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH
+
+static:
+  doAssert SHARD_COUNT == MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH
+
 import json_serialization
 export json_serialization
 export writeValue, readValue

beacon_chain/spec/helpers.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard lib
-  sequtils, math, endians,
+  math, endians,
   # Third-party
   blscurve, # defines Domain
   # Internal

beacon_chain/spec/presets/mainnet.nim

@@ -20,13 +20,9 @@ type
 const
   # Misc
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#misc
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/configs/mainnet.yaml#L6
 
-  MAX_COMMITTEES_PER_SLOT* = 64
+  MAX_COMMITTEES_PER_SLOT* {.intdefine.} = 64
-
-  # TODO remove
-  SHARD_COUNT* = 2048 ##\
-  ## SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT
 
   TARGET_COMMITTEE_SIZE* = 2^7 ##\
   ## Number of validators in the committee attesting to one shard

beacon_chain/spec/presets/minimal.nim

@@ -20,16 +20,12 @@ type
 const
   # Misc
   # ---------------------------------------------------------------
-  # https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/configs/minimal.yaml#L4
+  # https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/configs/minimal.yaml#L4
 
   # Changed
   MAX_COMMITTEES_PER_SLOT* = 4
   TARGET_COMMITTEE_SIZE* = 4
 
-  # TODO remove
-  SHARD_COUNT* = 32 ##\
-  ## SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT
-
   # Unchanged
   MAX_VALIDATORS_PER_COMMITTEE* = 2048
   MIN_PER_EPOCH_CHURN_LIMIT* = 4

beacon_chain/spec/state_transition_epoch.nim

@@ -32,7 +32,7 @@
 # improvements to be made - other than that, keep things similar to spec for
 # now.
 
-import # TODO - cleanup imports
+import
   math, options, sequtils, tables,
   stew/[bitseqs, bitops2], chronicles, json_serialization/std/sets,
   metrics, ../ssz,
@@ -52,6 +52,13 @@ declareGauge beacon_current_justified_root, "Current justified root" # On epoch
 declareGauge beacon_previous_justified_epoch, "Current previously justified epoch" # On epoch transition
 declareGauge beacon_previous_justified_root, "Current previously justified root" # On epoch transition
 
+# Non-spec
+declareGauge epoch_transition_justification_and_finalization, "Epoch transition justification and finalization time"
+declareGauge epoch_transition_times_rewards_and_penalties, "Epoch transition reward and penalty time"
+declareGauge epoch_transition_registry_updates, "Epoch transition registry updates time"
+declareGauge epoch_transition_slashings, "Epoch transition slashings time"
+declareGauge epoch_transition_final_updates, "Epoch transition final updates time"
+
 # Spec
 # --------------------------------------------------------
 

beacon_chain/state_transition.nim

@@ -39,7 +39,6 @@ import
 # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#additional-metrics
 declareGauge beacon_current_validators, """Number of status="pending|active|exited|withdrawable" validators in current epoch""" # On epoch transition
 declareGauge beacon_previous_validators, """Number of status="pending|active|exited|withdrawable" validators in previous epoch""" # On epoch transition
-declareGauge beacon_previous_epoch_orphaned_blocks, "Number of blocks orphaned in the previous epoch" # On epoch transition
 
 # Canonical state transition functions
 # ---------------------------------------------------------------

research/state_sim.nim

@@ -125,8 +125,7 @@ cli do(slots = 448'u,
           # by the randomness. We have to delay when the attestation is
           # actually added to the block per the attestation delay rule!
           let target_slot =
-            get_attestation_data_slot(state, attestation.data) +
+            attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY - 1
-            MIN_ATTESTATION_INCLUSION_DELAY - 1
 
           ## In principle, should enumerate possible shard/slot combinations by
           ## inverting get_attestation_data_slot(...), but this works. Could be

tests/simulation/start.sh

@@ -19,9 +19,9 @@ NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --verbosity:0
 # Run with "SHARD_COUNT=4 ./start.sh" to change these
 DEFS=""
 
-DEFS+="-d:SHARD_COUNT=${SHARD_COUNT:-8} "      # Spec default: 1024
+DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} "      # Spec default: 64
-DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-8} "   # Spec default: 64
+DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} "   # Spec default: 32
-DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-4} "  # Spec default: 6
+DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} "  # Spec default: 12
 
 LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))
 LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.json"

tests/spec_epoch_processing/justification_finalization_helpers.nim

@@ -18,16 +18,6 @@ import
 # Justification and finalization utils
 # ---------------------------------------------------------------
 
-iterator getShardsForSlot(state: BeaconState, slot: Slot): Shard =
-  let
-    epoch = compute_epoch_at_slot(slot)
-    epoch_start_shard = get_start_shard(state, epoch)
-    committees_per_slot = get_committee_count_at_slot(state, slot)
-    shard = epoch_start_shard + committees_per_slot * (slot mod SLOTS_PER_EPOCH)
-
-  for i in 0 ..< committees_per_slot.int:
-    yield shard + Shard(i)
-
 proc addMockAttestations*(
        state: var BeaconState, epoch: Epoch,
        source, target: Checkpoint,
@@ -52,15 +42,12 @@ proc addMockAttestations*(
 
   # for-loop of distinct type is broken: https://github.com/nim-lang/Nim/issues/12074
   for slot in start_slot.uint64 ..< start_slot.uint64 + SLOTS_PER_EPOCH:
-    for shard in getShardsForSlot(state, slot.Slot):
+    for index in 0 ..< get_committee_count_at_slot(state, slot.Slot):
-
       # TODO: can we move cache out of the loops
       var cache = get_empty_per_epoch_cache()
 
-      let committee = get_crosslink_committee(
+      let committee = get_beacon_committee(
-                        state, slot.Slot.compute_epoch_at_slot(),
+                        state, slot.Slot, index, cache)
-                        shard, cache
-                      )
 
       # Create a bitfield filled with the given count per attestation,
       # exactly on the right-most part of the committee field.
@@ -80,12 +67,11 @@ proc addMockAttestations*(
         if idx != -1:
           aggregation_bits[idx] = false
 
-      let (ad_slot, ad_index) = get_slot_and_index(state, epoch, shard)
       attestations[].add PendingAttestation(
         aggregation_bits: aggregation_bits,
         data: AttestationData(
-          slot: ad_slot,
+          slot: slot.Slot,
-          index: ad_index,
+          index: index,
           beacon_block_root: [byte 0xFF] * 32, # Irrelevant for testing
           source: source,
           target: target,

tests/test_attestation_pool.nim

@@ -68,10 +68,10 @@ suite "Attestation pool processing" & preset():
       process_slots(state.data, state.data.data.slot + 1)
 
       let
-        cc1 = get_crosslink_committee(state.data.data,
+        bc1 = get_beacon_committee(state.data.data,
-          compute_epoch_at_slot(state.data.data.slot), 2, cache)
+          state.data.data.slot, 0, cache)
         attestation1 = makeAttestation(
-          state.data.data, state.blck.root, cc1[0])
+          state.data.data, state.blck.root, bc1[0])
 
       # test reverse order
       pool.add(state.data.data, state.blck, attestation1)