state_sim optimizations (#597)

* switch out quadratically scaling and wasteful attestation in state_sim to attest only to exactly the correct slots; avoid pointless committee index interconversion for 9-10x increase in state_sim speed at d:release, 60k validators, and validate=off

* remove debugechos

research/state_sim.nim

@@ -57,7 +57,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
        validators = SLOTS_PER_EPOCH * 11, # One per shard is minimum
        json_interval = SLOTS_PER_EPOCH,
        prefix = 0,
-       attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.75,
+       attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73,
        validate = true):
   let
     flags = if validate: {} else: {skipValidation}
@@ -83,6 +83,10 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
     else:
       write(stdout, ".")
 
+  # TODO doAssert against this up-front
+  # indexed attestation: validator index beyond max validators per committee
+  # len(indices) <= MAX_VALIDATORS_PER_COMMITTEE
+
   for i in 0..<slots:
     maybeWrite()
     verifyConsensus(state, attesterRatio)
@@ -111,15 +115,13 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
       # work for every slot - we'll randomize it deterministically to give
       # some variation
       let
-        epoch = compute_epoch_at_slot(state.slot)
+        target_slot = state.slot + MIN_ATTESTATION_INCLUSION_DELAY - 1
         scass = withTimerRet(timers[tShuffle]):
           mapIt(
-            0'u64 .. (get_committee_count_at_slot(state, state.slot) *
-              SLOTS_PER_EPOCH - 1),
-            get_beacon_committee(state, epoch.compute_start_slot_at_epoch + (it mod SLOTS_PER_EPOCH),
-              it div SLOTS_PER_EPOCH, cache))
+            0'u64 ..< get_committee_count_at_slot(state, target_slot),
+            get_beacon_committee(state, target_slot, it, cache))
 
-      for scas in scass:
+      for i, scas in scass:
         var
           attestation: Attestation
           first = true
@@ -131,11 +133,13 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
             if (rand(r, high(int)).float * attesterRatio).int <= high(int):
               if first:
                 attestation =
-                  makeAttestation(state, latest_block_root, v, cache, flags)
+                  makeAttestation(state, latest_block_root, scas, target_slot,
+                    i.uint64, v, cache, flags)
                 first = false
               else:
                 attestation.combine(
-                  makeAttestation(state, latest_block_root, v, cache, flags),
+                  makeAttestation(state, latest_block_root, scas, target_slot,
+                    i.uint64, v, cache, flags),
                   flags)
 
         if not first:
@@ -145,15 +149,11 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
           let target_slot =
             attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY - 1
 
-          ## In principle, should enumerate possible shard/slot combinations by
-          ## inverting get_attestation_data_slot(...), but this works. Could be
-          ## filtering earlier if we know that this attestation's being created
-          ## too late to be useful, as well.
-          if target_slot > attestations_idx:
-            var target_slot_attestations =
-              getOrDefault(attestations, target_slot)
-            target_slot_attestations.add attestation
-            attestations[target_slot] = target_slot_attestations
+          doAssert target_slot > attestations_idx
+          var target_slot_attestations =
+            getOrDefault(attestations, target_slot)
+          target_slot_attestations.add attestation
+          attestations[target_slot] = target_slot_attestations
 
     flushFile(stdout)
 

tests/test_peer_pool.nim

@@ -1,8 +1,8 @@
 # beacon_chain
 # Copyright (c) 2019 Status Research & Development GmbH
 # Licensed and distributed under either of
-#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
-#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import

tests/testutil.nim

@@ -164,11 +164,14 @@ proc find_beacon_committee(
 
 proc makeAttestation*(
     state: BeaconState, beacon_block_root: Eth2Digest,
-    validator_index: ValidatorIndex, cache: var StateCache,
+    committee: seq[ValidatorIndex], slot: Slot, index: uint64,
+    validator_index: auto, cache: var StateCache,
     flags: UpdateFlags = {}): Attestation =
+  # Avoids state_sim silliness; as it's responsible for all validators,
+  # transforming, from monotonic enumerable index -> committee index ->
+  # montonoic enumerable index, is wasteful and slow. Most test callers
+  # want ValidatorIndex, so that's supported too.
   let
-    (committee, slot, index) =
-      find_beacon_committee(state, validator_index, cache)
     validator = state.validators[validator_index]
     sac_index = committee.find(validator_index)
     data = makeAttestationData(state, slot, index, beacon_block_root)
@@ -197,6 +200,15 @@ proc makeAttestation*(
     signature: sig
   )
 
+proc makeAttestation*(
+    state: BeaconState, beacon_block_root: Eth2Digest,
+    validator_index: ValidatorIndex, cache: var StateCache,
+    flags: UpdateFlags = {}): Attestation =
+  let (committee, slot, index) =
+    find_beacon_committee(state, validator_index, cache)
+  makeAttestation(state, beacon_block_root, committee, slot, index,
+    validator_index, cache, flags)
+
 proc makeTestDB*(tailState: BeaconState, tailBlock: BeaconBlock): BeaconChainDB =
   result = init(BeaconChainDB, newMemoryDB())
   BlockPool.preInit(result, tailState, tailBlock)