state_sim: clean up attestation production (#3274)

* use same naming as everywhere
* avoid iterator bug that leads to state copy

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -459,7 +459,9 @@ func init(
 
   template update_attestation_pool_cache(
       epoch: Epoch, participation_bitmap: untyped) =
-    for committee_index in get_committee_indices(state.data, epoch, cache):
+    let committees_per_slot = get_committee_count_per_slot(
+      state.data, epoch, cache)
+    for committee_index in get_committee_indices(committees_per_slot):
       for slot in epoch.slots():
         let committee = get_beacon_committee(
             state.data, slot, committee_index, cache)

beacon_chain/rpc/rest_beacon_api.nim

@@ -572,17 +572,15 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
 
       proc forSlot(slot: Slot, cindex: Option[CommitteeIndex],
                    res: var seq[RestBeaconStatesCommittees]) =
+        let committees_per_slot = get_committee_count_per_slot(
+          stateData.data, slot.epoch, cache)
 
         if cindex.isNone:
-          for committee_index in
+          for committee_index in get_committee_indices(committees_per_slot):
-              get_committee_indices(stateData.data, slot.epoch, cache):
             res.add(getCommittee(slot, committee_index))
         else:
           let
             idx = cindex.get()
-            committees_per_slot = get_committee_count_per_slot(
-              stateData.data, slot.epoch, cache)
-
           if idx < committees_per_slot:
             res.add(getCommittee(slot, idx))
 

beacon_chain/spec/validator.nim

@@ -184,15 +184,6 @@ iterator get_committee_indices*(committee_count_per_slot: uint64): CommitteeInde
     let committee_index = CommitteeIndex.init(idx).expect("value clamped")
     yield committee_index
 
-iterator get_committee_indices*(state: ForkyBeaconState | ForkedHashedBeaconState,
-                                epoch: Epoch,
-                                cache: var StateCache): CommitteeIndex =
-  let committee_count_per_slot =
-    get_committee_count_per_slot(state, epoch, cache)
-
-  for committee_index in get_committee_indices(committee_count_per_slot):
-    yield committee_index
-
 func get_previous_epoch*(state: ForkyBeaconState): Epoch =
   ## Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``).
   # Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``).

research/state_sim.nim

@@ -9,20 +9,18 @@
 # and attesting to them as if the network was running as a whole.
 
 import
-  confutils, stats, times,
+  std/[stats, times, strformat, random, tables],
-  strformat,
+  confutils,
-  options, sequtils, random, tables,
   ../tests/testblockutil,
   ../beacon_chain/spec/datatypes/phase0,
   ../beacon_chain/spec/eth2_apis/eth2_rest_serialization,
-  ../beacon_chain/spec/[beaconstate, forks, helpers],
+  ../beacon_chain/spec/[beaconstate, forks, helpers, signatures],
   ./simutils
 
 type Timers = enum
   tBlock = "Process non-epoch slot with block"
   tEpoch = "Process epoch slot with block"
   tHashBlock = "Tree-hash block"
-  tShuffle = "Retrieve committee once using get_beacon_committee"
   tAttest = "Combine committee attestations"
 
 func jsonName(prefix, slot: auto): string =
@@ -44,6 +42,7 @@ cli do(slots = SLOTS_PER_EPOCH * 5,
     flags = if validate: {} else: {skipBlsValidation}
     (state, _) = loadGenesis(validators, validate)
     genesisBlock = get_initial_beacon_block(state[])
+    genesis_validators_root = getStateField(state[], genesis_validators_root)
 
   echo "Starting simulation..."
 
@@ -105,56 +104,44 @@ cli do(slots = SLOTS_PER_EPOCH * 5,
       # attesterRatio is the fraction of attesters that actually do their
       # work for every slot - we'll randomize it deterministically to give
       # some variation
-      let
-        target_slot = getStateField(state[], slot) + MIN_ATTESTATION_INCLUSION_DELAY - 1
-        committees_per_slot =
-          get_committee_count_per_slot(state[], target_slot.epoch, cache)
 
-      let
+      withState(state[]):
-        scass = withTimerRet(timers[tShuffle]):
+        let
-          mapIt(
+          slot = state.data.slot
-            0 ..< committees_per_slot.int,
+          epoch = slot.epoch
-            get_beacon_committee(state[], target_slot, it.CommitteeIndex, cache))
+          committees_per_slot =
+            get_committee_count_per_slot(state.data, epoch, cache)
+        for committee_index in get_committee_indices(committees_per_slot):
+          let committee = get_beacon_committee(
+            state.data, slot, committee_index, cache)
+          var
+            attestation = Attestation(
+              aggregation_bits: CommitteeValidatorsBits.init(committee.len),
+              data: makeAttestationData(
+                state.data, slot, committee_index, latest_block_root),
+            )
+            first = true
 
-      for i, scas in scass:
+          attesters.push committee.len()
-        var
-          attestation: Attestation
-          first = true
 
-        attesters.push scas.len()
+          withTimer(timers[tAttest]):
+            for index_in_committee, validator_index in committee:
+              if (rand(r, high(int)).float * attesterRatio).int <= high(int):
+                attestation.aggregation_bits.setBit index_in_committee
 
-        withTimer(timers[tAttest]):
+            if attestation.aggregation_bits.countOnes() > 0:
-          var agg {.noInit.}: AggregateSignature
+              if validate:
-          for v in scas:
+                attestation.signature = makeAttestationSig(
-            if (rand(r, high(int)).float * attesterRatio).int <= high(int):
+                  state.data.fork, genesis_validators_root, attestation.data,
-              if first:
+                  committee, attestation.aggregation_bits)
-                attestation =
-                  makeAttestation(state[], latest_block_root, scas, target_slot,
-                    i.CommitteeIndex, v, cache, flags)
-                agg.init(attestation.signature.load.get())
-                first = false
-              else:
-                let att2 =
-                  makeAttestation(state[], latest_block_root, scas, target_slot,
-                    i.CommitteeIndex, v, cache, flags)
-                if not att2.aggregation_bits.overlaps(attestation.aggregation_bits):
-                  attestation.aggregation_bits.incl(att2.aggregation_bits)
-                  if skipBlsValidation notin flags:
-                    agg.aggregate(att2.signature.load.get())
-          attestation.signature = agg.finish().toValidatorSig()
 
-        if not first:
+              # add the attestation if any of the validators attested, as given
-          # add the attestation if any of the validators attested, as given
+              # by the randomness. We have to delay when the attestation is
-          # by the randomness. We have to delay when the attestation is
+              # actually added to the block per the attestation delay rule!
-          # actually added to the block per the attestation delay rule!
+              let
-          let target_slot =
+                target_slot = slot + MIN_ATTESTATION_INCLUSION_DELAY - 1
-            attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY - 1
+              attestations.mGetOrPut(target_slot, default(seq[Attestation])).add(
-
+                attestation)
-          doAssert target_slot > attestations_idx
-          var target_slot_attestations =
-            getOrDefault(attestations, target_slot)
-          target_slot_attestations.add attestation
-          attestations[target_slot] = target_slot_attestations
 
     flushFile(stdout)
 
@@ -167,4 +154,4 @@ cli do(slots = SLOTS_PER_EPOCH * 5,
 
   echo "Done!"
 
-  printTimers(state[], attesters, validate, timers)
+  printTimers(state[], attesters, true, timers)

tests/testblockutil.nim

@@ -178,6 +178,32 @@ func makeAttestationData*(
     )
   )
 
+func makeAttestationSig*(
+    fork: Fork, genesis_validators_root: Eth2Digest, data: AttestationData,
+    committee: openArray[ValidatorIndex],
+    bits: CommitteeValidatorsBits): ValidatorSig =
+  let signing_root = compute_attestation_signing_root(
+    fork, genesis_validators_root, data)
+
+  var
+    agg {.noInit.}: AggregateSignature
+    first = true
+
+  for i in 0..<bits.len():
+    if not bits[i]: continue
+    let sig = blsSign(MockPrivKeys[committee[i]], signing_root.data)
+
+    if first:
+      agg.init(sig)
+      first = false
+    else:
+      agg.aggregate(sig)
+
+  if first:
+    ValidatorSig.infinity()
+  else:
+    agg.finish().toValidatorSig()
+
 func makeAttestationData*(
     state: ForkedHashedBeaconState, slot: Slot, committee_index: CommitteeIndex,
     beacon_block_root: Eth2Digest): AttestationData =
@@ -206,15 +232,13 @@ func makeAttestation*(
   var aggregation_bits = CommitteeValidatorsBits.init(committee.len)
   aggregation_bits.setBit sac_index
 
-  let
+  let sig = if skipBLSValidation in flags:
-    sig =
+    ValidatorSig()
-      if skipBLSValidation notin flags:
+  else:
-        get_attestation_signature(
+    makeAttestationSig(
-          getStateField(state, fork),
+      getStateField(state, fork),
-          getStateField(state, genesis_validators_root),
+      getStateField(state, genesis_validators_root),
-          data, MockPrivKeys[validator_index]).toValidatorSig()
+      data, committee, aggregation_bits)
-      else:
-        ValidatorSig()
 
   Attestation(
     data: data,
@@ -251,35 +275,25 @@ func makeFullAttestations*(
     flags: UpdateFlags = {}): seq[Attestation] =
   # Create attestations in which the full committee participates for each shard
   # that should be attested to during a particular slot
-  for committee_index in get_committee_indices(state, slot.epoch, cache):
+  let committees_per_slot = get_committee_count_per_slot(
+    state, slot.epoch, cache)
+  for committee_index in get_committee_indices(committees_per_slot):
     let
       committee = get_beacon_committee(state, slot, committee_index, cache)
       data = makeAttestationData(state, slot, committee_index, beacon_block_root)
 
     doAssert committee.len() >= 1
-    # Initial attestation
     var attestation = Attestation(
       aggregation_bits: CommitteeValidatorsBits.init(committee.len),
       data: data)
+    for i in 0..<committee.len:
+      attestation.aggregation_bits.setBit(i)
 
-    var agg {.noInit.}: AggregateSignature
+    attestation.signature = makeAttestationSig(
-    agg.init(get_attestation_signature(
         getStateField(state, fork),
-        getStateField(state, genesis_validators_root), data,
+        getStateField(state, genesis_validators_root), data, committee,
-        MockPrivKeys[committee[0]]))
+        attestation.aggregation_bits)
 
-    # Aggregate the remainder
-    attestation.aggregation_bits.setBit 0
-    for j in 1 ..< committee.len():
-      attestation.aggregation_bits.setBit j
-      if skipBLSValidation notin flags:
-        agg.aggregate(get_attestation_signature(
-          getStateField(state, fork),
-          getStateField(state, genesis_validators_root), data,
-          MockPrivKeys[committee[j]]
-        ))
-
-    attestation.signature = agg.finish().toValidatorSig()
     result.add attestation
 
 proc makeSyncAggregate(