bugfix attestation creation so that it works on mainnet with multiple committees per slot, and improve bitfield index descriptions

test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py

@@ -36,6 +36,11 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support
     epoch_start_slot = spec.get_epoch_start_slot(epoch)
     for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH):
         for shard in get_shards_for_slot(spec, state, slot):
+            # Check if we already have had sufficient balance. (and undone if we don't want it).
+            # If so, do not create more attestations. (we do not have empty pending attestations normally anyway)
+            if remaining_balance < 0:
+                return
+
             committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard)
             # Create a bitfield filled with the given count per attestation,
             #  exactly on the right-most part of the committee field.
@@ -45,22 +50,23 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support
                 if remaining_balance > 0:
                     remaining_balance -= state.validators[v].effective_balance
                     aggregation_bits[v] = 1
-                elif not sufficient_support:
-                    aggregation_bits[v - 1] = 0
-                    break
                 else:
                     break
 
-        attestations.append(spec.PendingAttestation(
+            # remove just one attester to make the marginal support insufficient
-            aggregation_bits=aggregation_bits,
+            if not sufficient_support:
-            data=spec.AttestationData(
+                aggregation_bits[aggregation_bits.index(1)] = 0
-                beacon_block_root=b'\xaa' * 32,
+
-                source=source,
+            attestations.append(spec.PendingAttestation(
-                target=target,
+                aggregation_bits=aggregation_bits,
-                crosslink=spec.Crosslink(shard=shard)
+                data=spec.AttestationData(
-            ),
+                    beacon_block_root=b'\xaa' * 32,
-            inclusion_delay=1,
+                    source=source,
-        ))
+                    target=target,
+                    crosslink=spec.Crosslink(shard=shard)
+                ),
+                inclusion_delay=1,
+            ))
 
 
 def finalize_on_234(spec, state, epoch, sufficient_support):
@@ -82,7 +88,7 @@ def finalize_on_234(spec, state, epoch, sufficient_support):
     state.previous_justified_checkpoint = c4
     state.current_justified_checkpoint = c3
     state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]()
-    state.justification_bits[1:3] = [1, 1]  # mock 2nd and 3rd latest epochs as justified
+    state.justification_bits[1:3] = [1, 1]  # mock 3rd and 4th latest epochs as justified (indices are pre-shift)
     # mock the 2nd latest epoch as justifiable, with 4th as source
     add_mock_attestations(spec, state,
                           epoch=epoch - 2,
@@ -93,12 +99,11 @@ def finalize_on_234(spec, state, epoch, sufficient_support):
     # process!
     yield from run_process_just_and_fin(spec, state)
 
+    assert state.previous_justified_checkpoint == c3  # changed to old current
     if sufficient_support:
-        assert state.previous_justified_checkpoint == c3  # changed to old current
         assert state.current_justified_checkpoint == c2  # changed to 2nd latest
         assert state.finalized_checkpoint == c4  # finalized old previous justified epoch
     else:
-        assert state.previous_justified_checkpoint == c3  # changed to old current
         assert state.current_justified_checkpoint == c3  # still old current
         assert state.finalized_checkpoint == old_finalized  # no new finalized
 
@@ -108,7 +113,8 @@ def finalize_on_23(spec, state, epoch, sufficient_support):
     state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1  # skip ahead to just before epoch
 
     # 43210 -- epochs ago
-    # 3210x -- justification bitfield indices
+    # 210xx  -- justification bitfield indices (pre shift)
+    # 3210x -- justification bitfield indices (post shift)
     # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now
     # checkpoints for the epochs ago:
     c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32)
@@ -120,7 +126,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support):
     state.previous_justified_checkpoint = c3
     state.current_justified_checkpoint = c3
     state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]()
-    state.justification_bits[1] = 1  # mock 2nd latest epoch as justified
+    state.justification_bits[1] = 1  # mock 3rd latest epoch as justified (indices are pre-shift)
     # mock the 2nd latest epoch as justifiable, with 3rd as source
     add_mock_attestations(spec, state,
                           epoch=epoch - 2,
@@ -131,12 +137,11 @@ def finalize_on_23(spec, state, epoch, sufficient_support):
     # process!
     yield from run_process_just_and_fin(spec, state)
 
+    assert state.previous_justified_checkpoint == c3  # changed to old current
     if sufficient_support:
-        assert state.previous_justified_checkpoint == c3  # changed to old current
         assert state.current_justified_checkpoint == c2  # changed to 2nd latest
         assert state.finalized_checkpoint == c3  # finalized old previous justified epoch
     else:
-        assert state.previous_justified_checkpoint == c3  # changed to old current
         assert state.current_justified_checkpoint == c3  # still old current
         assert state.finalized_checkpoint == old_finalized  # no new finalized
 
@@ -146,7 +151,8 @@ def finalize_on_123(spec, state, epoch, sufficient_support):
     state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1  # skip ahead to just before epoch
 
     # 43210 -- epochs ago
-    # 3210x -- justification bitfield indices
+    # 210xx  -- justification bitfield indices (pre shift)
+    # 3210x -- justification bitfield indices (post shift)
     # 011*. -- justification bitfield contents, . = this epoch, * is being justified now
     # checkpoints for the epochs ago:
     c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32)
@@ -158,10 +164,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support):
     state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root
 
     old_finalized = state.finalized_checkpoint
-    state.previous_justified_checkpoint = c4
+    state.previous_justified_checkpoint = c4  # not c3, otherwise finalize 23 would trigger.
     state.current_justified_checkpoint = c2
     state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]()
-    state.justification_bits[0:2] = [1, 1]  # mock 1st and 2nd latest epochs as justified
+    state.justification_bits[0:2] = [1, 1]  # mock 2nd and 3rd latest epochs as justified (indices are pre-shift)
     # mock the 1st latest epoch as justifiable, with 3rd as source
     add_mock_attestations(spec, state,
                           epoch=epoch - 1,
@@ -172,12 +178,11 @@ def finalize_on_123(spec, state, epoch, sufficient_support):
     # process!
     yield from run_process_just_and_fin(spec, state)
 
+    assert state.previous_justified_checkpoint == c2  # changed to old current
     if sufficient_support:
-        assert state.previous_justified_checkpoint == c2  # changed to old current
         assert state.current_justified_checkpoint == c1  # changed to 1st latest
         assert state.finalized_checkpoint == c2  # finalized old current
     else:
-        assert state.previous_justified_checkpoint == c2  # changed to old current
         assert state.current_justified_checkpoint == c2  # still old current
         assert state.finalized_checkpoint == old_finalized  # no new finalized
 
@@ -187,7 +192,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support):
     state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1  # skip ahead to just before epoch
 
     # 43210 -- epochs ago
-    # 3210  -- justification bitfield indices
+    # 210xx  -- justification bitfield indices (pre shift)
+    # 3210x -- justification bitfield indices (post shift)
     # 001*. -- justification bitfield contents, . = this epoch, * is being justified now
     # checkpoints for the epochs ago:
     c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32)
@@ -199,7 +205,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support):
     state.previous_justified_checkpoint = c2
     state.current_justified_checkpoint = c2
     state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]()
-    state.justification_bits[0] = 1  # mock latest epoch as justified
+    state.justification_bits[0] = 1  # mock 2nd latest epoch as justified (this is pre-shift)
     # mock the 1st latest epoch as justifiable, with 2nd as source
     add_mock_attestations(spec, state,
                           epoch=epoch - 1,
@@ -210,12 +216,11 @@ def finalize_on_12(spec, state, epoch, sufficient_support):
     # process!
     yield from run_process_just_and_fin(spec, state)
 
+    assert state.previous_justified_checkpoint == c2  # changed to old current
     if sufficient_support:
-        assert state.previous_justified_checkpoint == c2  # changed to old current
         assert state.current_justified_checkpoint == c1  # changed to 1st latest
         assert state.finalized_checkpoint == c2  # finalized previous justified epoch
     else:
-        assert state.previous_justified_checkpoint == c2  # changed to old current
         assert state.current_justified_checkpoint == c2  # still old current
         assert state.finalized_checkpoint == old_finalized  # no new finalized