minor cleanups/fixes to shard data avail PR

specs/phase1/beacon-chain.md

@@ -23,20 +23,24 @@
 - [New containers](#new-containers)
   - [`DataCommitment`](#datacommitment)
   - [`ShardHeader`](#shardheader)
+  - [`SignedShardHeader`](#signedshardheader)
   - [`PendingShardHeader`](#pendingshardheader)
 - [Helper functions](#helper-functions)
   - [Misc](#misc-1)
+    - [`next_power_of_two`](#next_power_of_two)
+    - [`reverse_bit_order`](#reverse_bit_order)
     - [`compute_previous_slot`](#compute_previous_slot)
-    - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index)
     - [`compute_updated_gasprice`](#compute_updated_gasprice)
     - [`compute_committee_source_epoch`](#compute_committee_source_epoch)
   - [Beacon state accessors](#beacon-state-accessors)
     - [Updated `get_committee_count_per_slot`](#updated-get_committee_count_per_slot)
     - [`get_active_shard_count`](#get_active_shard_count)
     - [`get_shard_committee`](#get_shard_committee)
+    - [`compute_proposer_index`](#compute_proposer_index)
     - [`get_shard_proposer_index`](#get_shard_proposer_index)
     - [`get_start_shard`](#get_start_shard)
-  - [Predicates](#predicates)
+    - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index)
+    - [`compute_committee_index_from_shard`](#compute_committee_index_from_shard)
   - [Block processing](#block-processing)
     - [Operations](#operations)
   - [New Attestation processing](#new-attestation-processing)
@@ -139,7 +143,7 @@ class AttestationData(Container):
 
 ```python
 class BeaconBlock(phase0.BeaconBlock):
-    shard_headers: List[Signed[ShardHeader], MAX_SHARD_HEADERS]
+    shard_headers: List[SignedShardHeader, MAX_SHARD_HEADERS]
 ```
 
 ### `BeaconState`
@@ -184,6 +188,14 @@ class ShardHeader(Container):
     degree_proof: BLSCommitment
 ```
 
+### `SignedShardHeader`
+
+```python
+class SignedShardHeader(Container):
+    message: ShardHeader
+    signature: BLSSignature
+```
+
 ### `PendingShardHeader`
 
 ```python
@@ -193,8 +205,7 @@ class PendingShardHeader(Container):
     shard: Shard
     # KZG10 commitment to the data
     commitment: DataCommitment
-    # hash_tree_root of the ShardHeader (stored so that attestations
+    # hash_tree_root of the ShardHeader (stored so that attestations can be checked against it)
-    # can be checked against it)
     root: Root
     # Who voted for the header
     votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]
@@ -307,10 +318,14 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -
 
 #### `compute_proposer_index`
 
-Updated version to get a proposer index that will only allow proposers with a certain minimum balance, ensuring that the balance is always sufficient to cover gas costs.
+Updated version to get a proposer index that will only allow proposers with a certain minimum balance,
+ensuring that the balance is always sufficient to cover gas costs.
 
 ```python
-def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32, min_effective_balance: GWei = GWei(0)) -> ValidatorIndex:
+def compute_proposer_index(beacon_state: BeaconState,
+                           indices: Sequence[ValidatorIndex],
+                           seed: Bytes32,
+                           min_effective_balance: GWei = GWei(0)) -> ValidatorIndex:
     """
     Return from ``indices`` a random index sampled by effective balance.
     """
@@ -321,8 +336,10 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex]
     while True:
         candidate_index = indices[compute_shuffled_index(i % total, total, seed)]
         random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32]
-        effective_balance = state.validators[candidate_index].effective_balance
+        effective_balance = beacon_state.validators[candidate_index].effective_balance
-        if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte and effective_balance > min_effective_balance:
+        if effective_balance <= min_effective_balance:
+            continue
+        if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte:
             return candidate_index
         i += 1
 ```
@@ -336,11 +353,18 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard
     """
     epoch = compute_epoch_at_slot(slot)
     committee = get_shard_committee(beacon_state, epoch, shard)
-    seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot))
+    seed = hash(get_seed(beacon_state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(beacon_state.slot))
-    EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = EFFECTIVE_BALANCE_INCREMENT - EFFECTIVE_BALANCE_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT
 
-    return compute_proposer_index(state, committee, seed,
+    # Proposer must have sufficient balance to pay for worst case fee burn
-        state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION)
+    EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = (
+        (EFFECTIVE_BALANCE_INCREMENT - EFFECTIVE_BALANCE_INCREMENT)
+        * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT
+    )
+    min_effective_balance = (
+        beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK
+        + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION
+    )
+    return compute_proposer_index(beacon_state, committee, seed, min_effective_balance)
 ```
 
 #### `get_start_shard`
@@ -358,7 +382,6 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard:
             committee_count = get_committee_count_per_slot(state, compute_epoch_at_slot(Slot(_slot)))
             active_shard_count = get_active_shard_count(state, compute_epoch_at_slot(Slot(_slot)))
             shard = (shard + committee_count) % active_shard_count
-        return Shard(shard)
     elif slot < current_epoch_start_slot:
         # Previous epoch
         for _slot in list(range(slot, current_epoch_start_slot))[::-1]:
@@ -410,20 +433,16 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None:
 
     for_ops(body.proposer_slashings, process_proposer_slashing)
     for_ops(body.attester_slashings, process_attester_slashing)
+    # Limit is dynamic based on active shard count
+    assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state))
+    for_ops(body.shard_headers, process_shard_header)
     # New attestation processing
     for_ops(body.attestations, process_attestation)
     for_ops(body.deposits, process_deposit)
     for_ops(body.voluntary_exits, process_voluntary_exit)
-    # Limit is dynamic based on active shard count
-    assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state))
-    for_ops(body.shard_headers, process_shard_header)
 
     # See custody game spec.
     process_custody_game_operations(state, body)
-
-    process_shard_transitions(state, body.shard_transitions, body.attestations)
-
-    # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs)
 ```
 
 ### New Attestation processing
@@ -441,11 +460,11 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
 ```python
 def update_pending_votes(state: BeaconState,
                          attestation: Attestation) -> None:
-    if compute_epoch_at_slot(slot) == get_current_epoch(state):
+    # Find and update the PendingShardHeader object, invalid block if pending header not in state
+    if compute_epoch_at_slot(attestation.data.slot) == get_current_epoch(state):
         pending_headers = state.current_epoch_pending_shard_headers
     else:
         pending_headers = state.previous_epoch_pending_shard_headers
-    # Create or update the PendingShardHeader object
     pending_header = None
     for header in pending_headers:
         if header.root == attestation.data.shard_header_root:
@@ -457,10 +476,8 @@ def update_pending_votes(state: BeaconState,
         attestation.data.slot,
         attestation.data.index,
     )
-    pending_header.votes = bitwise_or(
+    for i in range(len(pending_header.votes)):
-        pending_header.votes,
+        pending_header.votes[i] = pending_header.votes[i] or attestation.aggregation_bits[i]
-        attestation.aggregation_bits
-    )
 
     # Check if the PendingShardHeader is eligible for expedited confirmation
     # Requirement 1: nothing else confirmed
@@ -468,47 +485,49 @@ def update_pending_votes(state: BeaconState,
         c for c in pending_headers if
         (c.slot, c.shard) == (pending_header.slot, pending_header.shard)
     ]
-    if True not in [c.confirmed for c in all_candidates]:
+    if True in [c.confirmed for c in all_candidates]:
-        # Requirement 2: >= 2/3 of balance attesting
+        return
-        participants = get_attesting_indices(state, attestation.data, pending_commitment.votes)
+
-        participants_balance = get_total_balance(state, participants)
+    # Requirement 2: >= 2/3 of balance attesting
-        full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index)
+    participants = get_attesting_indices(state, attestation.data, pending_commitment.votes)
-        full_committee_balance = get_total_balance(state, full_committee)
+    participants_balance = get_total_balance(state, participants)
-        if participants_balance * 3 > full_committee_balance * 2:
+    full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index)
-            pending_header.confirmed = True
+    full_committee_balance = get_total_balance(state, full_committee)
+    if participants_balance * 3 >= full_committee_balance * 2:
+        pending_header.confirmed = True
 ```
 
 #### `process_shard_header`
 
 ```python
 def process_shard_header(state: BeaconState,
-                         signed_header: Signed[ShardHeader]) -> None:
+                         signed_header: SignedShardHeader) -> None:
     header = signed_header.message
     header_root = hash_tree_root(header)
+    assert compute_epoch_at_slot(header.slot) in [get_previous_epoch(state), get_current_epoch(state)]
+
     # Verify signature
     signer_index = get_shard_proposer_index(state, header.slot, header.shard)
-    assert bls.Verify(
+    signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_HEADER))
-        state.validators[signer_index].pubkey,
+    assert bls.Verify(state.validators[signer_index].pubkey, signing_root, signed_header.signature)
-        compute_signing_root(header, get_domain(state, DOMAIN_SHARD_HEADER)),
+
-        signed_header.signature
-    )
     # Verify the length by verifying the degree.
     if header.commitment.length == 0:
         assert header.degree_proof == G1_SETUP[0]
     assert (
-        bls.Pairing(header.degree_proof, G2_SETUP[0]) ==
+        bls.Pairing(header.degree_proof, G2_SETUP[0])
-        bls.Pairing(header.commitment.point, G2_SETUP[-header.commitment.length]))
+        == bls.Pairing(header.commitment.point, G2_SETUP[-header.commitment.length]))
     )
+
     # Get the correct pending header list
     if compute_epoch_at_slot(header.slot) == get_current_epoch(state):
         pending_headers = state.current_epoch_pending_shard_headers
     else:
-        assert compute_epoch_at_slot(header.slot) == get_previous_epoch(state):
         pending_headers = state.previous_epoch_pending_shard_headers
 
     # Check that this header is not yet in the pending list
-    for pending_header in pending_headers:
+    assert header_root not in [pending_header.root for pending_header in pending_headers]
-        assert header_root != pending_header.root
+
     # Include it in the pending list
     index = compute_committee_index_from_shard(state, header.slot, header.shard)
     committee_length = len(get_beacon_committee(state, header.slot, index))
@@ -518,7 +537,7 @@ def process_shard_header(state: BeaconState,
         commitment=header.commitment,
         root=header_root,
         votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length),
-        confirmed=False
+        confirmed=False,
     ))
 ```
 
@@ -565,102 +584,107 @@ def process_epoch(state: BeaconState) -> None:
 
 ```python
 
-def process_pending_headers(state: BeaconState):
+def process_pending_headers(state: BeaconState) -> None:
-    for slot in range(SLOTS_PER_EPOCH):
+    # Pending header processing applies to the previous epoch.
+    # Skip if `GENESIS_EPOCH` because no prior epoch to process.
+    if get_current_epoch(state) == GENESIS_EPOCH:
+        return
+
+    previous_epoch_start_slot = compute_start_slot_at_epoch(get_previous_epoch(state))
+    for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH):
         for shard in range(get_active_shard_count(state)):
             # Pending headers for this (slot, shard) combo
             candidates = [
-                c for c in state.previous_epoch_pending_shard_headers if
+                c for c in state.previous_epoch_pending_shard_headers
-                (c.slot, c.shard) == (slot, shard)
+                if (c.slot, c.shard) == (slot, shard)
             ]
             # The entire committee (and its balance)
             full_committee = get_beacon_committee(state, slot, shard)
             full_committee_balance = get_total_balance(state, full_committee)
-            if True not in [c.confirmed for c in candidates]:
+            # If any candidates already confirmed, skip
-                # The set of voters who voted for each header
+            if True in [c.confirmed for c in candidates]:
-                # (and their total balances)
+                continue
-                voting_sets = [
+
-                    [v for i, v in enumerate(full_committee) if c.votes[i]]
+            # The set of voters who voted for each header (and their total balances)
-                    for c in candidates
+            voting_sets = [
-                ]
+                [v for i, v in enumerate(full_committee) if c.votes[i]]
-                voting_balances = [
+                for c in candidates
-                    get_total_balance(state, voters)
+            ]
-                    for voters in voting_sets
+            voting_balances = [
-                ]
+                get_total_balance(state, voters)
-                # Get the index with the most total balance voting for them.
+                for voters in voting_sets
-                # NOTE: if two choices get exactly the same voting balance,
+            ]
-                # the candidate earlier in the list wins
+            # Get the index with the most total balance voting for them.
-                if max(voting_balances) > 0:
+            # NOTE: if two choices get exactly the same voting balance,
-                    winning_index = voting_balances.index(max(voting_balances))
+            # the candidate earlier in the list wins
-                else:
+            if max(voting_balances) > 0:
-                    # If no votes, zero wins
+                winning_index = voting_balances.index(max(voting_balances))
-                    winning_index = [c.root for c in candidates].index(Root())
+            else:
-                candidates[winning_index].confirmed = True
+                # If no votes, zero wins
-    for slot in range(SLOTS_PER_EPOCH):
+                winning_index = [c.root for c in candidates].index(Root())
+            candidates[winning_index].confirmed = True
+    for slot_index in range(SLOTS_PER_EPOCH):
         for shard in range(SHARD_COUNT):
-            state.grandparent_epoch_confirmed_commitments[shard][slot] = DataCommitment()
+            state.grandparent_epoch_confirmed_commitments[shard][slot_index] = DataCommitment()
-    for c in state.previous_epoch_pending_shard_headers:
+    confirmed_headers = [candidate in state.previous_epoch_pending_shard_headers if candidate.confirmed]
-        if c.confirmed:
+    for header in confirmed_headers:
-            state.grandparent_epoch_confirmed_commitments[c.shard][c.slot % SLOTS_PER_EPOCH] = c.commitment
+        state.grandparent_epoch_confirmed_commitments[c.shard][c.slot % SLOTS_PER_EPOCH] = c.commitment
 ```
 
 ```python
 def charge_confirmed_header_fees(state: BeaconState) -> None:
     new_gasprice = state.shard_gasprice
-    adjustment_quotient = get_active_shard_count(state, get_current_epoch(state)) * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT
+    adjustment_quotient = (
-    for slot in range(SLOTS_PER_EPOCH):
+        get_active_shard_count(state, get_current_epoch(state))
+        * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT
+    )
+    previous_epoch_start_slot = compute_start_slot_at_epoch(get_previous_epoch(state))
+    for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH):
         for shard in range(SHARD_COUNT):
             confirmed_candidates = [
-                c for c in state.previous_epoch_pending_shard_headers if
+                c for c in state.previous_epoch_pending_shard_headers
-                (c.slot, c.shard, c.confirmed) == (slot, shard, True)
+                if (c.slot, c.shard, c.confirmed) == (slot, shard, True)
             ]
-            if confirmed_candidates:
+            if not any(confirmed_candidates):
-                candidate = confirmed_candidates[0]
+                continue
-                # Charge EIP 1559 fee
+            candidate = confirmed_candidates[0]
-                proposer = get_shard_proposer(state, slot, shard)
+
-                fee = (
+            # Charge EIP 1559 fee
-                    (state.shard_gasprice * candidates[i].commitment.length) //
+            proposer = get_shard_proposer(state, slot, shard)
-                    TARGET_SAMPLES_PER_BLOCK
+            fee = (
-                )
+                (state.shard_gasprice * candidate.commitment.length)
-                decrease_balance(state, proposer, fee)
+                // TARGET_SAMPLES_PER_BLOCK
-                new_gasprice = compute_updated_gasprice(
+            )
-                    new_gasprice,
+            decrease_balance(state, proposer, fee)
-                    candidates[i].commitment.length,
+
-                    adjustment_quotient
+            # Track updated gas price
-                )
+            new_gasprice = compute_updated_gasprice(
+                new_gasprice,
+                candidate.commitment.length,
+                adjustment_quotient,
+            )
     state.shard_gasprice = new_gasprice
 ```
 
 ```python
 def reset_pending_headers(state: BeaconState):
     state.previous_epoch_pending_shard_headers = state.current_epoch_pending_shard_headers
-    shards = [
-        compute_shard_from_committee_index(state, slot, index)
-        for i in range()
-        state,
-        attestation.data.index,
-        attestation.data.slot
-    )
     state.current_epoch_pending_shard_headers = []
-    # Add dummy "empty" PendingAttestations
+    # Add dummy "empty" PendingShardHeader (default vote for if no shard header available)
-    # (default to vote for if no shard header available)
+    next_epoch = get_current_epoch(state) + 1
-    for slot in range(SLOTS_IN_EPOCH):
+    next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch)
-        for index in range(get_committee_count_per_slot(get_current_epoch(state))):
+    for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_IN_EPOCH):
+        for index in range(get_committee_count_per_slot(next_epoch)
             shard = compute_shard_from_committee_index(state, slot, index)
-            committee_length = len(get_beacon_committee(
+            committee_length = len(get_beacon_committee(state, slot, shard))
-                state,
-                header.slot,
-                header.shard
-            ))
             state.current_epoch_pending_shard_headers.append(PendingShardHeader(
                 slot=slot,
                 shard=shard,
                 commitment=DataCommitment(),
                 root=Root(),
                 votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length),
-                confirmed=False
+                confirmed=False,
             ))
-
 ```
 
 #### Custody game updates