Merge pull request #2187 from ethereum/re_process_final_updates

Breaking down `process_final_updates`, take 2

specs/lightclient/beacon-chain.md

@@ -29,7 +29,7 @@
     - [Sync committee processing](#sync-committee-processing)
   - [Epoch processing](#epoch-processing)
     - [Components of attestation deltas](#components-of-attestation-deltas)
-    - [Final updates](#final-updates)
+    - [Sync committee updates](#sync-committee-updates)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
@@ -209,6 +209,22 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None:
 
 ### Epoch processing
 
+```python
+def process_epoch(state: BeaconState) -> None:
+    process_justification_and_finalization(state)
+    process_rewards_and_penalties(state)
+    process_registry_updates(state)
+    process_slashings(state)
+    process_eth1_data_reset(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_roots_update(state)
+    process_participation_record_updates(state)
+    # Light client patch
+    process_sync_committee_updates(state)
+```
+
 #### Components of attestation deltas
 
 *Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`.
@@ -235,14 +251,10 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S
     return rewards, penalties
 ```
 
-#### Final updates
+#### Sync committee updates
-
-*Note*: The function `process_final_updates` is modified to handle sync committee updates.
 
 ```python
-def process_final_updates(state: BeaconState) -> None:
+def process_sync_committee_updates(state: BeaconState) -> None:
-    # FIXME: unfold the full `process_final_updates` to avoid side effects.
-    phase0.process_final_updates(state)
     next_epoch = get_current_epoch(state) + Epoch(1)
     if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0:
         state.current_sync_committee = state.next_sync_committee

specs/phase0/beacon-chain.md

@@ -113,7 +113,12 @@
       - [`process_rewards_and_penalties`](#process_rewards_and_penalties)
     - [Registry updates](#registry-updates)
     - [Slashings](#slashings)
-    - [Final updates](#final-updates)
+    - [Eth1 data votes updates](#eth1-data-votes-updates)
+    - [Effective balances updates](#effective-balances-updates)
+    - [Slashings balances updates](#slashings-balances-updates)
+    - [Randao mixes updates](#randao-mixes-updates)
+    - [Historical roots updates](#historical-roots-updates)
+    - [Participation records rotation](#participation-records-rotation)
   - [Block processing](#block-processing)
     - [Block header](#block-header)
     - [RANDAO](#randao)
@@ -1250,7 +1255,12 @@ def process_epoch(state: BeaconState) -> None:
     process_rewards_and_penalties(state)
     process_registry_updates(state)
     process_slashings(state)
-    process_final_updates(state)
+    process_eth1_data_reset(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_roots_update(state)
+    process_participation_record_updates(state)
 ```
 
 #### Helper functions
@@ -1557,15 +1567,19 @@ def process_slashings(state: BeaconState) -> None:
             decrease_balance(state, ValidatorIndex(index), penalty)
 ```
 
-#### Final updates
+#### Eth1 data votes updates
-
 ```python
-def process_final_updates(state: BeaconState) -> None:
+def process_eth1_data_reset(state: BeaconState) -> None:
-    current_epoch = get_current_epoch(state)
+    next_epoch = Epoch(get_current_epoch(state) + 1)
-    next_epoch = Epoch(current_epoch + 1)
     # Reset eth1 data votes
     if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0:
         state.eth1_data_votes = []
+```
+
+#### Effective balances updates
+
+```python
+def process_effective_balance_updates(state: BeaconState) -> None:
     # Update effective balances with hysteresis
     for index, validator in enumerate(state.validators):
         balance = state.balances[index]
@@ -1577,14 +1591,41 @@ def process_final_updates(state: BeaconState) -> None:
             or validator.effective_balance + UPWARD_THRESHOLD < balance
         ):
             validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
+```
+
+#### Slashings balances updates
+
+```python
+def process_slashings_reset(state: BeaconState) -> None:
+    next_epoch = Epoch(get_current_epoch(state) + 1)
     # Reset slashings
     state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0)
+```
+
+#### Randao mixes updates
+
+```python
+def process_randao_mixes_reset(state: BeaconState) -> None:
+    current_epoch = get_current_epoch(state)
+    next_epoch = Epoch(current_epoch + 1)
     # Set randao mix
     state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch)
+```
+
+#### Historical roots updates
+```python
+def process_historical_roots_update(state: BeaconState) -> None:
     # Set historical root accumulator
+    next_epoch = Epoch(get_current_epoch(state) + 1)
     if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
         historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots)
         state.historical_roots.append(hash_tree_root(historical_batch))
+```
+
+#### Participation records rotation
+
+```python
+def process_participation_record_updates(state: BeaconState) -> None:
     # Rotate current/previous epoch attestations
     state.previous_epoch_attestations = state.current_epoch_attestations
     state.current_epoch_attestations = []

specs/phase1/beacon-chain.md

@@ -1054,11 +1054,16 @@ def process_epoch(state: BeaconState) -> None:
     process_justification_and_finalization(state)
     process_rewards_and_penalties(state)
     process_registry_updates(state)
-    process_reveal_deadlines(state)
+    process_reveal_deadlines(state)  # Phase 1
-    process_challenge_deadlines(state)
+    process_challenge_deadlines(state)  # Phase 1
     process_slashings(state)
-    process_final_updates(state)  # phase 0 final updates
+    process_eth1_data_reset(state)
-    process_phase_1_final_updates(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_roots_update(state)
+    process_participation_record_updates(state)
+    process_phase_1_final_updates(state)  # Phase 1
 ```
 
 #### Phase 1 final updates

tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py

@@ -1,13 +1,22 @@
 
 process_calls = [
+    # PHASE0
     'process_justification_and_finalization',
     'process_rewards_and_penalties',
     'process_registry_updates',
     'process_reveal_deadlines',
     'process_challenge_deadlines',
     'process_slashings',
-    'process_final_updates',
+    'process_eth1_data_reset',
-    'after_process_final_updates',
+    'process_effective_balance_updates',
+    'process_slashings_reset',
+    'process_randao_mixes_reset',
+    'process_historical_roots_update',
+    'process_participation_record_updates',
+    # LIGHTCLIENT_PATCH
+    'process_sync_committee_updates',
+    # PHASE1
+    'process_phase_1_final_updates',
 ]
 
 

tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py

@@ -27,7 +27,7 @@ def test_sync_committees_progress(spec, state):
     assert state.current_sync_committee == first_sync_committee
     assert state.next_sync_committee == second_sync_committee
 
-    yield from run_epoch_processing_with(spec, state, 'process_final_updates')
+    yield from run_epoch_processing_with(spec, state, 'process_sync_committee_updates')
 
     # Can compute the third committee having computed final balances in the last epoch
     # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD`

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py

@@ -2,45 +2,10 @@ from eth2spec.test.context import spec_state_test, with_all_phases
 from eth2spec.test.helpers.epoch_processing import (
     run_epoch_processing_with, run_epoch_processing_to
 )
-from eth2spec.test.helpers.state import transition_to
 
 
-def run_process_final_updates(spec, state):
+def run_process_effective_balance_updates(spec, state):
-    yield from run_epoch_processing_with(spec, state, 'process_final_updates')
+    yield from run_epoch_processing_with(spec, state, 'process_effective_balance_updates')
-
-
-@with_all_phases
-@spec_state_test
-def test_eth1_vote_no_reset(spec, state):
-    assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1
-    # skip ahead to the end of the epoch
-    transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1)
-
-    for i in range(state.slot + 1):  # add a vote for each skipped slot.
-        state.eth1_data_votes.append(
-            spec.Eth1Data(deposit_root=b'\xaa' * 32,
-                          deposit_count=state.eth1_deposit_index,
-                          block_hash=b'\xbb' * 32))
-
-    yield from run_process_final_updates(spec, state)
-
-    assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH
-
-
-@with_all_phases
-@spec_state_test
-def test_eth1_vote_reset(spec, state):
-    # skip ahead to the end of the voting period
-    state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1
-    for i in range(state.slot + 1):  # add a vote for each skipped slot.
-        state.eth1_data_votes.append(
-            spec.Eth1Data(deposit_root=b'\xaa' * 32,
-                          deposit_count=state.eth1_deposit_index,
-                          block_hash=b'\xbb' * 32))
-
-    yield from run_process_final_updates(spec, state)
-
-    assert len(state.eth1_data_votes) == 0
 
 
 @with_all_phases
@@ -48,7 +13,7 @@ def test_eth1_vote_reset(spec, state):
 def test_effective_balance_hysteresis(spec, state):
     # Prepare state up to the final-updates.
     # Then overwrite the balances, we only want to focus to be on the hysteresis based changes.
-    run_epoch_processing_to(spec, state, 'process_final_updates')
+    run_epoch_processing_to(spec, state, 'process_effective_balance_updates')
     # Set some edge cases for balances
     max = spec.MAX_EFFECTIVE_BALANCE
     min = spec.EJECTION_BALANCE
@@ -79,21 +44,7 @@ def test_effective_balance_hysteresis(spec, state):
         state.validators[i].effective_balance = pre_eff
         state.balances[i] = bal
 
-    yield 'pre', state
+    yield from run_process_effective_balance_updates(spec, state)
-    spec.process_final_updates(state)
-    yield 'post', state
 
     for i, (_, _, post_eff, name) in enumerate(cases):
         assert state.validators[i].effective_balance == post_eff, name
-
-
-@with_all_phases
-@spec_state_test
-def test_historical_root_accumulator(spec, state):
-    # skip ahead to near the end of the historical roots period (excl block before epoch processing)
-    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
-    history_len = len(state.historical_roots)
-
-    yield from run_process_final_updates(spec, state)
-
-    assert len(state.historical_roots) == history_len + 1

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py

@@ -0,0 +1,43 @@
+from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with,
+)
+from eth2spec.test.helpers.state import transition_to
+
+
+def run_process_eth1_data_reset(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_eth1_data_reset')
+
+
+@with_all_phases
+@spec_state_test
+def test_eth1_vote_no_reset(spec, state):
+    assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1
+    # skip ahead to the end of the epoch
+    transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1)
+
+    for i in range(state.slot + 1):  # add a vote for each skipped slot.
+        state.eth1_data_votes.append(
+            spec.Eth1Data(deposit_root=b'\xaa' * 32,
+                          deposit_count=state.eth1_deposit_index,
+                          block_hash=b'\xbb' * 32))
+
+    yield from run_process_eth1_data_reset(spec, state)
+
+    assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH
+
+
+@with_all_phases
+@spec_state_test
+def test_eth1_vote_reset(spec, state):
+    # skip ahead to the end of the voting period
+    state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1
+    for i in range(state.slot + 1):  # add a vote for each skipped slot.
+        state.eth1_data_votes.append(
+            spec.Eth1Data(deposit_root=b'\xaa' * 32,
+                          deposit_count=state.eth1_deposit_index,
+                          block_hash=b'\xbb' * 32))
+
+    yield from run_process_eth1_data_reset(spec, state)
+
+    assert len(state.eth1_data_votes) == 0

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py

@@ -0,0 +1,20 @@
+from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_historical_roots_update(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update')
+
+
+@with_all_phases
+@spec_state_test
+def test_historical_root_accumulator(spec, state):
+    # skip ahead to near the end of the historical roots period (excl block before epoch processing)
+    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
+    history_len = len(state.historical_roots)
+
+    yield from run_process_historical_roots_update(spec, state)
+
+    assert len(state.historical_roots) == history_len + 1

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py

@@ -0,0 +1,21 @@
+from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_participation_record_updates(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_participation_record_updates')
+
+
+@with_all_phases
+@spec_state_test
+def test_updated_participation_record(spec, state):
+    state.previous_epoch_attestations = [spec.PendingAttestation(proposer_index=100)]
+    current_epoch_attestations = [spec.PendingAttestation(proposer_index=200)]
+    state.current_epoch_attestations = current_epoch_attestations
+
+    yield from run_process_participation_record_updates(spec, state)
+
+    assert state.previous_epoch_attestations == current_epoch_attestations
+    assert state.current_epoch_attestations == []

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py

@@ -0,0 +1,21 @@
+from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_randao_mixes_reset(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_reset')
+
+
+@with_all_phases
+@spec_state_test
+def test_updated_randao_mixes(spec, state):
+    next_epoch = spec.get_current_epoch(state) + 1
+    state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] = b'\x56' * 32
+
+    yield from run_process_randao_mixes_reset(spec, state)
+
+    assert state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == spec.get_randao_mix(
+        state, spec.get_current_epoch(state)
+    )

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py

@@ -0,0 +1,20 @@
+from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_slashings_reset(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_slashings_reset')
+
+
+@with_all_phases
+@spec_state_test
+def test_flush_slashings(spec, state):
+    next_epoch = spec.get_current_epoch(state) + 1
+    state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] = 100
+    assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] != 0
+
+    yield from run_process_slashings_reset(spec, state)
+
+    assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] == 0

tests/formats/epoch_processing/README.md

@@ -37,10 +37,17 @@ The provided pre-state is already transitioned to just before the specific sub-t
 
 Sub-transitions:
 
+Sub-transitions:
+
 - `justification_and_finalization`
-- `rewards_and_penalties` (limited to `minimal` config)
+- `rewards_and_penalties`
 - `registry_updates`
 - `slashings`
-- `final_updates`
+- `eth1_data_reset`
+- `effective_balance_updates`
+- `slashings_reset`
+- `randao_mixes_reset`
+- `historical_roots_update`
+- `participation_record_updates`
 
 The resulting state should match the expected `post` state.

tests/generators/README.md

@@ -184,7 +184,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin
 
 if __name__ == "__main__":
     gen_runner.run_generator("epoch_processing", [
-        create_provider('final_updates', test_process_final_updates, 'minimal'),
+        create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'),
         ...
     ])
 

tests/generators/epoch_processing/main.py

@@ -33,16 +33,21 @@ def create_provider(fork_name: str, handler_name: str,
 
 if __name__ == "__main__":
     phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [
-        'final_updates',
         'justification_and_finalization',
-        'registry_updates',
         'rewards_and_penalties',
+        'registry_updates',
         'slashings',
+        'eth1_data_reset',
+        'effective_balance_updates',
+        'slashings_reset',
+        'randao_mixes_reset',
+        'historical_roots_update',
+        'participation_record_updates',
     ]}
     phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [
+        'reveal_deadlines',
         'challenge_deadlines',
         'custody_final_updates',
-        'reveal_deadlines',
     ]}, **phase_0_mods}  # also run the previous phase 0 tests (but against phase 1 spec)
 
     gen_runner.run_generator(f"epoch_processing", [