basic capella withdrawal in place

Makefile

@@ -133,7 +133,7 @@ codespell:
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.merge
+	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.merge -p eth2spec.capella
 
 lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

configs/minimal.yaml

@@ -36,11 +36,16 @@ ALTAIR_FORK_EPOCH: 18446744073709551615
 # Merge
 MERGE_FORK_VERSION: 0x02000001
 MERGE_FORK_EPOCH: 18446744073709551615
+# Capella
+CAPELLA_FORK_VERSION: 0x03000001
+CAPELLA_FORK_EPOCH: 18446744073709551615
 # Sharding
-SHARDING_FORK_VERSION: 0x03000001
+SHARDING_FORK_VERSION: 0x04000001
 SHARDING_FORK_EPOCH: 18446744073709551615
 
 
+
+
 # Time parameters
 # ---------------------------------------------------------------
 # [customized] Faster for testing purposes

setup.py

@@ -41,6 +41,7 @@ from marko.ext.gfm.elements import Table
 PHASE0 = 'phase0'
 ALTAIR = 'altair'
 MERGE = 'merge'
+CAPELLA = 'capella'
 
 # The helper functions that are used when defining constants
 CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = '''
@@ -548,9 +549,38 @@ EXECUTION_ENGINE = NoopExecutionEngine()"""
         return {**super().hardcoded_custom_type_dep_constants(), **constants}
 
 
+#
+# CapellaSpecBuilder
+#
+class CapellaSpecBuilder(MergeSpecBuilder):
+    fork: str = CAPELLA
+
+    @classmethod
+    def imports(cls, preset_name: str):
+        return super().imports(preset_name) + f'''
+from eth2spec.merge import {preset_name} as merge
+'''
+
+    @classmethod
+    def preparations(cls):
+        return super().preparations()
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return super().sundry_functions()
+
+    @classmethod
+    def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
+        constants = {
+            'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)',
+            'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)',
+        }
+        return {**super().hardcoded_ssz_dep_constants(), **constants}
+
+
 spec_builders = {
     builder.fork: builder
-    for builder in (Phase0SpecBuilder, AltairSpecBuilder, MergeSpecBuilder)
+    for builder in (Phase0SpecBuilder, AltairSpecBuilder, MergeSpecBuilder, CapellaSpecBuilder)
 }
 
 
@@ -845,14 +875,14 @@ class PySpecCommand(Command):
         if len(self.md_doc_paths) == 0:
             print("no paths were specified, using default markdown file paths for pyspec"
                   " build (spec fork: %s)" % self.spec_fork)
-            if self.spec_fork in (PHASE0, ALTAIR, MERGE):
+            if self.spec_fork in (PHASE0, ALTAIR, MERGE, CAPELLA):
                 self.md_doc_paths = """
                     specs/phase0/beacon-chain.md
                     specs/phase0/fork-choice.md
                     specs/phase0/validator.md
                     specs/phase0/weak-subjectivity.md
                 """
-            if self.spec_fork in (ALTAIR, MERGE):
+            if self.spec_fork in (ALTAIR, MERGE, CAPELLA):
                 self.md_doc_paths += """
                     specs/altair/beacon-chain.md
                     specs/altair/bls.md
@@ -861,13 +891,20 @@ class PySpecCommand(Command):
                     specs/altair/p2p-interface.md
                     specs/altair/sync-protocol.md
                 """
-            if self.spec_fork == MERGE:
+            if self.spec_fork in (MERGE, CAPELLA):
                 self.md_doc_paths += """
                     specs/merge/beacon-chain.md
                     specs/merge/fork.md
                     specs/merge/fork-choice.md
                     specs/merge/validator.md
                 """
+            if self.spec_fork == CAPELLA:
+                self.md_doc_paths += """
+                    specs/capella/beacon-chain.md
+                    specs/capella/fork.md
+                    specs/capella/validator.md
+                    specs/capella/p2p-interface.md
+                """
             if len(self.md_doc_paths) == 0:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)
 

specs/capella/beacon-chain.md

@@ -0,0 +1,163 @@
+# Cappela -- The Beacon Chain
+
+## Table of contents
+
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+## Introduction
+
+Cappela is a consensus-layer upgrade containin a number of features related
+to validator withdrawals. Including:
+* Automatic withdrawals of `withdrawable` validators
+* Partial withdrawals during block proposal
+* Operation to change from `BLS_WITHDRAWAL_PREFIX` to
+  `ETH1_ADDRESS_WITHDRAWAL_PREFIX` versioned withdrawal credentials to enable withdrawals for a validator
+
+## Custom types
+
+| Name | SSZ equivalent | Description |
+| - | - | - |
+| `WithdrawalReceiptIndex` | `uint64` | a withdrawal receipt index |
+
+## Constants
+
+## Preset
+
+### State list lengths
+
+| Name | Value | Unit | Duration |
+| - | - | :-: | :-: |
+| `WITHDRAWAL_RECEIPT_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | withdrawal receipts|
+
+## Configuration
+
+## Containers
+
+### Extended Containers
+
+#### `BeaconState`
+
+```python
+class BeaconState(Container):
+    # Versioning
+    genesis_time: uint64
+    genesis_validators_root: Root
+    slot: Slot
+    fork: Fork
+    # History
+    latest_block_header: BeaconBlockHeader
+    block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    # Eth1
+    eth1_data: Eth1Data
+    eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
+    eth1_deposit_index: uint64
+    # Registry
+    validators: List[Validator, VALIDATOR_REGISTRY_LIMIT]
+    balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT]
+    # Randomness
+    randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR]
+    # Slashings
+    slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR]  # Per-epoch sums of slashed effective balances
+    # Participation
+    previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    # Finality
+    justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH]  # Bit set for every recent justified epoch
+    previous_justified_checkpoint: Checkpoint
+    current_justified_checkpoint: Checkpoint
+    finalized_checkpoint: Checkpoint
+    # Inactivity
+    inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT]
+    # Sync
+    current_sync_committee: SyncCommittee
+    next_sync_committee: SyncCommittee
+    # Execution
+    latest_execution_payload_header: ExecutionPayloadHeader
+    # Withdrawals
+    withdrawal_receipts: List[WithdrawalReceipt, WITHDRAWAL_RECEIPT_LIMIT]  # [New in Cappela]
+```
+
+### New containers
+
+#### `WithdrawalReceipt`
+
+```python
+class WithdrawalReceipt(Container):
+    index: WithdrawalReceiptIndex
+    address: ExecutionAddress
+    amount: Gwei
+```
+
+## Helpers
+
+### Beacon state mutators
+
+#### `withdraw`
+
+```python
+def withdraw(state: BeaconState, index: ValidatorIndex, amount: Gwei) -> None:
+    # Decrease the validator's balance
+    decrease_balance(state, index, amount)
+    # Create a corresponding withdrawal receipt
+    receipt = WithdrawalReceipt(
+        index=WithdrawalReceiptIndex(len(state.withdrawal_receipts)),
+        address=state.validators[index].withdrawal_credentials[12:],
+        amount=amount,
+    )
+    state.withdrawal_receipts.append(receipt)
+```
+
+### Predicates
+
+#### `is_withdrawable_validator`
+
+```python
+def is_withdrawable_validator(validator: Validator, epoch: Epoch) -> bool:
+    """
+    Check if ``validator`` is withdrawable.
+    """
+    return validator.withdrawable_epoch <= epoch
+```
+
+## Beacon chain state transition function
+
+### Epoch processing
+
+```python
+def process_epoch(state: BeaconState) -> None:
+    process_justification_and_finalization(state)
+    process_inactivity_updates(state)
+    process_rewards_and_penalties(state)
+    process_registry_updates(state)
+    process_slashings(state)
+    process_eth1_data_reset(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_roots_update(state)
+    process_participation_flag_updates(state)
+    process_sync_committee_updates(state)
+    process_withdrawals(state)  # [New in Cappela]
+```
+
+#### Withdrawals
+
+*Note*: The function `process_inactivity_updates` is new.
+
+```python
+def process_withdrawals(state: BeaconState) -> None:
+    current_epoch = get_current_epoch(state)
+    for index, validator in enumerate(state.validators):
+        balance = state.balances[index]
+        is_balance_nonzero = state.balances[index] == 0
+        is_eth1_withdrawal_prefix = validator.withdrawal_credentials[0] != ETH1_ADDRESS_WITHDRAWAL_PREFIX
+        if is_balance_nonzero and is_eth1_withdrawal_prefix and is_withdrawable_validator(validator, current_epoch):
+            withdraw(state, ValidatorIndex(index), balance)
+```

specs/capella/fork.md

@@ -0,0 +1,90 @@
+# Capella -- Fork Logic
+
+## Table of contents
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+This document describes the process of the Capella upgrade.
+
+## Configuration
+
+Warning: this configuration is not definitive.
+
+| Name | Value |
+| - | - |
+| `CAPELLA_FORK_VERSION` | `Version('0x03000000')` |
+| `CAPELLA_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** |
+
+
+## Fork to Capella
+
+### Fork trigger
+
+The fork is triggered at epoch `CAPELLA_FORK_EPOCH`.
+
+Note that for the pure Capella networks, we don't apply `upgrade_to_capella` since it starts with Capella version logic.
+
+### Upgrading the state
+
+If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == CAPELLA_FORK_EPOCH`,
+an irregular state change is made to upgrade to Capella.
+
+The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `CAPELLA_FORK_EPOCH * SLOTS_PER_EPOCH`.
+Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document.
+In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead the logic must be within `process_slots`.
+
+```python
+def upgrade_to_capella(pre: merge.BeaconState) -> BeaconState:
+    epoch = merge.get_current_epoch(pre)
+    post = BeaconState(
+        # Versioning
+        genesis_time=pre.genesis_time,
+        genesis_validators_root=pre.genesis_validators_root,
+        slot=pre.slot,
+        fork=Fork(
+            previous_version=pre.fork.current_version,
+            current_version=CAPELLA_FORK_VERSION,
+            epoch=epoch,
+        ),
+        # History
+        latest_block_header=pre.latest_block_header,
+        block_roots=pre.block_roots,
+        state_roots=pre.state_roots,
+        historical_roots=pre.historical_roots,
+        # Eth1
+        eth1_data=pre.eth1_data,
+        eth1_data_votes=pre.eth1_data_votes,
+        eth1_deposit_index=pre.eth1_deposit_index,
+        # Registry
+        validators=pre.validators,
+        balances=pre.balances,
+        # Randomness
+        randao_mixes=pre.randao_mixes,
+        # Slashings
+        slashings=pre.slashings,
+        # Participation
+        previous_epoch_participation=pre.previous_epoch_participation,
+        current_epoch_participation=pre.current_epoch_participation,
+        # Finality
+        justification_bits=pre.justification_bits,
+        previous_justified_checkpoint=pre.previous_justified_checkpoint,
+        current_justified_checkpoint=pre.current_justified_checkpoint,
+        finalized_checkpoint=pre.finalized_checkpoint,
+        # Inactivity
+        inactivity_scores=pre.inactivity_scores,
+        # Sync
+        current_sync_committee=pre.current_sync_committee,
+        next_sync_committee=pre.next_sync_committee,
+        # Execution-layer
+        latest_execution_payload_header=pre.latest_execution_payload_header,
+        # Withdrawals
+        withdrawal_receipts=[],
+    )
+
+    return post
+```

tests/core/pyspec/eth2spec/capella/__init__.py

@@ -0,0 +1 @@
+from . import mainnet as spec  # noqa:F401

tests/core/pyspec/eth2spec/capella/fork/test_capella_fork_basic.py

@@ -0,0 +1,82 @@
+from eth2spec.test.context import (
+    with_phases,
+    with_custom_state,
+    with_presets,
+    spec_test, with_state,
+    low_balances, misc_balances, large_validator_set,
+)
+from eth2spec.test.utils import with_meta_tags
+from eth2spec.test.helpers.constants import (
+    MERGE, CAPELLA,
+    MINIMAL,
+)
+from eth2spec.test.helpers.state import (
+    next_epoch,
+    next_epoch_via_block,
+)
+from eth2spec.test.helpers.capella.fork import (
+    CAPELLA_FORK_TEST_META_TAGS,
+    run_fork_test,
+)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@spec_test
+@with_state
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_base_state(spec, phases, state):
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@spec_test
+@with_state
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_next_epoch(spec, phases, state):
+    next_epoch(spec, state)
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@spec_test
+@with_state
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_next_epoch_with_block(spec, phases, state):
+    next_epoch_via_block(spec, state)
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@spec_test
+@with_state
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_many_next_epoch(spec, phases, state):
+    for _ in range(3):
+        next_epoch(spec, state)
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_random_low_balances(spec, phases, state):
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_random_misc_balances(spec, phases, state):
+    yield from run_fork_test(phases[CAPELLA], state)
+
+
+@with_phases(phases=[MERGE], other_phases=[CAPELLA])
+@with_presets([MINIMAL],
+              reason="mainnet config leads to larger validator set than limit of public/private keys pre-generated")
+@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.config.EJECTION_BALANCE)
+@spec_test
+@with_meta_tags(CAPELLA_FORK_TEST_META_TAGS)
+def test_fork_random_large_validator_set(spec, phases, state):
+    yield from run_fork_test(phases[CAPELLA], state)

tests/core/pyspec/eth2spec/test/context.py

@@ -5,12 +5,14 @@ from dataclasses import dataclass
 from eth2spec.phase0 import mainnet as spec_phase0_mainnet, minimal as spec_phase0_minimal
 from eth2spec.altair import mainnet as spec_altair_mainnet, minimal as spec_altair_minimal
 from eth2spec.merge import mainnet as spec_merge_mainnet, minimal as spec_merge_minimal
+from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal
 from eth2spec.utils import bls
 
 from .exceptions import SkippedTest
 from .helpers.constants import (
-    PHASE0, ALTAIR, MERGE, MINIMAL, MAINNET,
-    ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE,
+    PHASE0, ALTAIR, MERGE, CAPELLA,
+    MINIMAL, MAINNET,
+    ALL_PHASES, FORKS_BEFORE_ALTAIR, FORKS_BEFORE_MERGE, FORKS_BEFORE_CAPELLA,
     ALL_FORK_UPGRADES,
 )
 from .helpers.typing import SpecForkName, PresetBaseName
@@ -56,6 +58,10 @@ class SpecMerge(Spec):
     ...
 
 
+class SpecCapella(Spec):
+    ...
+
+
 @dataclass(frozen=True)
 class ForkMeta:
     pre_fork_name: str
@@ -68,11 +74,13 @@ spec_targets: Dict[PresetBaseName, Dict[SpecForkName, Spec]] = {
         PHASE0: spec_phase0_minimal,
         ALTAIR: spec_altair_minimal,
         MERGE: spec_merge_minimal,
+        CAPELLA: spec_capella_minimal,
     },
     MAINNET: {
         PHASE0: spec_phase0_mainnet,
         ALTAIR: spec_altair_mainnet,
         MERGE: spec_merge_mainnet,
+        CAPELLA: spec_capella_mainnet,
     },
 }
 
@@ -81,6 +89,7 @@ class SpecForks(TypedDict, total=False):
     PHASE0: SpecPhase0
     ALTAIR: SpecAltair
     MERGE: SpecMerge
+    CAPELLA: SpecCapella
 
 
 def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int],
@@ -511,8 +520,13 @@ def is_post_merge(spec):
     return spec.fork not in FORKS_BEFORE_MERGE
 
 
+def is_post_capella(spec):
+    return spec.fork not in FORKS_BEFORE_CAPELLA
+
+
 with_altair_and_later = with_all_phases_except([PHASE0])
 with_merge_and_later = with_all_phases_except([PHASE0, ALTAIR])
+with_capella_and_later = with_all_phases_except([PHASE0, ALTAIR, MERGE])
 
 
 def only_generator(reason):

tests/core/pyspec/eth2spec/test/helpers/capella/fork.py

@@ -0,0 +1,47 @@
+CAPELLA_FORK_TEST_META_TAGS = {
+    'fork': 'capella',
+}
+
+
+def run_fork_test(post_spec, pre_state):
+    yield 'pre', pre_state
+
+    post_state = post_spec.upgrade_to_capella(pre_state)
+
+    # Stable fields
+    stable_fields = [
+        'genesis_time', 'genesis_validators_root', 'slot',
+        # History
+        'latest_block_header', 'block_roots', 'state_roots', 'historical_roots',
+        # Eth1
+        'eth1_data', 'eth1_data_votes', 'eth1_deposit_index',
+        # Registry
+        'validators', 'balances',
+        # Randomness
+        'randao_mixes',
+        # Slashings
+        'slashings',
+        # Participation
+        'previous_epoch_participation', 'current_epoch_participation',
+        # Finality
+        'justification_bits', 'previous_justified_checkpoint', 'current_justified_checkpoint', 'finalized_checkpoint',
+        # Inactivity
+        'inactivity_scores',
+        # Sync
+        'current_sync_committee', 'next_sync_committee',
+        # Execution
+        'latest_execution_payload_header',
+    ]
+    for field in stable_fields:
+        assert getattr(pre_state, field) == getattr(post_state, field)
+
+    # Modified fields
+    modified_fields = ['fork']
+    for field in modified_fields:
+        assert getattr(pre_state, field) != getattr(post_state, field)
+
+    assert pre_state.fork.current_version == post_state.fork.previous_version
+    assert post_state.fork.current_version == post_spec.config.CAPELLA_FORK_VERSION
+    assert post_state.fork.epoch == post_spec.get_current_epoch(post_state)
+
+    yield 'post', post_state

tests/core/pyspec/eth2spec/test/helpers/constants.py

@@ -8,6 +8,7 @@ from .typing import SpecForkName, PresetBaseName
 PHASE0 = SpecForkName('phase0')
 ALTAIR = SpecForkName('altair')
 MERGE = SpecForkName('merge')
+CAPELLA = SpecForkName('capella')
 
 # Experimental phases (not included in default "ALL_PHASES"):
 SHARDING = SpecForkName('sharding')
@@ -15,16 +16,18 @@ CUSTODY_GAME = SpecForkName('custody_game')
 DAS = SpecForkName('das')
 
 # The forks that pytest runs with.
-ALL_PHASES = (PHASE0, ALTAIR, MERGE)
+ALL_PHASES = (PHASE0, ALTAIR, MERGE, CAPELLA)
 # The forks that output to the test vectors.
 TESTGEN_FORKS = (PHASE0, ALTAIR, MERGE)
 
 FORKS_BEFORE_ALTAIR = (PHASE0,)
 FORKS_BEFORE_MERGE = (PHASE0, ALTAIR)
+FORKS_BEFORE_CAPELLA = (PHASE0, ALTAIR, MERGE)
 ALL_FORK_UPGRADES = {
     # pre_fork_name: post_fork_name
     PHASE0: ALTAIR,
     ALTAIR: MERGE,
+    MERGE: CAPELLA,
 }
 ALL_PRE_POST_FORKS = ALL_FORK_UPGRADES.items()
 AFTER_MERGE_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() if key not in FORKS_BEFORE_ALTAIR}

tests/core/pyspec/eth2spec/withdrawals/__init__.py

@@ -0,0 +1 @@
+from . import mainnet as spec  # noqa:F401