Merge branch 'dev' into lc-eph

This commit is contained in:
Etan Kissling 2023-01-05 13:52:53 +01:00
commit 375436094c
No known key found for this signature in database
GPG Key ID: B21DA824C5A3D03D
16 changed files with 334 additions and 40 deletions

133
.github/workflows/run-tests.yml vendored Normal file
View File

@ -0,0 +1,133 @@
name: Run spec tests and linter
defaults:
run:
shell: zsh {0}
env:
TEST_PRESET_TYPE: "minimal"
DEFAULT_BRANCH: "dev"
# Run tests on workflow_Dispatch
on:
push:
branches:
- dev
- master
pull_request:
workflow_dispatch:
inputs:
test_preset_type:
default: minimal
description: Type of test to run, either mainnet or minimal
type: string
required: true
commitRef:
description: The branch, tag or SHA to checkout and build from
default: dev
required: true
schedule:
- cron: '0 0 * * *'
jobs:
precleanup:
runs-on: self-hosted
if: always()
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
setup-env:
runs-on: self-hosted
needs: precleanup
steps:
- name: Checkout this repo
uses: actions/checkout@v3.2.0
with:
ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
- uses: actions/cache@v3.2.2
id: cache-git
with:
path: ./*
key: ${{ github.sha }}
table_of_contents:
runs-on: self-hosted
needs: setup-env
steps:
- uses: actions/cache@v3.2.2
id: restore-build
with:
path: ./*
key: ${{ github.sha }}
- name: Check table of contents
run: sudo npm install -g doctoc@2 && make check_toc
codespell:
runs-on: self-hosted
needs: setup-env
steps:
- name: Check codespell
run: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell
lint:
runs-on: self-hosted
needs: setup-env
steps:
- name: Run linter for pyspec
run: make lint
- name: Run linter for test generators
run: make lint_generators
pyspec-tests:
runs-on: self-hosted
needs: [setup-env,lint,codespell,table_of_contents]
strategy:
matrix:
version: ["phase0", "altair", "bellatrix", "capella", "eip4844"]
steps:
- uses: actions/cache@v3.2.2
id: restore-build
with:
path: ./*
key: ${{ github.sha }}
- name: set TEST_PRESET_TYPE
if: github.event.inputs.test_preset_type != ''
run: |
echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }}
run: |
echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE}}" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
run: |
echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
- name: set TEST_PRESET_TYPE
if: github.event.schedule=='0 0 * * *'
run: |
echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV
- name: Install pyspec requirements
run: make install_test
- name: test-${{ matrix.version }}
run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}}
- uses: actions/upload-artifact@v3
if: always()
with:
name: test-${{ matrix.version }}
path: tests/core/pyspec/test-reports
cleanup:
runs-on: self-hosted
needs: [setup-env,pyspec-tests,codespell,lint,table_of_contents]
if: always()
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./

View File

@ -13,7 +13,7 @@ SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_cont
SOLIDITY_FILE_NAME = deposit_contract.json
DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester
CONFIGS_DIR = ./configs
TEST_PRESET_TYPE ?= minimal
# Collect a list of generator names
GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
# Map this list of generator paths to "gen_{generator name}" entries
@ -96,30 +96,30 @@ generate_tests: $(GENERATOR_TARGETS)
# "make pyspec" to create the pyspec for all phases.
pyspec:
. venv/bin/activate; python3 setup.py pyspecdev
python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev
# installs the packages to run pyspec tests
install_test:
python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[lint]; python3 -m pip install -e .[test]
# Testing against `minimal` config by default
# Testing against `minimal` or `mainnet` config by default
test: pyspec
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
# Testing against `minimal` config by default
# Testing against `minimal` or `mainnet` config by default
find_test: pyspec
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
citest: pyspec
mkdir -p $(TEST_REPORT_DIR);
ifdef fork
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
else
. venv/bin/activate; cd $(PY_SPEC_DIR); \
python3 -m pytest -n 4 --bls-type=milagro --junitxml=test-reports/test_results.xml eth2spec
python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
endif
@ -135,7 +135,7 @@ check_toc: $(MARKDOWN_FILES:=.toc)
rm $*.tmp
codespell:
codespell . --skip ./.git -I .codespell-whitelist
codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist
# TODO: add future protocol upgrade patch packages to linting.
# NOTE: we use `pylint` just for catching unused arguments in spec code

View File

@ -675,6 +675,7 @@ def get_empty_list_result(fn): # type: ignore
process_withdrawals = no_op(process_withdrawals)
process_bls_to_execution_change = no_op(process_bls_to_execution_change)
get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals)
process_historical_summaries_update = no_op(process_historical_summaries_update)
#

View File

@ -18,6 +18,7 @@
- [`Withdrawal`](#withdrawal)
- [`BLSToExecutionChange`](#blstoexecutionchange)
- [`SignedBLSToExecutionChange`](#signedblstoexecutionchange)
- [`HistoricalSummary`](#historicalsummary)
- [Extended Containers](#extended-containers)
- [`ExecutionPayload`](#executionpayload)
- [`ExecutionPayloadHeader`](#executionpayloadheader)
@ -29,6 +30,8 @@
- [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator)
- [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator)
- [Beacon chain state transition function](#beacon-chain-state-transition-function)
- [Epoch processing](#epoch-processing)
- [Historical summaries updates](#historical-summaries-updates)
- [Block processing](#block-processing)
- [New `get_expected_withdrawals`](#new-get_expected_withdrawals)
- [New `process_withdrawals`](#new-process_withdrawals)
@ -115,6 +118,18 @@ class SignedBLSToExecutionChange(Container):
signature: BLSSignature
```
#### `HistoricalSummary`
```python
class HistoricalSummary(Container):
"""
`HistoricalSummary` matches the components of the phase0 `HistoricalBatch`
making the two hash_tree_root-compatible.
"""
block_summary_root: Root
state_summary_root: Root
```
### Extended Containers
#### `ExecutionPayload`
@ -196,7 +211,7 @@ class BeaconState(Container):
latest_block_header: BeaconBlockHeader
block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] # Frozen in Capella, replaced by historical_summaries
# Eth1
eth1_data: Eth1Data
eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
@ -226,6 +241,8 @@ class BeaconState(Container):
# Withdrawals
next_withdrawal_index: WithdrawalIndex # [New in Capella]
next_withdrawal_validator_index: ValidatorIndex # [New in Capella]
# Deep history valid from Capella onwards
historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT] # [New in Capella]
```
## Helpers
@ -270,6 +287,40 @@ def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) ->
## Beacon chain state transition function
### Epoch processing
*Note*: The function `process_historical_summaries_update` replaces `process_historical_roots_update` in Bellatrix.
```python
def process_epoch(state: BeaconState) -> None:
process_justification_and_finalization(state)
process_inactivity_updates(state)
process_rewards_and_penalties(state)
process_registry_updates(state)
process_slashings(state)
process_eth1_data_reset(state)
process_effective_balance_updates(state)
process_slashings_reset(state)
process_randao_mixes_reset(state)
process_historical_summaries_update(state) # [Modified in Capella]
process_participation_flag_updates(state)
process_sync_committee_updates(state)
```
#### Historical summaries updates
```python
def process_historical_summaries_update(state: BeaconState) -> None:
# Set historical block root accumulator.
next_epoch = Epoch(get_current_epoch(state) + 1)
if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
historical_summary = HistoricalSummary(
block_summary_root=hash_tree_root(state.block_roots),
state_summary_root=hash_tree_root(state.state_roots),
)
state.historical_summaries.append(historical_summary)
```
### Block processing
```python

View File

@ -141,8 +141,8 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/`
After `EIP4844_FORK_EPOCH`, `BeaconBlocksByRootV2` is replaced by `BeaconBlockAndBlobsSidecarByRootV1`
clients MUST support requesting blocks by root for pre-fork-epoch blocks.
After `EIP4844_FORK_EPOCH`, `BeaconBlocksByRootV2` is replaced by `BeaconBlockAndBlobsSidecarByRootV1`.
Clients MUST support requesting blocks by root for pre-fork-epoch blocks.
Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:

View File

@ -0,0 +1,27 @@
from eth2spec.test.context import (
CAPELLA,
spec_state_test,
with_phases,
)
from eth2spec.test.helpers.epoch_processing import (
run_epoch_processing_with
)
def run_process_historical_summaries_update(spec, state):
yield from run_epoch_processing_with(spec, state, 'process_historical_summaries_update')
@with_phases([CAPELLA])
@spec_state_test
def test_historical_summaries_accumulator(spec, state):
# skip ahead to near the end of the historical batch period (excl block before epoch processing)
state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
pre_historical_summaries = state.historical_summaries.copy()
yield from run_process_historical_summaries_update(spec, state)
assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
summary = state.historical_summaries[len(state.historical_summaries) - 1]
assert summary.block_summary_root == state.block_roots.hash_tree_root()
assert summary.state_summary_root == state.state_roots.hash_tree_root()

View File

@ -0,0 +1,23 @@
from eth2spec.test.context import (
spec_state_test,
with_eip4844_and_later,
)
from eth2spec.test.helpers.epoch_processing import (
run_epoch_processing_with
)
def run_process_historical_summaries_update(spec, state):
yield from run_epoch_processing_with(spec, state, 'process_historical_summaries_update')
@with_eip4844_and_later
@spec_state_test
def test_no_op(spec, state):
# skip ahead to near the end of the historical batch period (excl block before epoch processing)
state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
historical_summaries_len = len(state.historical_summaries)
yield from run_process_historical_summaries_update(spec, state)
assert len(state.historical_summaries) == historical_summaries_len

View File

@ -1,5 +1,8 @@
from eth2spec.test.helpers.forks import is_post_altair
from eth2spec.test.helpers.forks import (
is_post_altair,
is_post_capella,
)
def get_process_calls(spec):
@ -22,7 +25,10 @@ def get_process_calls(spec):
'process_effective_balance_updates',
'process_slashings_reset',
'process_randao_mixes_reset',
'process_historical_roots_update',
# Capella replaced `process_historical_roots_update` with `process_historical_summaries_update`
'process_historical_summaries_update' if is_post_capella(spec) else (
'process_historical_roots_update'
),
# Altair replaced `process_participation_record_updates` with `process_participation_flag_updates`
'process_participation_flag_updates' if is_post_altair(spec) else (
'process_participation_record_updates'

View File

@ -1,4 +1,8 @@
from eth2spec.test.context import spec_state_test, with_all_phases
from eth2spec.test.context import (
PHASE0, ALTAIR, BELLATRIX,
spec_state_test,
with_phases,
)
from eth2spec.test.helpers.epoch_processing import (
run_epoch_processing_with
)
@ -8,7 +12,7 @@ def run_process_historical_roots_update(spec, state):
yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update')
@with_all_phases
@with_phases([PHASE0, ALTAIR, BELLATRIX])
@spec_state_test
def test_historical_root_accumulator(spec, state):
# skip ahead to near the end of the historical roots period (excl block before epoch processing)

View File

@ -29,8 +29,8 @@ from eth2spec.test.helpers.sync_committee import (
compute_committee_indices,
compute_sync_committee_participant_reward_and_penalty,
)
from eth2spec.test.helpers.constants import PHASE0, MINIMAL
from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix
from eth2spec.test.helpers.constants import PHASE0, EIP4844, MINIMAL
from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix, is_post_capella
from eth2spec.test.context import (
spec_test, spec_state_test, dump_skipping_message,
with_phases, with_all_phases, single_phase,
@ -1026,7 +1026,10 @@ def test_balance_driven_status_transitions(spec, state):
@always_bls
def test_historical_batch(spec, state):
state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1
pre_historical_roots_len = len(state.historical_roots)
pre_historical_roots = state.historical_roots.copy()
if is_post_capella(spec):
pre_historical_summaries = state.historical_summaries.copy()
yield 'pre', state
@ -1038,7 +1041,18 @@ def test_historical_batch(spec, state):
assert state.slot == block.slot
assert spec.get_current_epoch(state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0
assert len(state.historical_roots) == pre_historical_roots_len + 1
# check history update
if is_post_capella(spec):
# Frozen `historical_roots`
assert state.historical_roots == pre_historical_roots
if spec.fork == EIP4844:
# TODO: no-op for now in EIP4844 testnet
assert state.historical_summaries == pre_historical_summaries
else:
assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
else:
assert len(state.historical_roots) == len(pre_historical_roots) + 1
@with_all_phases

View File

@ -1,3 +1,9 @@
from eth2spec.test.helpers.constants import (
EIP4844,
)
from eth2spec.test.helpers.forks import (
is_post_capella,
)
from eth2spec.test.helpers.state import get_state_root
from eth2spec.test.context import (
spec_state_test,
@ -61,3 +67,30 @@ def test_over_epoch_boundary(spec, state):
yield 'slots', int(slots)
spec.process_slots(state, state.slot + slots)
yield 'post', state
@with_all_phases
@spec_state_test
def test_historical_accumulator(spec, state):
pre_historical_roots = state.historical_roots.copy()
if is_post_capella(spec):
pre_historical_summaries = state.historical_summaries.copy()
yield 'pre', state
slots = spec.SLOTS_PER_HISTORICAL_ROOT
yield 'slots', int(slots)
spec.process_slots(state, state.slot + slots)
yield 'post', state
# check history update
if is_post_capella(spec):
# Frozen `historical_roots`
assert state.historical_roots == pre_historical_roots
if spec.fork == EIP4844:
# TODO: no-op for now in EIP4844 testnet
assert state.historical_summaries == pre_historical_summaries
else:
assert len(state.historical_summaries) == len(pre_historical_summaries) + 1
else:
assert len(state.historical_roots) == len(pre_historical_roots) + 1

View File

@ -41,11 +41,10 @@ Sub-transitions:
- `effective_balance_updates`
- `slashings_reset`
- `randao_mixes_reset`
- `historical_roots_update`
- `historical_roots_update` (Phase0, Altair, Bellatrix only)
- `historical_summaries_update` (Capella)
- `participation_record_updates` (Phase 0 only)
- `participation_flag_updates` (Altair)
- `sync_committee_updates` (Altair)
- `full_withdrawals` (Capella)
- `partial_withdrawals` (Capella)
The resulting state should match the expected `post` state.

View File

@ -33,17 +33,18 @@ This excludes the other parts of the block-transition.
Operations:
| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* |
|-------------------------|-----------------------|----------------------|----------------------------------------------------------------------|
| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` |
| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` |
| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` |
| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` |
| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` |
| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` |
| `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_aggregate(state, sync_aggregate)` (new in Altair) |
| `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Bellatrix) |
| `bls_to_execution_change` | `SignedBLSToExecutionChange` | `signed_address_change` | `process_bls_to_execution_change(state, signed_address_change)` (new in Capella) |
| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* |
|---------------------------|------------------------------|---------------------|----------------------------------------------------------------------------------|
| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` |
| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` |
| `block_header` | `BeaconBlock` | **`block`** | `process_block_header(state, block)` |
| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` |
| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` |
| `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` |
| `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_aggregate(state, sync_aggregate)` (new in Altair) |
| `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Bellatrix) |
| `withdrawals` | `ExecutionPayload` | `execution_payload` | `process_withdrawals(state, execution_payload)` (new in Capella) |
| `bls_to_execution_change` | `SignedBLSToExecutionChange` | `address_change` | `process_bls_to_execution_change(state, address_change)` (new in Capella) |
Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here.

View File

@ -27,13 +27,15 @@ if __name__ == "__main__":
# so no additional tests required.
bellatrix_mods = altair_mods
# No epoch-processing changes in Capella and previous testing repeats with new types,
# so no additional tests required.
capella_mods = bellatrix_mods
_new_capella_mods = {key: 'eth2spec.test.capella.epoch_processing.test_process_' + key for key in [
'historical_summaries_update',
]}
capella_mods = combine_mods(_new_capella_mods, bellatrix_mods)
# No epoch-processing changes in EIP4844 and previous testing repeats with new types,
# so no additional tests required.
eip4844_mods = capella_mods
_new_eip4844_mods = {key: 'eth2spec.test.eip4844.epoch_processing.test_process_' + key for key in [
'historical_summaries_update',
]}
eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods)
# TODO Custody Game testgen is disabled for now
# custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [