From a8160f1634eab48644a44d75ae22edb126d5f654 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 10 Apr 2021 00:53:37 +0800 Subject: [PATCH 1/9] Try to initialize state with pure Altair spec in tests --- specs/altair/beacon-chain.md | 47 +++++++++++++++++++ tests/core/pyspec/eth2spec/test/context.py | 14 ++---- .../eth2spec/test/helpers/fork_choice.py | 5 +- .../pyspec/eth2spec/test/helpers/genesis.py | 16 ++++++- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 9b476bce1..9f9af9de7 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -52,6 +52,7 @@ - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) - [Sync committee updates](#sync-committee-updates) +- [Initialize state for testnets](#initialize-state-for-testnets) @@ -661,3 +662,49 @@ def process_sync_committee_updates(state: BeaconState) -> None: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` + +## Initialize state for testnets + +This helper function is only for initialize Altair testnets and tests. + +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit]) -> BeaconState: + fork = Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=ALTAIR_FORK_VERSION, + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + # Fill in sync committees + state.current_sync_committee = get_sync_committee(state, get_current_epoch(state)) + state.next_sync_committee = get_sync_committee(state, get_current_epoch(state) + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + + return state +``` diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 7ae1b9541..34b6e79a1 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -75,17 +75,11 @@ class SpecForks(TypedDict, total=False): def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], spec: Spec, phases: SpecForks): - - p0 = phases[PHASE0] - balances = balances_fn(p0) - activation_threshold = threshold_fn(p0) - - state = create_genesis_state(spec=p0, validator_balances=balances, + phase = phases[spec.fork] + balances = balances_fn(phase) + activation_threshold = threshold_fn(phase) + state = create_genesis_state(spec=phase, validator_balances=balances, activation_threshold=activation_threshold) - # TODO: upgrade to merge spec, and later sharding. - if spec.fork == ALTAIR: - state = phases[ALTAIR].upgrade_to_altair(state) - return state diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 2ca37768b..f3b80b2ac 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -1,7 +1,5 @@ from eth_utils import encode_hex -from eth2spec.phase0 import spec as phase0_spec - def get_anchor_root(spec, state): anchor_block_header = state.latest_block_header.copy() @@ -58,8 +56,7 @@ def get_genesis_forkchoice_store(spec, genesis_state): def get_genesis_forkchoice_store_and_block(spec, genesis_state): assert genesis_state.slot == spec.GENESIS_SLOT - # The genesis block must be a Phase 0 `BeaconBlock` - genesis_block = phase0_spec.BeaconBlock(state_root=genesis_state.hash_tree_root()) + genesis_block = spec.BeaconBlock(state_root=genesis_state.hash_tree_root()) return spec.get_forkchoice_store(genesis_state, genesis_block), genesis_block diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 14fd9aa47..5f2cd5281 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -20,6 +20,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold): deposit_root = b'\x42' * 32 eth1_block_hash = b'\xda' * 32 + current_version = spec.GENESIS_FORK_VERSION + if spec.fork == 'altair': # TODO: fix `context.py` dependency + current_version = spec.ALTAIR_FORK_VERSION state = spec.BeaconState( genesis_time=0, eth1_deposit_index=len(validator_balances), @@ -30,7 +33,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): ), fork=spec.Fork( previous_version=spec.GENESIS_FORK_VERSION, - current_version=spec.GENESIS_FORK_VERSION, + current_version=current_version, epoch=spec.GENESIS_EPOCH, ), latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())), @@ -47,8 +50,19 @@ def create_genesis_state(spec, validator_balances, activation_threshold): if validator.effective_balance >= activation_threshold: validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH + if spec.fork != 'phase0': # TODO: fix `context.py` dependency + state.previous_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) + state.current_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) + state.inactivity_scores.append(spec.uint64(0)) # Set genesis validators root for domain separation and chain versioning state.genesis_validators_root = spec.hash_tree_root(state.validators) + if spec.fork != 'phase0': # TODO: fix `context.py` dependency + # Fill in sync committees + state.current_sync_committee = spec.get_sync_committee(state, spec.get_current_epoch(state)) + state.next_sync_committee = ( + spec.get_sync_committee(state, spec.get_current_epoch(state) + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + ) + return state From 8708ec4bb0e2e59ef8ac050b51ced5f4867afc1a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Apr 2021 22:02:04 +0800 Subject: [PATCH 2/9] Update doc --- specs/altair/beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 9f9af9de7..e14a0f5c0 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -663,9 +663,11 @@ def process_sync_committee_updates(state: BeaconState) -> None: state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` -## Initialize state for testnets +## Initialize state for Altair testnets -This helper function is only for initialize Altair testnets and tests. +This helper function is only for initializing the pure Altair testnets and tests, where we use the dummy where the `ALTAIR_FORK_SLOT` GENESIS_SLOT`. + +*Note*: The function `initialize_beacon_state_from_eth1` is modified with `ALTAIR_FORK_VERSION` fork version and initial sync committees. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, From bb63af53c9d1acd66eb5ab13ace6bb3fb71d320c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 13 Apr 2021 23:21:40 +0800 Subject: [PATCH 3/9] Refactor genesis.py --- tests/core/pyspec/eth2spec/test/context.py | 8 +++----- tests/core/pyspec/eth2spec/test/helpers/constants.py | 4 +++- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 12 +++++++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 147793afe..438e611cf 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -7,8 +7,8 @@ from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( - PHASE0, ALTAIR, MERGE, SHARDING, CUSTODY_GAME, DAS, - ALL_PHASES, + PHASE0, ALTAIR, + ALL_PHASES, FORKS_BEFORE_ALTAIR, ) from .helpers.genesis import create_genesis_state from .utils import vector_test, with_meta_tags @@ -362,8 +362,6 @@ def with_configs(configs, reason=None): def is_post_altair(spec): - # TODO: everything runs in parallel to Altair. - # After features are rebased on the Altair fork, this can be reduced to just PHASE0. - if spec.fork in [PHASE0, MERGE, SHARDING, CUSTODY_GAME, DAS]: + if spec.fork in FORKS_BEFORE_ALTAIR: return False return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index bffffe348..ccd7b20a2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -18,7 +18,9 @@ DAS = SpecForkName('das') ALL_PHASES = (PHASE0, ALTAIR) # The forks that output to the test vectors. TESTGEN_FORKS = (PHASE0, ALTAIR) - +# TODO: everything runs in parallel to Altair. +# After features are rebased on the Altair fork, this can be reduced to just PHASE0. +FORKS_BEFORE_ALTAIR = (PHASE0, MERGE, SHARDING, CUSTODY_GAME, DAS) # # Config diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 5f2cd5281..49af43ec1 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,3 +1,7 @@ +from eth2spec.test.helpers.constants import ( + ALTAIR, + FORKS_BEFORE_ALTAIR, +) from eth2spec.test.helpers.keys import pubkeys @@ -21,8 +25,10 @@ def create_genesis_state(spec, validator_balances, activation_threshold): eth1_block_hash = b'\xda' * 32 current_version = spec.GENESIS_FORK_VERSION - if spec.fork == 'altair': # TODO: fix `context.py` dependency + + if spec.fork == ALTAIR: current_version = spec.ALTAIR_FORK_VERSION + state = spec.BeaconState( genesis_time=0, eth1_deposit_index=len(validator_balances), @@ -50,7 +56,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): if validator.effective_balance >= activation_threshold: validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - if spec.fork != 'phase0': # TODO: fix `context.py` dependency + if spec.fork not in FORKS_BEFORE_ALTAIR: state.previous_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) state.current_epoch_participation.append(spec.ParticipationFlags(0b0000_0000)) state.inactivity_scores.append(spec.uint64(0)) @@ -58,7 +64,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): # Set genesis validators root for domain separation and chain versioning state.genesis_validators_root = spec.hash_tree_root(state.validators) - if spec.fork != 'phase0': # TODO: fix `context.py` dependency + if spec.fork not in FORKS_BEFORE_ALTAIR: # Fill in sync committees state.current_sync_committee = spec.get_sync_committee(state, spec.get_current_epoch(state)) state.next_sync_committee = ( From b65566f184f3ed84381f3e2581c246a84480b1ea Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 15 Apr 2021 15:41:12 +0800 Subject: [PATCH 4/9] Fix ToC --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index e14a0f5c0..f46df6bb2 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -52,7 +52,7 @@ - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) - [Sync committee updates](#sync-committee-updates) -- [Initialize state for testnets](#initialize-state-for-testnets) +- [Initialize state for Altair testnets](#initialize-state-for-altair-testnets) From 57ac8c3f281ff3cf0ad2b585995564aee817c320 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 16 Apr 2021 11:34:50 +0800 Subject: [PATCH 5/9] Fix doc --- specs/altair/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index f46df6bb2..978a7eadd 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -665,7 +665,7 @@ def process_sync_committee_updates(state: BeaconState) -> None: ## Initialize state for Altair testnets -This helper function is only for initializing the pure Altair testnets and tests, where we use the dummy where the `ALTAIR_FORK_SLOT` GENESIS_SLOT`. +This helper function is only for initializing the pure Altair testnets and tests, where we set `ALTAIR_FORK_SLOT = GENESIS_SLOT`. *Note*: The function `initialize_beacon_state_from_eth1` is modified with `ALTAIR_FORK_VERSION` fork version and initial sync committees. @@ -704,7 +704,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, # Set genesis validators root for domain separation and chain versioning state.genesis_validators_root = hash_tree_root(state.validators) - # Fill in sync committees + # [New in Altair] Fill in sync committees state.current_sync_committee = get_sync_committee(state, get_current_epoch(state)) state.next_sync_committee = get_sync_committee(state, get_current_epoch(state) + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) From 666f847354c4e513960290d8ad2ff4dcfdc43de8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 20 Apr 2021 23:37:53 +0800 Subject: [PATCH 6/9] Update specs and test format note. --- specs/altair/beacon-chain.md | 6 +++--- specs/altair/fork.md | 2 ++ tests/formats/README.md | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 978a7eadd..7f6556556 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -663,11 +663,11 @@ def process_sync_committee_updates(state: BeaconState) -> None: state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` -## Initialize state for Altair testnets +## Initialize state for pure Altair testnets and test vectors -This helper function is only for initializing the pure Altair testnets and tests, where we set `ALTAIR_FORK_SLOT = GENESIS_SLOT`. +This helper function is only for initializing the state for pure Altair testnets and tests. -*Note*: The function `initialize_beacon_state_from_eth1` is modified with `ALTAIR_FORK_VERSION` fork version and initial sync committees. +*Note*: The function `initialize_beacon_state_from_eth1` is modified with `ALTAIR_FORK_VERSION` fork version and initial sync committees. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, diff --git a/specs/altair/fork.md b/specs/altair/fork.md index b51466c1e..6312eb09f 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -34,6 +34,8 @@ Warning: this configuration is not definitive. TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `ALTAIR_FORK_SLOT`, where `ALTAIR_FORK_SLOT % SLOTS_PER_EPOCH == 0`. +Note that for the pure Altair testnets, we don't apply `upgrade_to_altair` since it starts with Altair version logic. + ### Upgrading the state After `process_slots` of Phase 0 finishes, if `state.slot == ALTAIR_FORK_SLOT`, an irregular state change is made to upgrade to Altair. diff --git a/tests/formats/README.md b/tests/formats/README.md index 8f817220e..16f756fc1 100644 --- a/tests/formats/README.md +++ b/tests/formats/README.md @@ -139,6 +139,8 @@ E.g. `pre.ssz_snappy`, `deposit.ssz_snappy`, `post.ssz_snappy`. Diffing a `pre.ssz_snappy` and `post.ssz_snappy` provides all the information for testing, when decompressed and decoded. Then the difference between pre and post can be compared to anything that changes the pre state, e.g. `deposit.ssz_snappy` +Note that by default, the SSZ data is in the given test case's version, e.g., if it's `altair` test case, use `altair.BeaconState` container to deserialize the given state. + YAML is generally used for test metadata, and for tests that do not use SSZ: e.g. shuffling and BLS tests. In this case, there is no point in adding special SSZ types. And the size and efficiency of YAML is acceptable. From baf4b73c1804763c926e18489027c0726431d5a5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 20 Apr 2021 23:38:48 +0800 Subject: [PATCH 7/9] Fix ToC --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 7f6556556..a2138103d 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -52,7 +52,7 @@ - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) - [Sync committee updates](#sync-committee-updates) -- [Initialize state for Altair testnets](#initialize-state-for-altair-testnets) +- [Initialize state for pure Altair testnets and test vectors](#initialize-state-for-pure-altair-testnets-and-test-vectors) From 93f6a541adf8116cb86f072fcc7ab5620e7ced52 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 21 Apr 2021 16:42:54 +0800 Subject: [PATCH 8/9] PR feedback from @djrtwo --- specs/altair/beacon-chain.md | 2 +- specs/altair/fork.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index a2138103d..438c892c6 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -667,7 +667,7 @@ def process_sync_committee_updates(state: BeaconState) -> None: This helper function is only for initializing the state for pure Altair testnets and tests. -*Note*: The function `initialize_beacon_state_from_eth1` is modified with `ALTAIR_FORK_VERSION` fork version and initial sync committees. +*Note*: The function `initialize_beacon_state_from_eth1` is modified: (1) using `ALTAIR_FORK_VERSION` as the current fork version, (2) utilizing the Altair `BeaconBlockBody` when constructing the initial `latest_block_header`, and (3) adding initial sync committees. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, diff --git a/specs/altair/fork.md b/specs/altair/fork.md index b5dac5b22..be06e183e 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -34,7 +34,7 @@ Warning: this configuration is not definitive. TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at epoch `ALTAIR_FORK_EPOCH`. -Note that for the pure Altair testnets, we don't apply `upgrade_to_altair` since it starts with Altair version logic. +Note that for the pure Altair networks, we don't apply `upgrade_to_altair` since it starts with Altair version logic. ### Upgrading the state From ae429a38a0dc4dc060f53c4aad41146f17056c07 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 27 Apr 2021 12:11:15 -0600 Subject: [PATCH 9/9] add Modified comment --- specs/altair/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 438c892c6..f498c40c5 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -675,7 +675,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, deposits: Sequence[Deposit]) -> BeaconState: fork = Fork( previous_version=GENESIS_FORK_VERSION, - current_version=ALTAIR_FORK_VERSION, + current_version=ALTAIR_FORK_VERSION, # [Modified in Altair] epoch=GENESIS_EPOCH, ) state = BeaconState(