From f12c32a6904417a4b8ed4b8bbbae2f6193c0f835 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:18:28 -0600 Subject: [PATCH 1/9] Xdist tests for parallelism (#1201) * add xdist parallelization * reduce circleci cpus to 8 * reduce cpus to 4 * reduce circleci cpus to 2 * circleci back to 4 cpus --- Makefile | 4 ++-- test_libs/pyspec/requirements-testing.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9764b3528..9090c3ef0 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,11 @@ install_test: test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ - python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index ad9c4de3b..b5229ae20 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -1,6 +1,7 @@ -r requirements.txt -pytest>=3.6,<3.7 +pytest>=4.4 ../config_helpers flake8==3.7.7 mypy==0.701 pytest-cov +pytest-xdist From f55499c0baad5d22a6f7749a9153374c7edf9267 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 14:30:22 -0600 Subject: [PATCH 2/9] Fix Gwei value table (#1203) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef9..5081dbf74 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -200,7 +200,7 @@ These configurations are updated for releases, but may be out of sync during `de ### Gwei values | Name | Value | -| - | - | :-: | +| - | - | | `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | | `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | | `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | From e15a649f37680ab086801f02b408e1b562381d6f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 15:34:36 -0600 Subject: [PATCH 3/9] reduce MAX_EPOCHS_PER_CROSSLINK in minimal config for testing purposes --- configs/constant_presets/minimal.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index c8c2853c7..201ac475f 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -64,8 +64,8 @@ SLOTS_PER_HISTORICAL_ROOT: 64 MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # 2**11 (= 2,048) epochs PERSISTENT_COMMITTEE_PERIOD: 2048 -# 2**6 (= 64) epochs -MAX_EPOCHS_PER_CROSSLINK: 64 +# [customized] fast catchup crosslinks +MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 # [customized] 2**12 (= 4,096) epochs From 99df7da94a026b1daf02cadb10cbb18ee091b71d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 22:11:58 -0600 Subject: [PATCH 4/9] Add `&&` between the commands --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9764b3528..9c7bfcf59 100644 --- a/Makefile +++ b/Makefile @@ -59,10 +59,9 @@ open_cov: lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \ - cd ./eth2spec; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0; \ - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1 + flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \ + && cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \ + && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1; install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt From 183fa3c7762acfc2f68a295db58b1b9bb749581e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 21 Jun 2019 23:00:30 -0600 Subject: [PATCH 5/9] Make linter happy --- specs/core/0_beacon-chain.md | 11 ++++++++--- test_libs/pyspec/eth2spec/test/test_fork_choice.py | 2 -- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef9..d478c2941 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -524,9 +524,11 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients + # Digests of the active registry, for light clients + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators + # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] @@ -1515,7 +1517,10 @@ def process_slashings(state: BeaconState) -> None: total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: + if ( + validator.slashed and + current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 + ): penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 4bc7e8b0a..4706f0eaf 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,5 +1,3 @@ -from typing import List - from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.test.context import with_all_phases, with_state, bls_switch From 613380bff452f829de9342cb66abf01e945792cf Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 07:29:25 +0200 Subject: [PATCH 6/9] Shorter in-line comments --- specs/core/0_beacon-chain.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d478c2941..7b5d2a42a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -524,11 +524,9 @@ class BeaconState(Container): # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] - # Digests of the active registry, for light clients - active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] + active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Active registry digests for light clients # Slashings - # Sums of the effective balances of slashed validators - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] From 1e1a3e5311fd54a15e1449340285624bc316a7e8 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sat, 22 Jun 2019 07:34:02 +0200 Subject: [PATCH 7/9] Avoid substraction in comparison --- specs/core/0_beacon-chain.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b5d2a42a..894b96a19 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1506,19 +1506,16 @@ def process_registry_updates(state: BeaconState) -> None: ```python def process_slashings(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) + epoch = get_current_epoch(state) total_balance = get_total_active_balance(state) # Compute slashed balances in the current epoch - total_at_start = state.slashed_balances[(current_epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] - total_at_end = state.slashed_balances[current_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_at_start = state.slashed_balances[(epoch + 1) % EPOCHS_PER_SLASHED_BALANCES_VECTOR] + total_at_end = state.slashed_balances[epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if ( - validator.slashed and - current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 - ): + if validator.slashed and epoch + EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2 == validator.withdrawable_epoch: penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT From b007d5aa92f183be7eb8001db09d205767b2838b Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 22 Jun 2019 17:38:30 +0200 Subject: [PATCH 8/9] Add note on default zero-values, and be explicit with state-root (#1208) * Add note on default zero-values, and be explicit with state-root --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a955bcab8..094c04a11 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -180,9 +180,7 @@ The following values are (non-configurable) constants used throughout the specif ## Configuration -*Note*: The default mainnet configuration values are included here for spec-design purposes. -The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. -These configurations are updated for releases, but may be out of sync during `dev` changes. +*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases and may be out of sync during `dev` changes. ### Misc @@ -292,6 +290,8 @@ The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containe *Note*: The definitions are ordered topologically to facilitate execution of the spec. +*Note*: Fields missing in container instantiations default to their zero value. + ### Misc dependencies #### `Fork` @@ -1585,6 +1585,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, + state_root=ZERO_HASH, # Overwritten in next `process_slot` call body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed From 82167ff0a33a3476df5901bd6f0645cd1cdf852a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 24 Jun 2019 15:40:37 -0700 Subject: [PATCH 9/9] Clarify comment on phase 0 transfers The comment seems to imply conjunction over each case, but the code implies disjunction; at the very least it is ambiguous. This PR makes the comment less ambiguous. --- specs/core/0_beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 094c04a11..103c436b5 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1807,10 +1807,13 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert state.balances[transfer.sender] >= max(transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot - # Sender must be not yet eligible for activation, withdrawn, or transfer balance over MAX_EFFECTIVE_BALANCE + # Sender must satisfy at least one of the following conditions in the parenthesis: assert ( + # * Has not been activated state.validators[transfer.sender].activation_eligibility_epoch == FAR_FUTURE_EPOCH or + # * Is withdrawable get_current_epoch(state) >= state.validators[transfer.sender].withdrawable_epoch or + # * Balance after transfer is more than the effective balance threshold transfer.amount + transfer.fee + MAX_EFFECTIVE_BALANCE <= state.balances[transfer.sender] ) # Verify that the pubkey is valid