From b817b7690ced79d06da521673befb4ede4f83be9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 12:13:47 +0800 Subject: [PATCH 001/140] Disable the deposit contract tests. Only keep the `build_deposit_contract` to verify the bytecode. --- .circleci/config.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e00463cc5..bd46e8542 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -216,15 +216,17 @@ workflows: - lint: requires: - test - - install_deposit_contract_web3_tester: - requires: - - checkout_specs - - test_deposit_contract_web3_tests: - requires: - - install_deposit_contract_web3_tester + # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. + # - install_deposit_contract_web3_tester: + # requires: + # - checkout_specs + # - test_deposit_contract_web3_tests: + # requires: + # - install_deposit_contract_web3_tester build_and_test_deposit_contract: jobs: - build_deposit_contract - - test_deposit_contract: - requires: - - build_deposit_contract + # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. + # - test_deposit_contract: + # requires: + # - build_deposit_contract From 56048cbf5637d5834ad4ab570888e7928ceba1a4 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 6 Dec 2020 20:56:06 +0000 Subject: [PATCH 002/140] Cosmetic cleanups to BLS signatures section Miscellaneous cosmetic cleanups to the " BLS signatures" section: 1) fix section capitalisation for spec consistency 2) avoid uncapitalised "eth2" for spec consistency 3) avoid contractions (such as `PK` for `pubkey`) for spec consistency 4) various copyedits --- specs/phase0/beacon-chain.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index d336cb2f8..857b6de94 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -57,7 +57,7 @@ - [Crypto](#crypto) - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - - [BLS Signatures](#bls-signatures) + - [BLS signatures](#bls-signatures) - [Predicates](#predicates) - [`is_active_validator`](#is_active_validator) - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue) @@ -600,17 +600,17 @@ def bytes_to_uint64(data: bytes) -> uint64: `def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization). -#### BLS Signatures +#### BLS signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-04](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04). Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: +The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` define the following functions: -- `def Sign(SK: int, message: Bytes) -> BLSSignature` -- `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` +- `def Sign(secret_key: int, message: Bytes) -> BLSSignature` +- `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` -- `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(PKs: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` +- `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` +- `def AggregateVerify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` -Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. +The above functions are accessed through the `bls` module, e.g. `bls.Verify`. ### Predicates From 0b9e80d1eb0fe1609792fbd366b32b89837a94a6 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 6 Dec 2020 21:04:11 +0000 Subject: [PATCH 003/140] grammar --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 857b6de94..34ce3c9c5 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -602,7 +602,7 @@ def bytes_to_uint64(data: bytes) -> uint64: #### BLS signatures -The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` define the following functions: +The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: - `def Sign(secret_key: int, message: Bytes) -> BLSSignature` - `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` From f2fe3906388bbc049616cd0f25755d73356fc36e Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 8 Dec 2020 09:20:50 +0000 Subject: [PATCH 004/140] secret_key => privkey --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 34ce3c9c5..b47558484 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -604,7 +604,7 @@ def bytes_to_uint64(data: bytes) -> uint64: The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: -- `def Sign(secret_key: int, message: Bytes) -> BLSSignature` +- `def Sign(privkey: int, message: Bytes) -> BLSSignature` - `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` From 6b728c18800cc0611d8630e35e15bba4003f6784 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 11 Dec 2020 14:46:16 -0700 Subject: [PATCH 005/140] add eth1 withdrawal credentials to spec --- specs/phase0/beacon-chain.md | 5 +++ specs/phase0/deposit-contract.md | 13 ++++-- specs/phase0/validator.md | 42 +++++++++++++++--- .../block_processing/test_process_deposit.py | 43 +++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b47558484..7fdf027bc 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,6 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) + - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -209,7 +210,11 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | + +### Validator withdrawal credential versions + | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | +| `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | ### Time parameters diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 23d831917..ef21d35e0 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,12 +58,17 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: +One of the `DepositData` fields is `withdrawal_credentials`. -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey +This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). +The first byte of `withdrawal_credentials` is a version number. The remaining +bytes are content specific to the version. -The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. +Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` +versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). + +*Note*: The deposit contract does *not* validate withdrawal credentials. +Thus, new versions can be added without modifications of the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index cb45e65e8..4fafcbad4 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -15,7 +15,9 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - [BLS public key](#bls-public-key) - - [BLS withdrawal key](#bls-withdrawal-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [BLS key credentials](#bls-key-credentials) + - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) @@ -100,14 +102,42 @@ A validator must initialize many parameters locally before submitting a deposit Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BLS12-381 curve](https://z.cash/blog/new-snark-curve). A private key, `privkey`, must be securely generated along with the resultant `pubkey`. This `privkey` must be "hot", that is, constantly available to sign data throughout the lifetime of the validator. -#### BLS withdrawal key +#### Withdrawal credentials -A secondary withdrawal private key, `withdrawal_privkey`, must also be securely generated along with the resultant `withdrawal_pubkey`. This `withdrawal_privkey` does not have to be available for signing during the normal lifetime of a validator and can live in "cold storage". +The `withdrawal_credentials` ultimately control the deposited ETH once the +validator has exited and is withdrawable. The 0th byte of this 32-byte field, +called the "withdrawal prefix" defines the withdrawal credential version +while the remaining 31 bytes define the version-specific content. -The validator constructs their `withdrawal_credentials` via the following: +The following withdrawal credentials versions are currently supported. The +validator must choose and construct a version for the `withdrawal_credentials` field. -* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. +##### BLS key credentials + +BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. +This key is securely generated along with the resultant `bls_withdrawal_pubkey`. +This `withdrawal_privkey` does not have to be available for signing during +the normal lifetime of a validator and can live in "cold storage". + +The validator constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. + +##### Eth1 address credentials + +Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. + +The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) +triggered by an eth1 transaction that will handle gas price/limit and payment of fees. + +As long as such a withdrawal account/contract can receive ETH transfers, +the future withdrawal protocol is agnostic to all other implementation details. + +The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. + +*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. ### Submit deposit diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index b7a0de6c8..36e76f46c 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -94,6 +94,49 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) +@with_all_phases +@spec_state_test +def test_new_deposit_eth1_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + + b'\x00' * 11 # specified 0s + + b'\x59' * 20 # a 20-byte eth1 address + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_non_versioned_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + b'\xFF' # Non specified withdrawal credentials version + + b'\x02' * 31 # Garabage bytes + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + @with_all_phases @spec_state_test @always_bls From 784e567d4b3e2dc9bdccb85d792c3d1696ac83a1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:38:40 +0000 Subject: [PATCH 006/140] Fix table for withdrawal credentials prefixes Minor cosmetic fixes (misformated table, section title). --- specs/phase0/beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 7fdf027bc..8df10c80a 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) + - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -211,8 +211,10 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Validator withdrawal credential versions +### Withdrawal credentials prefixes +| Name | Value | +| - | - | | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | | `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | From ef3802b8e7dbe40880fa29ee0977193b7bbc6982 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:40:59 +0000 Subject: [PATCH 007/140] Cleaner section title "Withdrawal prefixes" matches `[BLS]/[ETH1_ADDRESS]_WITHDRAWAL_PREFIX` --- specs/phase0/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8df10c80a..c0e026e97 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) + - [Withdrawal prefixes](#withdrawal-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -211,7 +211,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Withdrawal credentials prefixes +### Withdrawal prefixes | Name | Value | | - | - | From 2cc9a12c982089439adc1a29b991cc805e5c29fc Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:50:10 +0000 Subject: [PATCH 008/140] Copy-edit deposit-contract.md --- specs/phase0/deposit-contract.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index ef21d35e0..2228aed90 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -60,15 +60,12 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac One of the `DepositData` fields is `withdrawal_credentials`. -This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). -The first byte of `withdrawal_credentials` is a version number. The remaining -bytes are content specific to the version. +This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. +The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. -Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). -*Note*: The deposit contract does *not* validate withdrawal credentials. -Thus, new versions can be added without modifications of the deposit contract. +*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. #### `DepositEvent` log From ae0b10edb7143b0f285b33cf72548f083d0534f8 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:46:39 +0000 Subject: [PATCH 009/140] Update validator.md --- specs/phase0/validator.md | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4fafcbad4..d68c4b4af 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,40 +104,28 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` ultimately control the deposited ETH once the -validator has exited and is withdrawable. The 0th byte of this 32-byte field, -called the "withdrawal prefix" defines the withdrawal credential version -while the remaining 31 bytes define the version-specific content. +The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. -The following withdrawal credentials versions are currently supported. The -validator must choose and construct a version for the `withdrawal_credentials` field. +##### `BLS_WITHDRAWAL_PREFIX` -##### BLS key credentials +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: -BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. -This key is securely generated along with the resultant `bls_withdrawal_pubkey`. -This `withdrawal_privkey` does not have to be available for signing during -the normal lifetime of a validator and can live in "cold storage". +* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -The validator constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. +*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. -##### Eth1 address credentials +##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: -The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) -triggered by an eth1 transaction that will handle gas price/limit and payment of fees. +* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == Bytes32()[1:12]` +* `withdrawal_credentials[12:] = eth1_withdrawal_address` -As long as such a withdrawal account/contract can receive ETH transfers, -the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. -The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. - -*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. +**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. ### Submit deposit From b085436f2f4db99d8b7bf226e07bab11642aec60 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:51:21 +0000 Subject: [PATCH 010/140] Update validator.md --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index d68c4b4af..8fb13ead1 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -14,8 +14,8 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [BLS key credentials](#bls-key-credentials) - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) @@ -104,7 +104,7 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` @@ -113,7 +113,7 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. +*Note*: The `bls_withdrawal_privkey` is not required for validating and can be kept in cold storage. ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` @@ -123,7 +123,7 @@ Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte * `withdrawal_credentials[1:12] == Bytes32()[1:12]` * `withdrawal_credentials[12:] = eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. **TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. From f9edecf23c5c18bc0f1871581275d0da4aaaf073 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:56:30 +0000 Subject: [PATCH 011/140] Update validator.md --- specs/phase0/validator.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 8fb13ead1..fb40dd94f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,11 +104,11 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -117,11 +117,11 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: -* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:12] == Bytes32()[1:12]` -* `withdrawal_credentials[12:] = eth1_withdrawal_address` +* `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == b'\x00' * 11` +* `withdrawal_credentials[12:] == eth1_withdrawal_address` Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 85c1347ffc5f00d44829538d62626bedf0f0edc1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:57:58 +0000 Subject: [PATCH 012/140] Update validator.md --- specs/phase0/validator.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index fb40dd94f..3f60fc386 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -14,10 +14,10 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) - - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - - [BLS key credentials](#bls-key-credentials) - - [Eth1 address credentials](#eth1-address-credentials) + - [BLS public key](#bls-public-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) From 645a3851af222a229936d6c93b9a4e7ce5839afb Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 22:02:56 +0000 Subject: [PATCH 013/140] Update deposit-contract.md --- specs/phase0/deposit-contract.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 2228aed90..5c35da8d4 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,14 +58,9 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). -This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. -The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. - -The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). - -*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log From 2c90ffa2f499569fdaf7fea6af1a0e30ff14ae59 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 14 Dec 2020 13:09:25 -0700 Subject: [PATCH 014/140] copy edits --- specs/phase0/deposit-contract.md | 8 ++++++-- specs/phase0/validator.md | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 5c35da8d4..02e762dae 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -58,9 +58,13 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +Read more in the [validator guide](./validator.md#withdrawal-credentials). -*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. +Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 3f60fc386..5106c2b73 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -104,11 +104,16 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. + +The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair +`(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -117,15 +122,19 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify +a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. +The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. - -**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers +the future withdrawal protocol is agnostic to all other implementation details. ### Submit deposit From e63c96416a16109a7984603cb46dbef506c4f1a9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 16 Dec 2020 15:10:54 +0800 Subject: [PATCH 015/140] Add a FIXME comment. --- specs/lightclient/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 4eb1f24ab..eabcb3a86 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -240,6 +240,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S ```python def process_final_updates(state: BeaconState) -> None: + # FIXME: unfold the full `process_final_updates` to avoid side effects. phase0.process_final_updates(state) next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: From cc9a4cdc46131b6d320eb4ac4bbd663db16bf917 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 16 Dec 2020 17:12:51 -0700 Subject: [PATCH 016/140] add base sanity light client tests --- specs/lightclient/beacon-chain.md | 2 +- specs/lightclient/lightclient-fork.md | 6 +- .../lightclient_patch/sanity/test_blocks.py | 102 ++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index eabcb3a86..3b03ad853 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -163,7 +163,7 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] - return SyncCommittee(pubkeys, aggregates) + return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` ### Block processing diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index bb67fa54a..568a9793b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -75,9 +75,9 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, - # Light-client - current_sync_committee=SyncCommittee(), - next_sync_committee=SyncCommittee(), ) + # Fill in sync committees + post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) + post.next_sync_committee = get_sync_committee(post, get_current_epoch(post) + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) return post ``` diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py new file mode 100644 index 000000000..df8e2545c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -0,0 +1,102 @@ +import random +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.utils import bls +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, + next_epoch, +) +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.context import ( + PHASE0, PHASE1, + with_all_phases_except, + spec_state_test, +) + + +def compute_light_client_signature(spec, state, slot, privkey): + domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) + signing_root = spec.compute_signing_root(block_root, domain) + return bls.Sign(privkey, signing_root) + + +def compute_aggregate_light_client_signature(spec, state, slot, participants): + if len(participants) == 0: + return spec.G2_POINT_AT_INFINITY + + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + compute_light_client_signature( + spec, + state, + slot, + privkey, + ) + ) + return bls.Aggregate(signatures) + + +def run_light_client_sanity_test(spec, state, fraction_full=1.0): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + participants = random.sample(committee, int(len(committee) * fraction_full)) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [index in participants for index in committee] + block.body.sync_committee_signature = compute_aggregate_light_client_signature( + spec, + state, + block.slot - 1, + participants, + ) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_full_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_half_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_empty_light_client_committee(spec, state): + next_epoch(spec, state) + yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_full_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_half_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_empty_light_client_committee_genesis(spec, state): + yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) From 89c5ca6bcd2fa9a1f57d342038bb08effa969fe7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 17 Dec 2020 06:25:58 -0700 Subject: [PATCH 017/140] 'light_client' -> 'sync_committee' --- .../lightclient_patch/sanity/test_blocks.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index df8e2545c..4fbdfc371 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -1,5 +1,5 @@ import random -from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.test.helpers.state import ( state_transition_and_sign_block, @@ -15,7 +15,7 @@ from eth2spec.test.context import ( ) -def compute_light_client_signature(spec, state, slot, privkey): +def compute_sync_committee_signature(spec, state, slot, privkey): domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) if slot == state.slot: block_root = build_empty_block_for_next_slot(spec, state).parent_root @@ -25,7 +25,7 @@ def compute_light_client_signature(spec, state, slot, privkey): return bls.Sign(privkey, signing_root) -def compute_aggregate_light_client_signature(spec, state, slot, participants): +def compute_aggregate_sync_committee_signature(spec, state, slot, participants): if len(participants) == 0: return spec.G2_POINT_AT_INFINITY @@ -33,7 +33,7 @@ def compute_aggregate_light_client_signature(spec, state, slot, participants): for validator_index in participants: privkey = privkeys[validator_index] signatures.append( - compute_light_client_signature( + compute_sync_committee_signature( spec, state, slot, @@ -43,7 +43,7 @@ def compute_aggregate_light_client_signature(spec, state, slot, participants): return bls.Aggregate(signatures) -def run_light_client_sanity_test(spec, state, fraction_full=1.0): +def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) participants = random.sample(committee, int(len(committee) * fraction_full)) @@ -51,7 +51,7 @@ def run_light_client_sanity_test(spec, state, fraction_full=1.0): block = build_empty_block_for_next_slot(spec, state) block.body.sync_committee_bits = [index in participants for index in committee] - block.body.sync_committee_signature = compute_aggregate_light_client_signature( + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, @@ -65,38 +65,38 @@ def run_light_client_sanity_test(spec, state, fraction_full=1.0): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_full_light_client_committee(spec, state): +def test_full_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_half_light_client_committee(spec, state): +def test_half_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_empty_light_client_committee(spec, state): +def test_empty_sync_committee_committee(spec, state): next_epoch(spec, state) - yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_full_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=1.0) +def test_full_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=1.0) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_half_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=0.5) +def test_half_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.5) @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_empty_light_client_committee_genesis(spec, state): - yield from run_light_client_sanity_test(spec, state, fraction_full=0.0) +def test_empty_sync_committee_committee_genesis(spec, state): + yield from run_sync_committee_sanity_test(spec, state, fraction_full=0.0) From aa16da10994bb8ae645075c4f4a6cc19a3f824d4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 18 Dec 2020 19:03:05 +0800 Subject: [PATCH 018/140] Updated readme (#2157) --- README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a3ca7c586..d5eecd620 100644 --- a/README.md +++ b/README.md @@ -11,29 +11,28 @@ This repository hosts the current Eth2 specifications. Discussions about design [![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) - Core specifications for Eth2 clients be found in [specs](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 + * [The Beacon Chain](specs/phase0/beacon-chain.md) * [Beacon Chain Fork Choice](specs/phase0/fork-choice.md) * [Deposit Contract](specs/phase0/deposit-contract.md) * [Honest Validator](specs/phase0/validator.md) * [P2P Networking](specs/phase0/p2p-interface.md) -### Phase 1 -* [From Phase 0 to Phase 1](specs/phase1/phase1-fork.md) -* [The Beacon Chain for Shards](specs/phase1/beacon-chain.md) -* [Custody Game](specs/phase1/custody-game.md) -* [Shard Transition and Fraud Proofs](specs/phase1/shard-transition.md) -* [Light client syncing protocol](specs/phase1/light-client-sync.md) -* [Beacon Chain Fork Choice for Shards](specs/phase1/fork-choice.md) +### Light clients -### Phase 2 +* [Beacon chain changes](specs/lightclient/beacon-chain.md) +* [Light client sync protocol](specs/lightclient/sync-protocol.md) -Phase 2 is still actively in R&D and does not yet have any formal specifications. +### Sharding -See the [Eth2 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for current progress, discussions, and definitions regarding this work. +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146), and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). + +### Merge + +The merge is still actively in R&D; see an [ethresear.ch](https://ethresear.ch) post describing the proposed basic mechanism [here](https://ethresear.ch/t/the-eth1-eth2-transition/6265) and the section of [ethereum.org](https://ethereum.org) describing the merge at a high level [here](https://ethereum.org/en/eth2/docking/). ### Accompanying documents can be found in [specs](specs) and include: From 71c5eb5c2f3897651f7007de681fc6ab3ac96e14 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 18 Dec 2020 11:11:44 -0800 Subject: [PATCH 019/140] Update names in pyspec README Names for pip categories have been updated --- tests/core/pyspec/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index f7092dbce..d84054f11 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -27,7 +27,7 @@ python setup.py pyspec --spec-fork=phase0 --md-doc-paths="specs/phase0/beacon-ch After installing, you can install the optional dependencies for testing and linting. With makefile: `make install_test`. -Or manually: run `pip install .[testing]` and `pip install .[linting]`. +Or manually: run `pip install .[test]` and `pip install .[lint]`. These tests are not intended for client-consumption. These tests are testing the spec itself, to verify consistency and provide feedback on modifications of the spec. From 9e2fa305673353a7a659166c0e3cdbd9efe8a243 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 19 Dec 2020 07:22:11 +0800 Subject: [PATCH 020/140] Complete HF1 beacon-chain.md --- specs/lightclient/beacon-chain.md | 312 +++++++++++++++++++++++++++--- 1 file changed, 281 insertions(+), 31 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3b03ad853..11187e25b 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -1,4 +1,4 @@ -# Ethereum 2.0 Light Client Support +# Ethereum 2.0 HF1 ## Table of contents @@ -36,13 +36,39 @@ ## Introduction -This is a standalone beacon chain patch adding light client support via sync committees. +This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: + +* Light client support via sync committees +* Incentive accounting reforms, reducing spec complexity and reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Constants +### Participation flags + | Name | Value | -| - | - | -| `BASE_REWARDS_PER_EPOCH` | `uint64(5)` | +| - | - | +| `TIMELY_HEAD_FLAG` | `0` | +| `TIMELY_SOURCE_FLAG` | `1` | +| `TIMELY_TARGET_FLAG` | `2` | + +### Participation rewards + +| Name | Value | +| - | - | +| `TIMELY_HEAD_NUMERATOR` | `12` | +| `TIMELY_SOURCE_NUMERATOR` | `12` | +| `TIMELY_TARGET_NUMERATOR` | `32` | +| `REWARD_DENOMINATOR` | `64` | + +The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewards and other future micro-rewards. + +### Misc + +| Name | Value | +| - | - | +| `PARTICIPATION_FLAGS_LENGTH` | `8` | +| `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | ## Configuration @@ -90,10 +116,41 @@ class BeaconBlockBody(phase0.BeaconBlockBody): #### `BeaconState` ```python -class BeaconState(phase0.BeaconState): - # Sync committees +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Participation + previous_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee + # Denominator to real-time-updated balances (NOT effective balances!) + balance_denominator: uint64 ``` ### New containers @@ -110,6 +167,24 @@ class SyncCommittee(Container): ### `Predicates` +#### `get_base_reward` + +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to + +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH +``` + +##### `get_base_reward_per_eth` + +```python +def get_base_reward_per_eth(state: BeaconState) -> Gwei: + total_balance = get_total_active_balance(state) + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) + #### `eth2_fast_aggregate_verify` ```python @@ -166,6 +241,69 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` +#### `get_unslashed_participating_indices` + +```python +def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: Epoch) -> Set[ValidatorIndex]: + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) + if epoch == get_current_epoch(state): + epoch_participation = state.current_epoch_participation + else: + epoch_participation = state.previous_epoch_participation + participating_indices = [index in get_active_validator_indices(state, epoch) if epoch_participation[index][flag]] + return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) +``` + +#### `get_flag_deltas` + +```python +def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Gwei]: + """ + Computes the rewards and penalties associated with a particular duty, by scanning through the participation + flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. + """ + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow + unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment + active_increments = get_total_active_balance(state) // increment + for index in get_eligible_validator_indices(state): + base_reward = get_base_reward(state, index) + if index in unslashed_participating_indices: + if is_in_inactivity_leak(state): + # Optimal participatition is fully rewarded to cancel the inactivity penalty + rewards[index] = base_reward * numerator // REWARD_DENOMINATOR + else: + rewards[index] = ( + (base_reward * unslashed_participating_increments // active_increments + base_reward) + * numerator // REWARD_DENOMINATOR + ) + return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR +``` + +##### `get_inactivity_penalty_deltas` + +*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. + +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Compute the penalties associated with the inactivity leak, by scanning through the participation + flags to determine who participated and who did not, applying the leak penalty globally and applying + compensatory rewards to participants. + """ + rewards = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, get_previous_epoch(state)) + for index in get_eligible_validator_indices(state): + if index in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) +``` + ### Block processing ```python @@ -178,6 +316,52 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_sync_committee(state, block.body) ``` +#### New `process_attestation` + +*Note*: The function `process_attestation` is modified to do incentive accounting with epoch participation flags. + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + assert data.index < get_committee_count_per_slot(state, data.target.epoch) + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + if data.target.epoch == get_current_epoch(state): + epoch_participation = state.current_epoch_participation + justified_checkpoint = state.current_justified_checkpoint + else: + epoch_participation = state.previous_epoch_participation + justified_checkpoint = state.previous_justified_checkpoint + # Matching roots + is_matching_head = data.beacon_block_root == get_block_root_at_slot(state, data.slot) + is_matching_source = data.source == justified_checkpoint + is_matching_target = data.target.root == get_block_root(state, data.target.epoch) + assert is_matching_source + # Participation flags + participation_flags = [] + if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: + participation_flags.append(TIMELY_HEAD_FLAG) + if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): + participation_flags.append(TIMELY_SOURCE_FLAG) + if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: + participation_flags.append(TIMELY_TARGET_FLAG) + # Update epoch participation flags + proposer_reward_numerator = 0 + for index in get_attesting_indices(state, data, attestation.aggregation_bits): + for flag, numerator in FLAGS_AND_NUMERATORS: + if flag in participation_flags and not epoch_participation[index][flag]: + epoch_participation[index][flag] = True + proposer_reward_numerator += get_base_reward(state, index) * numerator + # Reward proposer + proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) + increase_balance(state, get_beacon_proposer_index(state), proposer_reward) + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) +``` + #### Sync committee processing ```python @@ -208,42 +392,108 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ### Epoch processing -#### Components of attestation deltas +#### New `process_justification_and_finalization` -*Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`. +*Note*: The function `process_justification_and_finalization` is modified with `matching_target_attestations` replaced by `matching_target_indices`. ```python -def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return inactivity reward/penalty deltas for each validator. - """ - penalties = [Gwei(0) for _ in range(len(state.validators))] - if is_in_inactivity_leak(state): - matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) - for index in get_eligible_validator_indices(state): - # Penalize validator so that optimal attestation performance is rewarded with one base reward per epoch - base_reward = get_base_reward(state, index) - penalties[index] += Gwei((BASE_REWARDS_PER_EPOCH - 1) * base_reward - get_proposer_reward(state, index)) - if index not in matching_target_attesting_indices: - effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) +def process_justification_and_finalization(state: BeaconState) -> None: + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, previous_epoch) + if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) + state.justification_bits[1] = 0b1 + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, current_epoch) + if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) + state.justification_bits[0] = 0b1 + # Process finalizations + bits = state.justification_bits + # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint +``` - # No rewards associated with inactivity penalties - rewards = [Gwei(0) for _ in range(len(state.validators))] - return rewards, penalties +#### New `process_rewards_and_penalties` + +*Note*: The function `process_rewards_and_penalties` is modified to use participation flag deltas. + +```python +def process_rewards_and_penalties(state: BeaconState) -> None: + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch + if get_current_epoch(state) == GENESIS_EPOCH: + return + flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in FLAGS_AND_NUMERATORS] + deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] + for (rewards, penalty) in deltas: + for index in range(len(state.validators)): + increase_balance(state, ValidatorIndex(index), rewards[index]) + # Bounds-friendly expansion for `denom *= (1 + penalty // GWEI_PER_ETH)` + state.balance_denominator = ( + state.balance_denominator + + penalty + + (state.balance_denominator - GWEI_PER_ETH) * penalty // GWEI_PER_ETH + ) ``` #### Final updates -*Note*: The function `process_final_updates` is modified to handle sync committee updates. +*Note*: The function `process_final_updates` is modified to handle sync committee updates, replacement of `PendingAttestation`s with participation flags, and a sliding-denominator-aware hysteresis mechanism. ```python def process_final_updates(state: BeaconState) -> None: - # FIXME: unfold the full `process_final_updates` to avoid side effects. - phase0.process_final_updates(state) - next_epoch = get_current_epoch(state) + Epoch(1) + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) + # Reset eth1 data votes + if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] + # Update sync committees if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) -``` + # Update effective balances with hysteresis + for index, validator in enumerate(state.validators): + balance_increments = state.balances[index] * HYSTERESIS_QUOTIENT // state.balance_denominator + effective_increments = validator.effective_balance // GWEI_PER_ETH * HYSTERESIS_QUOTIENT + if ( + balance_increments + HYSTERESIS_DOWNWARD_MULTIPLIER < effective_increments + or effective_increments + HYSTERESIS_UPWARD_MULTIPLIER < balance_increments + or get_current_epoch(state) == validator.withdrawable_epoch + ): + validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH + if state.balance_denominator >= 2 * GWEI_PER_ETH: + state.balances = [x//2 for x in state.balances] + state.balance_denominator //= 2 + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) + # Set randao mix + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) + # Set historical root accumulator + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) + state.historical_roots.append(hash_tree_root(historical_batch)) + # Rotate current/previous epoch participation flags + state.previous_epoch_participation = state.current_epoch_participation + state.current_epoch_participation = [] From 682f6c02c7a1ede584127ca60e4fdd65549f8c0b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 20 Dec 2020 18:54:57 +0800 Subject: [PATCH 021/140] Update specs/lightclient/beacon-chain.md --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 11187e25b..c4224fbaa 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -169,7 +169,7 @@ class SyncCommittee(Container): #### `get_base_reward` -*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to allow penalties to be computed to the denominator. ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: From 452e3301bed9211199329ef45ceee9bf080a68a9 Mon Sep 17 00:00:00 2001 From: Ben Edgington Date: Mon, 21 Dec 2020 10:22:04 +0000 Subject: [PATCH 022/140] Fix typo --- specs/phase0/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b47558484..08ddc853a 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -250,7 +250,7 @@ The following values are (non-configurable) constants used throughout the specif - The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**13` epochs (about 36 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. Note this value will be upgraded to `2**24` after Phase 0 mainnet stabilizes to provide a faster recovery in the event of an inactivity leak. -- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accoutable safety margin. +- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accountable safety margin. ### Max operations per block From edfd04c21287f162fab5848a24ef0cc57c8e727c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 22 Dec 2020 10:42:59 -0800 Subject: [PATCH 023/140] Refactor sync committee rewards to use helper This change is functionally equivalent but uses the helper we already have for proposer rewards. The argument for this change is better encapsulation of the reward which makes it easier in general to reason about properties of the spec ("are the attestation proposer rewards and the sync committee proposer rewards equivalent?") and a single point of maintenance in the event that rewards get refactored in the future (which makes refactoring safer overall). --- specs/lightclient/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3b03ad853..14a6cc850 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -197,10 +197,11 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: base_reward = get_base_reward(state, participant_index) - max_participant_reward = base_reward - base_reward // PROPOSER_REWARD_QUOTIENT + proposer_reward = get_proposer_reward(state, participant_index) + max_participant_reward = base_reward - proposer_reward reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) increase_balance(state, participant_index, reward) - proposer_reward += base_reward // PROPOSER_REWARD_QUOTIENT + proposer_reward += proposer_reward # Reward beacon proposer increase_balance(state, get_beacon_proposer_index(state), proposer_reward) From cc80dd758cf5120b11af3747f5826de5cdcbd87c Mon Sep 17 00:00:00 2001 From: multisignature <72713080+multisignature@users.noreply.github.com> Date: Sat, 26 Dec 2020 06:53:42 +0000 Subject: [PATCH 024/140] Update README.md (#2164) * Update README.md I've removed an incorrectly placed comma. Since the comma is placed before 'and', and the conjunction isn't being used to either a) denote the last element of a list with more than two elements or b) separate two independent clauses, it's not necessary here. * Update validator.md Fixed a couple of minor errors. --- README.md | 2 +- specs/phase1/validator.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d5eecd620..f79c2b858 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi ### Sharding -The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146), and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). ### Merge diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index c5893fbc6..ccc877be0 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -67,7 +67,7 @@ A validator is an entity that participates in the consensus of the Ethereum 2.0 This document is an extension of the [Phase 0 -- Validator](../phase0/validator.md). All behaviors and definitions defined in the Phase 0 doc carry over unless explicitly noted or overridden. -All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use as a reference throughout. +All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use them as a reference throughout. ## Constants @@ -352,7 +352,7 @@ Aggregation selection and the core of this duty are largely unchanged from Phase Note the timing of when to broadcast aggregates is altered in Phase 1+. -If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot`-that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot` -- that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. ##### `AggregateAndProof` From 6a8cb48f490664adcd41fead613aaf90c4f6dbb7 Mon Sep 17 00:00:00 2001 From: Saulius Grigaitis Date: Sun, 27 Dec 2020 00:18:22 +0200 Subject: [PATCH 025/140] Updated Sharding technical details link in readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f79c2b858..937841127 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi ### Sharding -The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/r1XzqYIOv). +The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD). ### Merge From 7d60e482b90c65ac6abd4c316f4dc4b7f27781e0 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Tue, 5 Jan 2021 08:48:25 +0300 Subject: [PATCH 026/140] fix typo --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index b4d78cb12..4bb950540 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -42,7 +42,7 @@ This document uses data structures, constants, functions, and terminology from ## Weak Subjectivity Checkpoint -Any `Checkpoint` can used be a Weak Subjectivity Checkpoint. +Any `Checkpoint` can be used as a Weak Subjectivity Checkpoint. These Weak Subjectivity Checkpoints are distributed by providers, downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. From 844c879f1c7adfb2ae04135fae50ee86e2aa2423 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 21:00:52 +0800 Subject: [PATCH 027/140] Bump `py_ecc` and `milagro_bls_binding` (#2169) * Bump py_ecc to v5.1.0 and milagro_bls_binding to v1.6.2 * python3.8 -> python3 for py39 compatibility * fix * Try python:3.9 * Revert: using Python3.8 in CI now --- .circleci/config.yml | 4 ++-- Makefile | 12 ++++++------ setup.py | 4 ++-- tests/generators/bls/requirements.txt | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd46e8542..bb1df9909 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v22-pyspec + venv_name: v24-pyspec reqs_checksum: cache-{{ checksum "setup.py" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v22-pyspec + venv_name: v24-pyspec reqs_checksum: cache-{{ checksum "setup.py" }} venv_path: ./venv restore_deposit_contract_tester_cached_venv: diff --git a/Makefile b/Makefile index 987650948..08e822bf6 100644 --- a/Makefile +++ b/Makefile @@ -82,19 +82,19 @@ pyspec: # installs the packages to run pyspec tests install_test: - python3.8 -m venv venv; . venv/bin/activate; pip3 install .[lint]; pip3 install -e .[test] + python3 -m venv venv; . venv/bin/activate; python3 -m pip install .[lint]; python3 -m pip install -e .[test] test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec -cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.lightclient_patch.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec + python3 -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & @@ -133,11 +133,11 @@ test_deposit_contract: dapp test -v --fuzz-runs 5 install_deposit_contract_web3_tester: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt + cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; python3 -m pip install -r requirements.txt test_deposit_contract_web3_tests: cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \ - python -m pytest . + python3 -m pytest . # Runs a generator, identified by param 1 define run_generator diff --git a/setup.py b/setup.py index bd043dccc..6b8520075 100644 --- a/setup.py +++ b/setup.py @@ -582,8 +582,8 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==5.0.0", - "milagro_bls_binding==1.5.0", + "py_ecc==5.1.0", + "milagro_bls_binding==1.6.2", "dataclasses==0.6", "remerkleable==0.1.18", "ruamel.yaml==0.16.5", diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index fd54b5b32..5f830773a 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==5.0.0 +py_ecc==5.1.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ From 5cf05148167c38dd135f12de7d0bbc8feb478cb9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 21:56:32 +0800 Subject: [PATCH 028/140] Fix ToC and minor linter issues --- specs/lightclient/beacon-chain.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index c4224fbaa..29678ef45 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -8,9 +8,12 @@ - [Introduction](#introduction) - [Constants](#constants) + - [Participation flags](#participation-flags) + - [Participation rewards](#participation-rewards) + - [Misc](#misc) - [Configuration](#configuration) - [Constants](#constants-1) - - [Misc](#misc) + - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) - [Containers](#containers) @@ -21,14 +24,21 @@ - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [`Predicates`](#predicates) + - [`get_base_reward`](#get_base_reward) + - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) + - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) + - [`get_flag_deltas`](#get_flag_deltas) + - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) - [Block processing](#block-processing) + - [New `process_attestation`](#new-process_attestation) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - - [Components of attestation deltas](#components-of-attestation-deltas) + - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) + - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) - [Final updates](#final-updates) @@ -77,6 +87,7 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | +| `GWEI_PER_ETH` | `10**9` | ### Misc @@ -184,6 +195,7 @@ def get_base_reward_per_eth(state: BeaconState) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validators[index].effective_balance return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) +``` #### `eth2_fast_aggregate_verify` @@ -250,7 +262,8 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [index in get_active_validator_indices(state, epoch) if epoch_participation[index][flag]] + participating_indices = [index for index in get_active_validator_indices(state, epoch) + if epoch_participation[index][flag]] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -263,7 +276,6 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) - penalties = [Gwei(0)] * len(state.validators) unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment @@ -279,7 +291,7 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple (base_reward * unslashed_participating_increments // active_increments + base_reward) * numerator // REWARD_DENOMINATOR ) - return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR + return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR ``` ##### `get_inactivity_penalty_deltas` @@ -296,12 +308,14 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S rewards = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, get_previous_epoch(state)) + matching_target_indices = get_unslashed_participating_indices( + state, TIMELY_TARGET_FLAG, get_previous_epoch(state) + ) for index in get_eligible_validator_indices(state): if index in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) - return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) ``` ### Block processing @@ -484,7 +498,7 @@ def process_final_updates(state: BeaconState) -> None: ): validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH if state.balance_denominator >= 2 * GWEI_PER_ETH: - state.balances = [x//2 for x in state.balances] + state.balances = [x // 2 for x in state.balances] state.balance_denominator //= 2 # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) From 958173b5bfcb070945545bd317ecec613b9e307e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 5 Jan 2021 22:19:08 +0800 Subject: [PATCH 029/140] Fix typo --- specs/lightclient/beacon-chain.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 29678ef45..b4c776afd 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -25,7 +25,7 @@ - [Helper functions](#helper-functions) - [`Predicates`](#predicates) - [`get_base_reward`](#get_base_reward) - - [`get_base_reward_per_eth`](#get_base_reward_per_eth) + - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) @@ -187,13 +187,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH ``` -##### `get_base_reward_per_eth` +#### `get_base_reward_per_eth` ```python def get_base_reward_per_eth(state: BeaconState) -> Gwei: total_balance = get_total_active_balance(state) - total_balance = get_total_active_balance(state) - effective_balance = state.validators[index].effective_balance return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) ``` @@ -308,7 +306,7 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S rewards = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) - matching_target_indices = get_unslashed_participating_indices( + matching_target_attesting_indices = get_unslashed_participating_indices( state, TIMELY_TARGET_FLAG, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): From 39d3a18d4845f4f994a2f306a7cf97102764afe4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 00:42:01 +0800 Subject: [PATCH 030/140] Fix `upgrade_to_lightclient_patch` --- specs/lightclient/lightclient-fork.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 568a9793b..422d0a0a4 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -43,6 +43,7 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: epoch = get_current_epoch(pre) post = BeaconState( genesis_time=pre.genesis_time, + genesis_validators_root=pre.genesis_validators_root, slot=pre.slot, fork=Fork( previous_version=pre.fork.current_version, @@ -67,14 +68,15 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: slashings=pre.slashings, # Attestations # previous_epoch_attestations is cleared on upgrade. - previous_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + previous_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), # empty in pre state, since the upgrade is performed just after an epoch boundary. - current_epoch_attestations=List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH](), + current_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, + balance_denominator=GWEI_PER_ETH, ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) From 53ad66a4e841c4bbb207dc1bc2cea304e8c17cf2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 01:11:46 +0800 Subject: [PATCH 031/140] Arrange constants and configurations --- specs/lightclient/beacon-chain.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index b4c776afd..0c3ed2b73 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -12,7 +12,6 @@ - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) - - [Constants](#constants-1) - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) @@ -79,16 +78,11 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | - | - | | `PARTICIPATION_FLAGS_LENGTH` | `8` | | `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | - -## Configuration - -### Constants - -| Name | Value | -| - | - | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | | `GWEI_PER_ETH` | `10**9` | +## Configuration + ### Misc | Name | Value | From 9c75c3819da5f57b58a756096c0da00fcde50bcd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 6 Jan 2021 02:09:39 +0800 Subject: [PATCH 032/140] Fix `previous_epoch_participation` and `current_epoch_participation` initialization --- specs/lightclient/beacon-chain.md | 48 ++++++++++++++++++- specs/lightclient/lightclient-fork.md | 6 +-- .../eth2spec/test/helpers/attestations.py | 20 +++++--- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 0c3ed2b73..4fff3a4f1 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -34,6 +34,7 @@ - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) + - [New `process_deposit`](#new-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) @@ -368,6 +369,51 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` + +#### New `process_deposit` + +*Note*: The function `process_deposit` is modified to initialize `previous_epoch_participation` and `current_epoch_participation`. + +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + #### Sync committee processing ```python @@ -502,4 +548,4 @@ def process_final_updates(state: BeaconState) -> None: state.historical_roots.append(hash_tree_root(historical_batch)) # Rotate current/previous epoch participation flags state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [] + state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 422d0a0a4..8e2791d4b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,10 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - # previous_epoch_attestations is cleared on upgrade. - previous_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), - # empty in pre state, since the upgrade is performed just after an epoch boundary. - current_epoch_participation=List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT](), + previous_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], + current_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index b924da378..efc8d7a5d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE1 +from eth2spec.test.context import expect_assertion_error, PHASE1, LIGHTCLIENT_PATCH from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee @@ -30,17 +30,25 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - current_epoch_count = len(state.current_epoch_attestations) - previous_epoch_count = len(state.previous_epoch_attestations) + if spec.fork != LIGHTCLIENT_PATCH: + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) # process attestation spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert len(state.current_epoch_attestations) == current_epoch_count + 1 + if spec.fork != LIGHTCLIENT_PATCH: + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert len(state.current_epoch_attestations) == current_epoch_count + 1 + else: + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 else: - assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 + for index in spec.get_attesting_indices(state, attestation.data, attestation.aggregation_bits): + if attestation.data.target.epoch == spec.get_current_epoch(state): + assert state.current_epoch_participation[index][spec.TIMELY_TARGET_FLAG] + else: + assert state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] # yield post-state yield 'post', state From b8d3589b46cdfede1db990650fc196fbbfc6e5f7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 Jan 2021 13:48:26 -0700 Subject: [PATCH 033/140] remove global quotient penalties from fork spec [temp] --- specs/lightclient/beacon-chain.md | 133 +++++++++++++++----------- specs/lightclient/lightclient-fork.md | 1 - 2 files changed, 77 insertions(+), 57 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 4fff3a4f1..afcb3a36d 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -49,7 +49,8 @@ This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: * Light client support via sync committees -* Incentive accounting reforms, reducing spec complexity and reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Incentive accounting reforms, reducing spec complexity + and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Constants @@ -78,9 +79,7 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | | `PARTICIPATION_FLAGS_LENGTH` | `8` | -| `FLAGS_AND_NUMERATORS` | `((TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR))` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | -| `GWEI_PER_ETH` | `10**9` | ## Configuration @@ -155,8 +154,6 @@ class BeaconState(Container): # Light client sync committees current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee - # Denominator to real-time-updated balances (NOT effective balances!) - balance_denominator: uint64 ``` ### New containers @@ -173,23 +170,6 @@ class SyncCommittee(Container): ### `Predicates` -#### `get_base_reward` - -*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. Additionally, it is split into `get_base_reward_per_eth` to allow penalties to be computed to the denominator. - -```python -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - return get_base_reward_per_eth(state) * state.validators[index].effective_balance // GWEI_PER_ETH -``` - -#### `get_base_reward_per_eth` - -```python -def get_base_reward_per_eth(state: BeaconState) -> Gwei: - total_balance = get_total_active_balance(state) - return Gwei(GWEI_PER_ETH * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) -``` - #### `eth2_fast_aggregate_verify` ```python @@ -202,6 +182,21 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s return bls.FastAggregateVerify(pubkeys, message, signature) ``` +### Misc + +#### `flags_and_numerators` + +```python +def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: + return ( + (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), + (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), + (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR) + ) +``` + + + ### Beacon state accessors #### `get_sync_committee_indices` @@ -246,6 +241,17 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` +#### `get_base_reward` + +*Note*: The function `get_base_reward` is modified with the removal of `BASE_REWARDS_PER_EPOCH`. + +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance)) +``` + #### `get_unslashed_participating_indices` ```python @@ -255,20 +261,24 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [index for index in get_active_validator_indices(state, epoch) - if epoch_participation[index][flag]] + participating_indices = [ + index for index in get_active_validator_indices(state, epoch) + if epoch_participation[index][flag] + ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` #### `get_flag_deltas` ```python -def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Gwei]: +def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Computes the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment @@ -281,10 +291,12 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple rewards[index] = base_reward * numerator // REWARD_DENOMINATOR else: rewards[index] = ( - (base_reward * unslashed_participating_increments // active_increments + base_reward) - * numerator // REWARD_DENOMINATOR + (base_reward * numerator * unslashed_participating_increments) + // (active_increments * REWARD_DENOMINATOR) ) - return rewards, get_base_reward_per_eth(state) * numerator // REWARD_DENOMINATOR + else: + penalties[index] = base_reward * numerator // REWARD_DENOMINATOR + return rewards, penalties ``` ##### `get_inactivity_penalty_deltas` @@ -298,17 +310,21 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S flags to determine who participated and who did not, applying the leak penalty globally and applying compensatory rewards to participants. """ - rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): - reward_numerator_sum = sum(numerator for (_, numerator) in FLAGS_AND_NUMERATORS) + reward_numerator_sum = sum(numerator for (_, numerator) in get_flags_and_numerators()) matching_target_attesting_indices = get_unslashed_participating_indices( state, TIMELY_TARGET_FLAG, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): - if index in matching_target_attesting_indices: + # If validator is performing optimally this cancels all attestation rewards for a neutral balance + penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) + if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - rewards[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) - return rewards, Gwei(GWEI_PER_ETH * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties ``` ### Block processing @@ -334,19 +350,26 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.target.epoch == compute_epoch_at_slot(data.slot) assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH assert data.index < get_committee_count_per_slot(state, data.target.epoch) + committee = get_beacon_committee(state, data.slot, data.index) assert len(attestation.aggregation_bits) == len(committee) + if data.target.epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation justified_checkpoint = state.current_justified_checkpoint else: epoch_participation = state.previous_epoch_participation justified_checkpoint = state.previous_justified_checkpoint + # Matching roots is_matching_head = data.beacon_block_root == get_block_root_at_slot(state, data.slot) is_matching_source = data.source == justified_checkpoint is_matching_target = data.target.root == get_block_root(state, data.target.epoch) assert is_matching_source + + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) + # Participation flags participation_flags = [] if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: @@ -355,18 +378,18 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: participation_flags.append(TIMELY_SOURCE_FLAG) if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: participation_flags.append(TIMELY_TARGET_FLAG) + # Update epoch participation flags proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): - for flag, numerator in FLAGS_AND_NUMERATORS: + for flag, numerator in get_flags_and_numerators(): if flag in participation_flags and not epoch_participation[index][flag]: epoch_participation[index][flag] = True proposer_reward_numerator += get_base_reward(state, index) * numerator + # Reward proposer proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) increase_balance(state, get_beacon_proposer_index(state), proposer_reward) - # Verify signature - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` @@ -406,6 +429,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Add validator and balance entries state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) + # [Added in hf-1] Initialize empty participation flags for new validator state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) else: @@ -458,6 +482,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: current_epoch = get_current_epoch(state) old_previous_justified_checkpoint = state.previous_justified_checkpoint old_current_justified_checkpoint = state.current_justified_checkpoint + # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] @@ -472,6 +497,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 + # Process finalizations bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source @@ -497,22 +523,17 @@ def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch if get_current_epoch(state) == GENESIS_EPOCH: return - flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in FLAGS_AND_NUMERATORS] + flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in get_flags_and_numerators()] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] - for (rewards, penalty) in deltas: + for (rewards, penalties) in deltas: for index in range(len(state.validators)): increase_balance(state, ValidatorIndex(index), rewards[index]) - # Bounds-friendly expansion for `denom *= (1 + penalty // GWEI_PER_ETH)` - state.balance_denominator = ( - state.balance_denominator + - penalty + - (state.balance_denominator - GWEI_PER_ETH) * penalty // GWEI_PER_ETH - ) + decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` #### Final updates -*Note*: The function `process_final_updates` is modified to handle sync committee updates, replacement of `PendingAttestation`s with participation flags, and a sliding-denominator-aware hysteresis mechanism. +*Note*: The function `process_final_updates` is modified to handle sync committee updates and with the replacement of `PendingAttestation`s with participation flags. ```python def process_final_updates(state: BeaconState) -> None: @@ -521,23 +542,21 @@ def process_final_updates(state: BeaconState) -> None: # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] - # Update sync committees + # [Added in hf-1] Update sync committees if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) # Update effective balances with hysteresis for index, validator in enumerate(state.validators): - balance_increments = state.balances[index] * HYSTERESIS_QUOTIENT // state.balance_denominator - effective_increments = validator.effective_balance // GWEI_PER_ETH * HYSTERESIS_QUOTIENT + balance = state.balances[index] + HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) + DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER + UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER if ( - balance_increments + HYSTERESIS_DOWNWARD_MULTIPLIER < effective_increments - or effective_increments + HYSTERESIS_UPWARD_MULTIPLIER < balance_increments - or get_current_epoch(state) == validator.withdrawable_epoch + balance + DOWNWARD_THRESHOLD < validator.effective_balance + or validator.effective_balance + UPWARD_THRESHOLD < balance ): - validator.effective_balance = (state.balances[index] // state.balance_denominator) * GWEI_PER_ETH - if state.balance_denominator >= 2 * GWEI_PER_ETH: - state.balances = [x // 2 for x in state.balances] - state.balance_denominator //= 2 + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) # Set randao mix @@ -546,6 +565,8 @@ def process_final_updates(state: BeaconState) -> None: if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) - # Rotate current/previous epoch participation flags + # [Added in hf-1] Rotate current/previous epoch participation flags state.previous_epoch_participation = state.current_epoch_participation state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] + # [Removed in hf-1] Rotate current/previous epoch attestations +``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index 8e2791d4b..a10e8c5f6 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -74,7 +74,6 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: previous_justified_checkpoint=pre.previous_justified_checkpoint, current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, - balance_denominator=GWEI_PER_ETH, ) # Fill in sync committees post.current_sync_committee = get_sync_committee(post, get_current_epoch(post)) From 271b9dff83521aac4e24e24e168d046949133f1b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 5 Jan 2021 14:17:34 -0700 Subject: [PATCH 034/140] toc --- specs/lightclient/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index afcb3a36d..1112ea126 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -23,12 +23,13 @@ - [`SyncCommittee`](#synccommittee) - [Helper functions](#helper-functions) - [`Predicates`](#predicates) - - [`get_base_reward`](#get_base_reward) - - [`get_base_reward_per_eth`](#get_base_reward_per_eth) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) + - [Misc](#misc-2) + - [`flags_and_numerators`](#flags_and_numerators) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) + - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) From 70e25e63900db8ff0fdf088750cbe2de821b9204 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Tue, 5 Jan 2021 21:13:35 -0800 Subject: [PATCH 035/140] Update specs/phase0/weak-subjectivity.md Co-authored-by: Hsiao-Wei Wang --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 4bb950540..a99887926 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -42,7 +42,7 @@ This document uses data structures, constants, functions, and terminology from ## Weak Subjectivity Checkpoint -Any `Checkpoint` can be used as a Weak Subjectivity Checkpoint. +Any `Checkpoint` object can be used as a Weak Subjectivity Checkpoint. These Weak Subjectivity Checkpoints are distributed by providers, downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. From a3bf632b4f01c21151c5011c70f3efac290dafd2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Jan 2021 08:39:21 -0800 Subject: [PATCH 036/140] Bugfix in sync committee proposer rewards The variable used to accumulate proposer rewards across the sync committee processing was shadowed by the per-participant proposer reward. This means the total proposer reward would simply be twice the output of `get_proposer_reward` for the last participant in the sync committee. I believe we want to sum all contributions to the proposer reward across sync committee participants which is what this PR does. --- specs/lightclient/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 14a6cc850..2fbe190e4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -193,7 +193,7 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) # Reward sync committee participants - proposer_reward = Gwei(0) + total_proposer_reward = Gwei(0) active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: base_reward = get_base_reward(state, participant_index) @@ -201,10 +201,10 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: max_participant_reward = base_reward - proposer_reward reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) increase_balance(state, participant_index, reward) - proposer_reward += proposer_reward + total_proposer_reward += proposer_reward # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), proposer_reward) + increase_balance(state, get_beacon_proposer_index(state), total_proposer_reward) ``` ### Epoch processing From b94af435dab7c97e137f83bbfb99d824159b5f81 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 12:38:50 -0800 Subject: [PATCH 037/140] Refactor helpers into separate module --- .../test/lightclient_patch/__init__.py | 0 .../test/lightclient_patch/helpers.py | 33 +++++++++++++++++++ .../lightclient_patch/sanity/test_blocks.py | 33 ++----------------- 3 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py new file mode 100644 index 000000000..b7b2381e3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py @@ -0,0 +1,33 @@ +from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.utils import bls + + +def compute_sync_committee_signature(spec, state, slot, privkey): + domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) + if slot == state.slot: + block_root = build_empty_block_for_next_slot(spec, state).parent_root + else: + block_root = spec.get_block_root_at_slot(state, slot) + signing_root = spec.compute_signing_root(block_root, domain) + return bls.Sign(privkey, signing_root) + + +def compute_aggregate_sync_committee_signature(spec, state, slot, participants): + if len(participants) == 0: + return spec.G2_POINT_AT_INFINITY + + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + compute_sync_committee_signature( + spec, + state, + slot, + privkey, + ) + ) + return bls.Aggregate(signatures) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 4fbdfc371..1698ecce1 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -1,6 +1,4 @@ import random -from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils import bls from eth2spec.test.helpers.state import ( state_transition_and_sign_block, next_epoch, @@ -8,6 +6,9 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) +from eth2spec.test.lightclient_patch.helpers import ( + compute_aggregate_sync_committee_signature, +) from eth2spec.test.context import ( PHASE0, PHASE1, with_all_phases_except, @@ -15,34 +16,6 @@ from eth2spec.test.context import ( ) -def compute_sync_committee_signature(spec, state, slot, privkey): - domain = spec.get_domain(state, spec.DOMAIN_SYNC_COMMITTEE, spec.compute_epoch_at_slot(slot)) - if slot == state.slot: - block_root = build_empty_block_for_next_slot(spec, state).parent_root - else: - block_root = spec.get_block_root_at_slot(state, slot) - signing_root = spec.compute_signing_root(block_root, domain) - return bls.Sign(privkey, signing_root) - - -def compute_aggregate_sync_committee_signature(spec, state, slot, participants): - if len(participants) == 0: - return spec.G2_POINT_AT_INFINITY - - signatures = [] - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - compute_sync_committee_signature( - spec, - state, - slot, - privkey, - ) - ) - return bls.Aggregate(signatures) - - def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) participants = random.sample(committee, int(len(committee) * fraction_full)) From 955a01c49bc816642523ce772210bae5dcbe8b08 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 12:39:13 -0800 Subject: [PATCH 038/140] Add basic test for invalid sync committee bits --- .../block_processing/__init__.py | 0 .../test_process_sync_committee.py | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py new file mode 100644 index 000000000..578c1a1c6 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -0,0 +1,36 @@ +import random +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.lightclient_patch.helpers import ( + compute_aggregate_sync_committee_signature, +) +from eth2spec.test.context import ( + PHASE0, PHASE1, + expect_assertion_error, + with_all_phases_except, + spec_state_test, +) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_sync_committee_bits(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + random_participant = random.choice(committee) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + # Exclude one participant whose signature was included. + block.body.sync_committee_bits = [index != random_participant for index in committee] + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None From 781f3444098eceda820436d440228a60af6751eb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 15:18:56 -0800 Subject: [PATCH 039/140] Add test for invalid sync committee signature --- .../test_process_sync_committee.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 578c1a1c6..90d4af3cc 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -34,3 +34,26 @@ def test_invalid_sync_committee_bits(spec, state): yield 'blocks', [block] expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_sync_committee_signature(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + random_participant = random.choice(committee) + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + # Exclude one signature even though the block claims the entire committee participated. + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + [index for index in committee if index != random_participant], + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None From 547cb0f38f78e490760e544ac549feb0c48bd622 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 5 Jan 2021 17:04:40 -0800 Subject: [PATCH 040/140] Add epoch processing test for sync committee updates --- .../epoch_processing/__init__.py | 0 .../test_process_final_updates.py | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py new file mode 100644 index 000000000..227b2461d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -0,0 +1,37 @@ +from eth2spec.test.context import ( + PHASE0, PHASE1, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( + run_epoch_processing_with, +) + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_sync_committees_progress(spec, state): + current_epoch = spec.get_current_epoch(state) + # NOTE: if not in the genesis epoch, period math below needs to be + # adjusted relative to the current epoch + assert current_epoch == 0 + + first_sync_committee = state.current_sync_committee + second_sync_committee = state.next_sync_committee + + slot_at_end_of_current_period = spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - 1 + transition_to(spec, state, slot_at_end_of_current_period) + + # Ensure assignments have not changed: + assert state.current_sync_committee == first_sync_committee + assert state.next_sync_committee == second_sync_committee + + yield from run_epoch_processing_with(spec, state, 'process_final_updates') + + # Can compute the third committee having computed final balances in the last epoch + # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` + third_sync_committee = spec.get_sync_committee(state, 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + + assert state.current_sync_committee == second_sync_committee + assert state.next_sync_committee == third_sync_committee From cc7ae4abd0551f72c22b0493755f41a2349a4563 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 6 Jan 2021 09:44:17 -0800 Subject: [PATCH 041/140] Add test for sync committee block rewards --- .../test_process_sync_committee.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 90d4af3cc..1a3b41fa4 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -2,6 +2,9 @@ import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block, +) from eth2spec.test.lightclient_patch.helpers import ( compute_aggregate_sync_committee_signature, ) @@ -57,3 +60,53 @@ def test_invalid_sync_committee_signature(spec, state): yield 'blocks', [block] expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) yield 'post', None + + +def compute_sync_committee_participant_reward(spec, state, participant_index, active_validator_count, committee_size): + base_reward = spec.get_base_reward(state, participant_index) + proposer_reward = spec.get_proposer_reward(state, participant_index) + max_participant_reward = base_reward - proposer_reward + return max_participant_reward * active_validator_count // committee_size // spec.SLOTS_PER_EPOCH + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_sync_committee_rewards(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee_size = len(committee) + active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + + yield 'pre', state + + pre_balances = state.balances.copy() + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + for index in range(len(state.validators)): + expected_reward = 0 + + if index == block.proposer_index: + expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) + + if index in committee: + expected_reward += compute_sync_committee_participant_reward( + spec, + state, + index, + active_validator_count, + committee_size + ) + + assert state.balances[index] == pre_balances[index] + expected_reward From 7e82b54131ad2252c0b8749128297da1b1633753 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 09:45:20 -0800 Subject: [PATCH 042/140] Update tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py Add clarifying comment Co-authored-by: Danny Ryan --- .../block_processing/test_process_sync_committee.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 1a3b41fa4..a4e31335a 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -31,7 +31,7 @@ def test_invalid_sync_committee_bits(spec, state): spec, state, block.slot - 1, - committee, + committee, # full committee signs ) yield 'blocks', [block] From 049075b44a3bd7d2ebc363d29ecddf471b03ef3a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:42:21 -0800 Subject: [PATCH 043/140] Refactor sync committee helpers --- .../{lightclient_patch/helpers.py => helpers/sync_committee.py} | 0 .../block_processing/test_process_sync_committee.py | 2 +- .../eth2spec/test/lightclient_patch/sanity/test_blocks.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/core/pyspec/eth2spec/test/{lightclient_patch/helpers.py => helpers/sync_committee.py} (100%) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/helpers.py rename to tests/core/pyspec/eth2spec/test/helpers/sync_committee.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index a4e31335a..f29d1ef2b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.block import ( from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) -from eth2spec.test.lightclient_patch.helpers import ( +from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, ) from eth2spec.test.context import ( diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 1698ecce1..9033a0f15 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) -from eth2spec.test.lightclient_patch.helpers import ( +from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, ) from eth2spec.test.context import ( From 1a3fefcc931d6961a099e30afa5f8d282fc2c4eb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:55:51 -0800 Subject: [PATCH 044/140] Refactor epoch processing test helpers --- .../run_epoch_process_base.py => helpers/epoch_processing.py} | 0 .../epoch_processing/test_process_final_updates.py | 2 +- .../test/phase0/epoch_processing/test_process_final_updates.py | 2 +- .../test_process_justification_and_finalization.py | 2 +- .../phase0/epoch_processing/test_process_registry_updates.py | 2 +- .../epoch_processing/test_process_rewards_and_penalties.py | 2 +- .../test/phase0/epoch_processing/test_process_slashings.py | 2 +- .../phase1/epoch_processing/test_process_challenge_deadlines.py | 2 +- .../epoch_processing/test_process_custody_final_updates.py | 2 +- .../phase1/epoch_processing/test_process_reveal_deadlines.py | 2 +- 10 files changed, 9 insertions(+), 9 deletions(-) rename tests/core/pyspec/eth2spec/test/{phase0/epoch_processing/run_epoch_process_base.py => helpers/epoch_processing.py} (100%) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/run_epoch_process_base.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/run_epoch_process_base.py rename to tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py index 227b2461d..012438cd8 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -4,7 +4,7 @@ from eth2spec.test.context import ( spec_state_test, ) from eth2spec.test.helpers.state import transition_to -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, ) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py index 27676a59a..f9b5c872b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) from eth2spec.test.helpers.state import transition_to diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index f8504fc4f..921289f5b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with ) from eth2spec.test.helpers.state import transition_to diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py index c734010f3..ee40984a6 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_registry_updates.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with def run_process_registry_updates(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index dd3cbb791..8560b66a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -19,7 +19,7 @@ from eth2spec.test.helpers.attestations import ( ) from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from random import Random diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 3e96f8b58..2e09f5c8a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -1,5 +1,5 @@ from eth2spec.test.context import spec_state_test, with_all_phases -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( +from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) from eth2spec.test.helpers.state import next_epoch diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py index 0350324a5..1d8adecbc 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_challenge_deadlines.py @@ -15,7 +15,7 @@ from eth2spec.test.context import ( with_configs, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py index 5994306d9..82ecde7ab 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py @@ -17,7 +17,7 @@ from eth2spec.test.context import ( spec_state_test, ) from eth2spec.test.phase0.block_processing.test_process_attestation import run_attestation_processing -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_chunk_challenge import ( run_chunk_challenge_processing, diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py index 3c2060ba5..b95082491 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_reveal_deadlines.py @@ -10,7 +10,7 @@ from eth2spec.test.context import ( with_configs, spec_state_test, ) -from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.helpers.epoch_processing import run_epoch_processing_with from eth2spec.test.phase1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing From 61d141b4db8978d6e3b825b6f7ccdc37ea559ab6 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 10:57:54 -0800 Subject: [PATCH 045/140] Use more clear names for tests --- .../block_processing/test_process_sync_committee.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index f29d1ef2b..e36500918 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,7 +18,7 @@ from eth2spec.test.context import ( @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_invalid_sync_committee_bits(spec, state): +def test_invalid_signature_missing_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) @@ -31,7 +31,7 @@ def test_invalid_sync_committee_bits(spec, state): spec, state, block.slot - 1, - committee, # full committee signs + committee, # full committee signs ) yield 'blocks', [block] @@ -41,7 +41,7 @@ def test_invalid_sync_committee_bits(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_invalid_sync_committee_signature(spec, state): +def test_invalid_signature_extra_participant(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) random_participant = random.choice(committee) From ac6dbd1c3531c24cd2d0f07d3a5cda180ca26dbb Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 11:28:21 -0800 Subject: [PATCH 046/140] Add sync committee test for signature over incorrect block --- .../test_process_sync_committee.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index e36500918..5f49e2131 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -1,6 +1,7 @@ import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, + transition_unsigned_block, ) from eth2spec.test.helpers.state import ( state_transition_and_sign_block, @@ -110,3 +111,42 @@ def test_sync_committee_rewards(spec, state): ) assert state.balances[index] == pre_balances[index] + expected_reward + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_signature_past_block(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + + yield 'pre', state + + blocks = [] + for _ in range(2): + # NOTE: need to transition twice to move beyond the degenerate case at genesis + block = build_empty_block_for_next_slot(spec, state) + # Valid sync committee signature here... + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + blocks.append(signed_block) + + invalid_block = build_empty_block_for_next_slot(spec, state) + # Invalid signature from a slot other than the previous + invalid_block.body.sync_committee_bits = [True] * len(committee) + invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + invalid_block.slot - 2, + committee, + ) + blocks.append(invalid_block) + + expect_assertion_error(lambda: transition_unsigned_block(spec, state, invalid_block)) + + yield 'blocks', blocks + yield 'post', None From 500158828502bafb980a48ade5285d384bd9a183 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 7 Jan 2021 12:27:24 -0800 Subject: [PATCH 047/140] Add additional sync committee tests --- .../test_process_sync_committee.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 5f49e2131..522e248fb 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.state import ( state_transition_and_sign_block, + transition_to, ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, @@ -112,6 +113,7 @@ def test_sync_committee_rewards(spec, state): assert state.balances[index] == pre_balances[index] + expected_reward + @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test def test_invalid_signature_past_block(spec, state): @@ -150,3 +152,68 @@ def test_invalid_signature_past_block(spec, state): yield 'blocks', blocks yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_invalid_signature_previous_committee(spec, state): + # NOTE: the `state` provided is at genesis and the process to select + # sync committees currently returns the same committee for the first and second + # periods at genesis. + # To get a distinct committee so we can generate an "old" signature, we need to advance + # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. + current_epoch = spec.get_current_epoch(state) + previous_committee = state.next_sync_committee + + epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_future_sync_committee_period) + + pubkeys = [validator.pubkey for validator in state.validators] + committee = [pubkeys.index(pubkey) for pubkey in previous_committee.pubkeys] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + yield 'blocks', [block] + expect_assertion_error(lambda: spec.process_sync_committee(state, block.body)) + yield 'post', None + + +@with_all_phases_except([PHASE0, PHASE1]) +@spec_state_test +def test_valid_signature_next_committee(spec, state): + current_epoch = spec.get_current_epoch(state) + next_committee = state.next_sync_committee + epoch_in_next_sync_committee_period = current_epoch + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_next_sync_committee_period = epoch_in_next_sync_committee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_next_sync_committee_period) + + assert state.current_sync_committee == next_committee + + pubkeys = [validator.pubkey for validator in state.validators] + committee = [pubkeys.index(pubkey) for pubkey in next_committee.pubkeys] + + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state From f113413e5b3fabb8d41efcbd3dc5c52601399cb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 11 Jan 2021 18:06:37 +0800 Subject: [PATCH 048/140] Fix ToC --- specs/lightclient/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1112ea126..892f5bbb3 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -32,7 +32,7 @@ - [`get_base_reward`](#get_base_reward) - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - - [`get_inactivity_penalty_deltas`](#get_inactivity_penalty_deltas) + - [New `get_inactivity_penalty_deltas`](#new-get_inactivity_penalty_deltas) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) - [New `process_deposit`](#new-process_deposit) @@ -300,7 +300,7 @@ def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple return rewards, penalties ``` -##### `get_inactivity_penalty_deltas` +#### New `get_inactivity_penalty_deltas` *Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. From e518c4d04d1fb741d4b22a71d48a6f4b38582b6d Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 12 Jan 2021 09:31:07 -0800 Subject: [PATCH 049/140] update test to use fresh sync committees the way the test infra is built we end up with two identical sync committees at epoch 0. --- .../test_process_sync_committee.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 522e248fb..37cd0992b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -190,27 +190,40 @@ def test_invalid_signature_previous_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test -def test_valid_signature_next_committee(spec, state): +def test_valid_signature_future_committee(spec, state): + # NOTE: the `state` provided is at genesis and the process to select + # sync committees currently returns the same committee for the first and second + # periods at genesis. + # To get a distinct committee so we can generate an "old" signature, we need to advance + # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) - next_committee = state.next_sync_committee - epoch_in_next_sync_committee_period = current_epoch + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD - slot_in_next_sync_committee_period = epoch_in_next_sync_committee_period * spec.SLOTS_PER_EPOCH - transition_to(spec, state, slot_in_next_sync_committee_period) + old_current_sync_committee = state.current_sync_committee + old_next_sync_committee = state.next_sync_committee - assert state.current_sync_committee == next_committee + epoch_in_future_sync_committee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + slot_in_future_sync_committee_period = epoch_in_future_sync_committee_period * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_future_sync_committee_period) + + sync_committee = state.current_sync_committee + + expected_sync_committee = spec.get_sync_committee(state, epoch_in_future_sync_committee_period) + + assert sync_committee == expected_sync_committee + assert sync_committee != old_current_sync_committee + assert sync_committee != old_next_sync_committee pubkeys = [validator.pubkey for validator in state.validators] - committee = [pubkeys.index(pubkey) for pubkey in next_committee.pubkeys] + committee_indices = [pubkeys.index(pubkey) for pubkey in sync_committee.pubkeys] yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = [True] * len(committee_indices) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - committee, + committee_indices, ) signed_block = state_transition_and_sign_block(spec, state, block) From 252c331255adc4920ea88758b007825e895be6fb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 14:29:18 +0800 Subject: [PATCH 050/140] Fix existing tests on #2176 (accounting reform) (#2180) * Fixing tests * Add `is_post_lightclient_patch` helper to determine the fork version condition --- tests/core/pyspec/eth2spec/test/context.py | 8 + .../eth2spec/test/helpers/attestations.py | 9 +- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../pyspec/eth2spec/test/helpers/rewards.py | 137 +++++++++++++----- ..._process_justification_and_finalization.py | 55 ++++--- .../test_process_rewards_and_penalties.py | 21 +-- .../test/phase0/rewards/test_basic.py | 22 +-- .../eth2spec/test/phase0/rewards/test_leak.py | 12 +- .../test/phase0/sanity/test_blocks.py | 24 ++- 9 files changed, 197 insertions(+), 95 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index d19547477..4f07a790a 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -390,3 +390,11 @@ def only_full_crosslink(fn): return None return fn(*args, spec=spec, state=state, **kw) return wrapper + + +def is_post_lightclient_patch(spec): + if spec.fork in [PHASE0, PHASE1]: + # TODO: PHASE1 fork is temporarily parallel to LIGHTCLIENT_PATCH. + # Will make PHASE1 fork inherit LIGHTCLIENT_PATCH later. + return False + return True diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index efc8d7a5d..cf62482da 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -2,7 +2,7 @@ from lru import LRU from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import expect_assertion_error, PHASE1, is_post_lightclient_patch from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee @@ -30,7 +30,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', None return - if spec.fork != LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): current_epoch_count = len(state.current_epoch_attestations) previous_epoch_count = len(state.previous_epoch_attestations) @@ -38,7 +38,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): spec.process_attestation(state, attestation) # Make sure the attestation has been processed - if spec.fork != LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): if attestation.data.target.epoch == spec.get_current_epoch(state): assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: @@ -323,7 +323,8 @@ def prepare_state_with_attestations(spec, state, participation_fn=None): next_slot(spec, state) assert state.slot == next_epoch_start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY - assert len(state.previous_epoch_attestations) == len(attestations) + if not is_post_lightclient_patch(spec): + assert len(state.previous_epoch_attestations) == len(attestations) return attestations diff --git a/tests/core/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py index c4d8f1931..7501c8268 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/block.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -91,7 +91,7 @@ def build_empty_block(spec, state, slot=None): empty_block.body.eth1_data.deposit_count = state.eth1_deposit_index empty_block.parent_root = parent_block_root - if spec.fork == LIGHTCLIENT_PATCH: + if is_post_lightclient_patch(spec): empty_block.body.sync_committee_signature = spec.G2_POINT_AT_INFINITY apply_randao_reveal(spec, state, empty_block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index dc355972f..770519384 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -2,11 +2,11 @@ from random import Random from lru import LRU from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List, Bitvector class Deltas(Container): @@ -38,24 +38,35 @@ def run_deltas(spec, state): - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + + if is_post_lightclient_patch(spec): + def get_source_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG, spec.TIMELY_SOURCE_NUMERATOR) + + def get_head_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG, spec.TIMELY_HEAD_NUMERATOR) + + def get_target_deltas(state): + return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG, spec.TIMELY_TARGET_NUMERATOR) + yield from run_attestation_component_deltas( spec, state, - spec.get_source_deltas, + spec.get_source_deltas if not is_post_lightclient_patch(spec) else get_source_deltas, spec.get_matching_source_attestations, 'source_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_target_deltas, + spec.get_target_deltas if not is_post_lightclient_patch(spec) else get_target_deltas, spec.get_matching_target_attestations, 'target_deltas', ) yield from run_attestation_component_deltas( spec, state, - spec.get_head_deltas, + spec.get_head_deltas if not is_post_lightclient_patch(spec) else get_head_deltas, spec.get_matching_head_attestations, 'head_deltas', ) @@ -63,6 +74,16 @@ def run_deltas(spec, state): yield from run_get_inactivity_penalty_deltas(spec, state) +def deltas_name_to_flag(spec, deltas_name): + if 'source' in deltas_name: + return spec.TIMELY_SOURCE_FLAG + elif 'head' in deltas_name: + return spec.TIMELY_HEAD_FLAG + elif 'target' in deltas_name: + return spec.TIMELY_TARGET_FLAG + raise ValueError("Wrong deltas_name %s" % deltas_name) + + def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): """ Run ``component_delta_fn``, yielding: @@ -72,8 +93,14 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a yield deltas_name, Deltas(rewards=rewards, penalties=penalties) - matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) - matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + if not is_post_lightclient_patch(spec): + matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) + matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + else: + matching_indices = spec.get_unslashed_participating_indices( + state, deltas_name_to_flag(spec, deltas_name), spec.get_previous_epoch(state) + ) + eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): if index not in eligible_indices: @@ -102,6 +129,12 @@ def run_get_inclusion_delay_deltas(spec, state): Run ``get_inclusion_delay_deltas``, yielding: - inclusion delay deltas ('inclusion_delay_deltas') """ + if is_post_lightclient_patch(spec): + # No inclusion_delay_deltas + yield 'inclusion_delay_deltas', Deltas(rewards=[0] * len(state.validators), + penalties=[0] * len(state.validators)) + return + rewards, penalties = spec.get_inclusion_delay_deltas(state) yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) @@ -149,8 +182,14 @@ def run_get_inactivity_penalty_deltas(spec, state): yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + if not is_post_lightclient_patch(spec): + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + else: + matching_attesting_indices = spec.get_unslashed_participating_indices( + state, spec.TIMELY_TARGET_FLAG, spec.get_previous_epoch(state) + ) + reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flags_and_numerators()) eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): @@ -160,12 +199,14 @@ def run_get_inactivity_penalty_deltas(spec, state): continue if spec.is_in_inactivity_leak(state): - if spec.fork == LIGHTCLIENT_PATCH: - cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - 1 - else: + # Compute base_penalty + if not is_post_lightclient_patch(spec): cancel_base_rewards_per_epoch = spec.BASE_REWARDS_PER_EPOCH - base_reward = spec.get_base_reward(state, index) - base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) + base_reward = spec.get_base_reward(state, index) + base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) + else: + base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.REWARD_DENOMINATOR + if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: @@ -267,8 +308,13 @@ def run_test_full_all_correct(spec, state): def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): cached_prepare_state_with_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] + if not is_post_lightclient_patch(spec): + for a in state.previous_epoch_attestations: + a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] + else: + for index in range(len(state.validators)): + if rng.choice([True, False]): + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -277,8 +323,12 @@ def run_test_partial(spec, state, fraction_filled): cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations - num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + if not is_post_lightclient_patch(spec): + num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + else: + for index in range(int(len(state.validators) * fraction_filled)): + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -333,13 +383,18 @@ def run_test_some_very_low_effective_balances_that_attested(spec, state): def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): cached_prepare_state_with_attestations(spec, state) - # Remove attestation - attestation = state.previous_epoch_attestations[0] - state.previous_epoch_attestations = state.previous_epoch_attestations[1:] - # Set removed indices effective balance to very low amount - indices = spec.get_unslashed_attesting_indices(state, [attestation]) - for i, index in enumerate(indices): - state.validators[index].effective_balance = i + if not is_post_lightclient_patch(spec): + # Remove attestation + attestation = state.previous_epoch_attestations[0] + state.previous_epoch_attestations = state.previous_epoch_attestations[1:] + # Set removed indices effective balance to very low amount + indices = spec.get_unslashed_attesting_indices(state, [attestation]) + for i, index in enumerate(indices): + state.validators[index].effective_balance = i + else: + index = 0 + state.validators[index].effective_balance = 1 + state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() yield from run_deltas(spec, state) @@ -447,16 +502,26 @@ def run_test_full_random(spec, state, rng=Random(8020)): cached_prepare_state_with_attestations(spec, state) - for pending_attestation in state.previous_epoch_attestations: - # ~1/3 have bad target - if rng.randint(0, 2) == 0: - pending_attestation.data.target.root = b'\x55' * 32 - # ~1/3 have bad head - if rng.randint(0, 2) == 0: - pending_attestation.data.beacon_block_root = b'\x66' * 32 - # ~50% participation - pending_attestation.aggregation_bits = [rng.choice([True, False]) for _ in pending_attestation.aggregation_bits] - # Random inclusion delay - pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + if not is_post_lightclient_patch(spec): + for pending_attestation in state.previous_epoch_attestations: + # ~1/3 have bad target + if rng.randint(0, 2) == 0: + pending_attestation.data.target.root = b'\x55' * 32 + # ~1/3 have bad head + if rng.randint(0, 2) == 0: + pending_attestation.data.beacon_block_root = b'\x66' * 32 + # ~50% participation + pending_attestation.aggregation_bits = [rng.choice([True, False]) + for _ in pending_attestation.aggregation_bits] + # Random inclusion delay + pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + else: + for index in range(len(state.validators)): + # ~1/3 have bad target + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.randint(0, 2) != 0 + # ~1/3 have bad head + state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = rng.randint(0, 2) != 0 + # ~50% participation + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index f8504fc4f..14623ea9a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import is_post_lightclient_patch, spec_state_test, with_all_phases from eth2spec.test.phase0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with ) @@ -16,12 +16,20 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support previous_epoch = spec.get_previous_epoch(state) current_epoch = spec.get_current_epoch(state) - if current_epoch == epoch: - attestations = state.current_epoch_attestations - elif previous_epoch == epoch: - attestations = state.previous_epoch_attestations + if not is_post_lightclient_patch(spec): + if current_epoch == epoch: + attestations = state.current_epoch_attestations + elif previous_epoch == epoch: + attestations = state.previous_epoch_attestations + else: + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") else: - raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") + if current_epoch == epoch: + epoch_participation = state.current_epoch_participation + elif previous_epoch == epoch: + epoch_participation = state.previous_epoch_participation + else: + raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") total_balance = spec.get_total_active_balance(state) remaining_balance = int(total_balance * 2 // 3) # can become negative @@ -52,19 +60,28 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support for i in range(max(len(committee) // 5, 1)): aggregation_bits[i] = 0 - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - slot=slot, - beacon_block_root=b'\xff' * 32, # irrelevant to testing - source=source, - target=target, - index=index, - ), - inclusion_delay=1, - )) - if messed_up_target: - attestations[len(attestations) - 1].data.target.root = b'\x99' * 32 + # Update state + if not is_post_lightclient_patch(spec): + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + slot=slot, + beacon_block_root=b'\xff' * 32, # irrelevant to testing + source=source, + target=target, + index=index, + ), + inclusion_delay=1, + )) + if messed_up_target: + attestations[len(attestations) - 1].data.target.root = b'\x99' * 32 + else: + for i, index in enumerate(committee): + if aggregation_bits[i]: + epoch_participation[index][spec.TIMELY_HEAD_FLAG] = True + epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + if not messed_up_target: + epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py index dd3cbb791..cdc0be8cd 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_rewards_and_penalties.py @@ -1,11 +1,11 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, spec_state_test, spec_test, with_all_phases, single_phase, - with_phases, PHASE0, + with_phases, PHASE0, PHASE1, with_custom_state, zero_activation_threshold, misc_balances, low_single_balance, + is_post_lightclient_patch, ) from eth2spec.test.helpers.state import ( next_epoch, @@ -66,7 +66,7 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): attestations = prepare_state_with_attestations(spec, state) @@ -159,11 +159,11 @@ def run_with_participation(spec, state, participation_fn): return att_participants attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) - proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] - pre_state = state.copy() - if spec.fork == LIGHTCLIENT_PATCH: + if not is_post_lightclient_patch(spec): + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] + else: sync_committee_indices = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) yield from run_process_rewards_and_penalties(spec, state) @@ -173,11 +173,11 @@ def run_with_participation(spec, state, participation_fn): for index in range(len(pre_state.validators)): if spec.is_in_inactivity_leak(state): - # Proposers can still make money during a leak - if index in proposer_indices and index in participated: + # Proposers can still make money during a leak before LIGHTCLIENT_PATCH + if not is_post_lightclient_patch(spec) and index in proposer_indices and index in participated: assert state.balances[index] > pre_state.balances[index] elif index in attesting_indices: - if spec.fork == LIGHTCLIENT_PATCH and index in sync_committee_indices: + if is_post_lightclient_patch(spec) and index in sync_committee_indices: # The sync committee reward has not been canceled, so the sync committee participants still earn it assert state.balances[index] >= pre_state.balances[index] else: @@ -428,7 +428,8 @@ def test_attestations_some_slashed(spec, state): for i in range(spec.MIN_PER_EPOCH_CHURN_LIMIT): spec.slash_validator(state, attesting_indices_before_slashings[i]) - assert len(state.previous_epoch_attestations) == len(attestations) + if not is_post_lightclient_patch(spec): + assert len(state.previous_epoch_attestations) == len(attestations) pre_state = state.copy() diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py index 92277fdd7..7871d3fcf 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_basic.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test import eth2spec.test.helpers.rewards as rewards_helpers @@ -32,7 +32,7 @@ def test_full_but_partial_participation(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_one_attestation_one_correct(spec, state): yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) @@ -75,7 +75,7 @@ def test_some_very_low_effective_balances_that_did_not_attest(spec, state): # -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_correct_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -86,7 +86,7 @@ def test_full_half_correct_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_correct_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -97,7 +97,7 @@ def test_full_correct_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_incorrect_target_incorrect_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -108,7 +108,7 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_half_incorrect_target_correct_head(spec, state): yield from rewards_helpers.run_test_full_fraction_incorrect( @@ -119,31 +119,31 @@ def test_full_half_incorrect_target_correct_head(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_delay_one_slot(spec, state): yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_delay_max_slots(spec, state): yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_full_mixed_delay(spec, state): yield from rewards_helpers.run_test_full_mixed_delay(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_proposer_not_in_attestations(spec, state): yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test def test_duplicate_attestations_at_later_slots(spec, state): yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py index b0f9767b2..b2ed6f5d8 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_leak.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import PHASE0, PHASE1, with_all_phases, with_phases, spec_state_test from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers @@ -38,7 +38,7 @@ def test_full_but_partial_participation_leak(spec, state): yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_one_attestation_one_correct_leak(spec, state): @@ -87,7 +87,7 @@ def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): # -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_correct_target_incorrect_head_leak(spec, state): @@ -99,7 +99,7 @@ def test_full_half_correct_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_correct_target_incorrect_head_leak(spec, state): @@ -111,7 +111,7 @@ def test_full_correct_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_incorrect_target_incorrect_head_leak(spec, state): @@ -123,7 +123,7 @@ def test_full_half_incorrect_target_incorrect_head_leak(spec, state): ) -@with_all_phases +@with_phases([PHASE0, PHASE1]) @spec_state_test @leaking() def test_full_half_incorrect_target_correct_head_leak(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 358fd5211..9cc8ab721 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -35,6 +35,7 @@ from eth2spec.test.context import ( with_configs, with_custom_state, large_validator_set, + is_post_lightclient_patch, ) @@ -780,15 +781,19 @@ def test_attestation(spec, state): spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True ) + if not is_post_lightclient_patch(spec): + pre_current_attestations_len = len(state.current_epoch_attestations) + # Add to state via block transition - pre_current_attestations_len = len(state.current_epoch_attestations) attestation_block.body.attestations.append(attestation) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) - assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - - # Epoch transition should move to previous_epoch_attestations - pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + if not is_post_lightclient_patch(spec): + assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 + # Epoch transition should move to previous_epoch_attestations + pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) + else: + pre_current_epoch_participation_root = spec.hash_tree_root(state.current_epoch_participation) epoch_block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH) signed_epoch_block = state_transition_and_sign_block(spec, state, epoch_block) @@ -796,8 +801,13 @@ def test_attestation(spec, state): yield 'blocks', [signed_attestation_block, signed_epoch_block] yield 'post', state - assert len(state.current_epoch_attestations) == 0 - assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + if not is_post_lightclient_patch(spec): + assert len(state.current_epoch_attestations) == 0 + assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root + else: + for index in range(len(state.validators)): + assert state.current_epoch_participation[index] == spec.Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root # In phase1 a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, From 50765edc300d9d06aad35d5428157dc961d9090e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 15:11:24 +0800 Subject: [PATCH 051/140] Set minimal config's `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` to 8 --- configs/minimal/lightclient_patch.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index ba1179a2b..0ee39a30b 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -12,8 +12,8 @@ SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 # Time parameters # --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 +# [customized] +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 # Signature domains From 002dfaa89192f29ce5e742962cf2c41f4927e78c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 17:29:33 +0800 Subject: [PATCH 052/140] Set minimal config's `SYNC_COMMITTEE_SIZE` to 32 --- configs/minimal/lightclient_patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 0ee39a30b..afe7d897e 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -5,7 +5,7 @@ CONFIG_NAME: "minimal" # Misc # --------------------------------------------------------------- # [customized] -SYNC_COMMITTEE_SIZE: 64 +SYNC_COMMITTEE_SIZE: 32 # [customized] SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 From b2658f1091482a3993b311696d5c50b744bde51f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 13 Jan 2021 17:28:23 +0800 Subject: [PATCH 053/140] Fix SyncCommittee 1. Make `get_sync_committee_indices` do not return duplicate indices 2. Pad default values to Vectors --- specs/lightclient/beacon-chain.md | 17 +++++++++++++++-- .../eth2spec/test/helpers/sync_committee.py | 6 ++++++ .../test_process_sync_committee.py | 17 ++++++++++------- .../lightclient_patch/sanity/test_blocks.py | 5 ++++- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..1b434faf4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -50,6 +50,7 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | +| `G1_POINT_AT_INFINITY` | `BLSPubkey(b'\xc0' + b'\x00' * 47)` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ### Misc @@ -138,12 +139,19 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) i = 0 sync_committee_indices: List[ValidatorIndex] = [] - while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: + if len(active_validator_indices) < SYNC_COMMITTEE_SIZE: + committee_size = len(active_validator_indices) + else: + committee_size = SYNC_COMMITTEE_SIZE + while len(sync_committee_indices) < committee_size: shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + if ( + effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte + and candidate_index not in sync_committee_indices + ): sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices @@ -163,6 +171,11 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] + # Pad G1_POINT_AT_INFINITY to the BLSPubkey Vectors + if len(pubkeys) < SYNC_COMMITTEE_SIZE: + pubkeys += [G1_POINT_AT_INFINITY] * (SYNC_COMMITTEE_SIZE - len(pubkeys)) + aggregates_length = SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE + aggregates += [G1_POINT_AT_INFINITY] * (aggregates_length - len(aggregates)) return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index b7b2381e3..21557dedd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -31,3 +31,9 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): ) ) return bls.Aggregate(signatures) + + +def get_padded_sync_committee_bits(spec, sync_committee_bits): + if len(sync_committee_bits) < spec.SYNC_COMMITTEE_SIZE: + return sync_committee_bits + [False] * (spec.SYNC_COMMITTEE_SIZE - len(sync_committee_bits)) + return sync_committee_bits diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 37cd0992b..14f3a9f0a 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -9,6 +9,7 @@ from eth2spec.test.helpers.state import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, + get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -28,7 +29,9 @@ def test_invalid_signature_missing_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one participant whose signature was included. - block.body.sync_committee_bits = [index != random_participant for index in committee] + block.body.sync_committee_bits = get_padded_sync_committee_bits( + spec, [index != random_participant for index in committee] + ) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -51,7 +54,7 @@ def test_invalid_signature_extra_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -83,7 +86,7 @@ def test_sync_committee_rewards(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * committee_size) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -126,7 +129,7 @@ def test_invalid_signature_past_block(spec, state): # NOTE: need to transition twice to move beyond the degenerate case at genesis block = build_empty_block_for_next_slot(spec, state) # Valid sync committee signature here... - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -139,7 +142,7 @@ def test_invalid_signature_past_block(spec, state): invalid_block = build_empty_block_for_next_slot(spec, state) # Invalid signature from a slot other than the previous - invalid_block.body.sync_committee_bits = [True] * len(committee) + invalid_block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -175,7 +178,7 @@ def test_invalid_signature_previous_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -218,7 +221,7 @@ def test_valid_signature_future_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee_indices) + block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee_indices)) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 9033a0f15..93f8cd3ee 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -8,6 +8,7 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, + get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -23,7 +24,9 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [index in participants for index in committee] + block.body.sync_committee_bits = get_padded_sync_committee_bits( + spec, [index in participants for index in committee] + ) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, From 2a6699290f14bccddc9992b4a1e49eb71a6eb52b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 14 Jan 2021 01:47:40 +0800 Subject: [PATCH 054/140] Revert "Fix SyncCommittee" This reverts commit b2658f1091482a3993b311696d5c50b744bde51f. --- specs/lightclient/beacon-chain.md | 17 ++--------------- .../eth2spec/test/helpers/sync_committee.py | 6 ------ .../test_process_sync_committee.py | 17 +++++++---------- .../lightclient_patch/sanity/test_blocks.py | 5 +---- 4 files changed, 10 insertions(+), 35 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1b434faf4..2fbe190e4 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -50,7 +50,6 @@ This is a standalone beacon chain patch adding light client support via sync com | Name | Value | | - | - | -| `G1_POINT_AT_INFINITY` | `BLSPubkey(b'\xc0' + b'\x00' * 47)` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ### Misc @@ -139,19 +138,12 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val seed = get_seed(state, base_epoch, DOMAIN_SYNC_COMMITTEE) i = 0 sync_committee_indices: List[ValidatorIndex] = [] - if len(active_validator_indices) < SYNC_COMMITTEE_SIZE: - committee_size = len(active_validator_indices) - else: - committee_size = SYNC_COMMITTEE_SIZE - while len(sync_committee_indices) < committee_size: + while len(sync_committee_indices) < SYNC_COMMITTEE_SIZE: shuffled_index = compute_shuffled_index(uint64(i % active_validator_count), active_validator_count, seed) candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if ( - effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte - and candidate_index not in sync_committee_indices - ): + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices @@ -171,11 +163,6 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) ] - # Pad G1_POINT_AT_INFINITY to the BLSPubkey Vectors - if len(pubkeys) < SYNC_COMMITTEE_SIZE: - pubkeys += [G1_POINT_AT_INFINITY] * (SYNC_COMMITTEE_SIZE - len(pubkeys)) - aggregates_length = SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE - aggregates += [G1_POINT_AT_INFINITY] * (aggregates_length - len(aggregates)) return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py index 21557dedd..b7b2381e3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sync_committee.py @@ -31,9 +31,3 @@ def compute_aggregate_sync_committee_signature(spec, state, slot, participants): ) ) return bls.Aggregate(signatures) - - -def get_padded_sync_committee_bits(spec, sync_committee_bits): - if len(sync_committee_bits) < spec.SYNC_COMMITTEE_SIZE: - return sync_committee_bits + [False] * (spec.SYNC_COMMITTEE_SIZE - len(sync_committee_bits)) - return sync_committee_bits diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 14f3a9f0a..37cd0992b 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -9,7 +9,6 @@ from eth2spec.test.helpers.state import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, - get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -29,9 +28,7 @@ def test_invalid_signature_missing_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one participant whose signature was included. - block.body.sync_committee_bits = get_padded_sync_committee_bits( - spec, [index != random_participant for index in committee] - ) + block.body.sync_committee_bits = [index != random_participant for index in committee] block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -54,7 +51,7 @@ def test_invalid_signature_extra_participant(spec, state): block = build_empty_block_for_next_slot(spec, state) # Exclude one signature even though the block claims the entire committee participated. - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -86,7 +83,7 @@ def test_sync_committee_rewards(spec, state): pre_balances = state.balances.copy() block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * committee_size) + block.body.sync_committee_bits = [True] * committee_size block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -129,7 +126,7 @@ def test_invalid_signature_past_block(spec, state): # NOTE: need to transition twice to move beyond the degenerate case at genesis block = build_empty_block_for_next_slot(spec, state) # Valid sync committee signature here... - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -142,7 +139,7 @@ def test_invalid_signature_past_block(spec, state): invalid_block = build_empty_block_for_next_slot(spec, state) # Invalid signature from a slot other than the previous - invalid_block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + invalid_block.body.sync_committee_bits = [True] * len(committee) invalid_block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -178,7 +175,7 @@ def test_invalid_signature_previous_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee)) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, @@ -221,7 +218,7 @@ def test_valid_signature_future_committee(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits(spec, [True] * len(committee_indices)) + block.body.sync_committee_bits = [True] * len(committee_indices) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py index 93f8cd3ee..9033a0f15 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/test_blocks.py @@ -8,7 +8,6 @@ from eth2spec.test.helpers.block import ( ) from eth2spec.test.helpers.sync_committee import ( compute_aggregate_sync_committee_signature, - get_padded_sync_committee_bits, ) from eth2spec.test.context import ( PHASE0, PHASE1, @@ -24,9 +23,7 @@ def run_sync_committee_sanity_test(spec, state, fraction_full=1.0): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = get_padded_sync_committee_bits( - spec, [index in participants for index in committee] - ) + block.body.sync_committee_bits = [index in participants for index in committee] block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, From 65696ca68be3badd36a784ddd3c5a0e2d62ae469 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 16 Jan 2021 22:48:22 +0100 Subject: [PATCH 055/140] fix typo: same condition, but non-aggregate attestation here --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 5dc892991..5a0953f76 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -419,7 +419,7 @@ The following validations MUST pass before forwarding the `attestation` on the s - _[REJECT]_ The signature of `attestation` is valid. - _[IGNORE]_ The block being voted for (`attestation.data.beacon_block_root`) has been seen (via both gossip and non-gossip sources) - (a client MAY queue aggregates for processing once block is retrieved). + (a client MAY queue attestations for processing once block is retrieved). - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root` From 37874f9b790ba457f160f9a77e625af5bd212974 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 19 Jan 2021 01:53:12 -0800 Subject: [PATCH 056/140] Update WS calc --- specs/phase0/weak-subjectivity.md | 101 ++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index a99887926..6e2515185 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -59,38 +59,70 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. ### Calculating the Weak Subjectivity Period -*Note*: `compute_weak_subjectivity_period()` is planned to be updated when a more accurate calculation is made. +A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). ```python -def compute_weak_subjectivity_period(state: BeaconState) -> uint64: - weak_subjectivity_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) - if validator_count >= MIN_PER_EPOCH_CHURN_LIMIT * CHURN_LIMIT_QUOTIENT: - weak_subjectivity_period += SAFETY_DECAY * CHURN_LIMIT_QUOTIENT // (2 * 100) - else: - weak_subjectivity_period += SAFETY_DECAY * validator_count // (2 * 100 * MIN_PER_EPOCH_CHURN_LIMIT) - return weak_subjectivity_period -``` +def get_active_validator_count(state: BeaconState) -> uint64: + active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) + return active_validator_count -*Details about the calculation*: -- `100` appears in the denominator to get the actual percentage ratio from `SAFETY_DECAY` -- For more information about other terms in this equation, refer to - [Weak Subjectivity in Eth2.0](https://notes.ethereum.org/@adiasg/weak-subjectvity-eth2) +def compute_avg_active_validator_balance(state: BeaconState) -> Gwei: + total_active_balance = get_total_active_balance(state) + active_validator_count = get_active_validator_count(state) + avg_active_validator_balance = total_active_balance // active_validator_count + return avg_active_validator_balance//10**9 + +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = get_active_validator_count(state) + t = compute_avg_active_validator_balance(state) + T = MAX_EFFECTIVE_BALANCE//10**9 + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + case = ( + T*(200+3*D) < t*(200+12*D) + ) + + if case == 1: + arg1 = ( + N*(t*(200+12*D) - T*(200+3*D)) // (600*delta*(2*t+T)) + ) + arg2 = ( + N*(200+3*D) // (600*Delta) + ) + ws_period += max(arg1, arg2) + else: + ws_period += ( + 3*N*D*t // (200*Delta*(T-t)) + ) + + return ws_period +``` A brief reference for what these values look like in practice: -| `validator_count` | `weak_subjectivity_period` | -| ---- | ---- | -| 1024 | 268 | -| 2048 | 281 | -| 4096 | 307 | -| 8192 | 358 | -| 16384 | 460 | -| 32768 | 665 | -| 65536 | 1075 | -| 131072 | 1894 | -| 262144 | 3532 | -| 524288 | 3532 | +| SAFETY_DECAY | validator_count | average_active_validator_balance | weak_subjectivity_period | +| ---- | ---- | ---- | ---- | +| 10 | 8192 | 28 | 318 | +| 10 | 8192 | 32 | 358 | +| 10 | 16384 | 28 | 380 | +| 10 | 16384 | 32 | 460 | +| 10 | 32768 | 28 | 504 | +| 10 | 32768 | 32 | 665 | +| 20 | 8192 | 28 | 411 | +| 20 | 8192 | 32 | 460 | +| 20 | 16384 | 28 | 566 | +| 20 | 16384 | 32 | 665 | +| 20 | 32768 | 28 | 876 | +| 20 | 32768 | 32 | 1075 | +| 33 | 8192 | 28 | 532 | +| 33 | 8192 | 32 | 593 | +| 33 | 16384 | 28 | 808 | +| 33 | 16384 | 32 | 931 | +| 33 | 32768 | 28 | 1360 | +| 33 | 32768 | 32 | 1607 | ## Weak Subjectivity Sync @@ -101,17 +133,21 @@ Clients should allow users to input a Weak Subjectivity Checkpoint at startup, a 1. Input a Weak Subjectivity Checkpoint as a CLI parameter in `block_root:epoch_number` format, where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. Example of the format: -``` + +```python 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 ``` -2. - *IF* `epoch_number > store.finalized_checkpoint.epoch`, - then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. - Emit descriptive critical error if this assert fails, then exit client process. + +2. Check the weak subjectivity requirements: + - *IF* `epoch_number > store.finalized_checkpoint.epoch`, + then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. + Emit descriptive critical error if this assert fails, then exit client process. - *IF* `epoch_number <= store.finalized_checkpoint.epoch`, - then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. - Emit descriptive critical error if this assert fails, then exit client process. + then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. + Emit descriptive critical error if this assert fails, then exit client process. ### Checking for Stale Weak Subjectivity Checkpoint + Clients may choose to validate that the input Weak Subjectivity Checkpoint is not stale at the time of startup. To support this mechanism, the client needs to take the state at the Weak Subjectivity Checkpoint as a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). @@ -130,4 +166,5 @@ def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_c ``` ## Distributing Weak Subjectivity Checkpoints + This section will be updated soon. From 93c4c62900de17ebffa5786fe5f118392de9fc36 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 19 Jan 2021 02:04:55 -0800 Subject: [PATCH 057/140] Remove python tag from fenced code block --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 6e2515185..f5dcdba98 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -134,7 +134,7 @@ Clients should allow users to input a Weak Subjectivity Checkpoint at startup, a where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. Example of the format: -```python +``` 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 ``` From c5d9aa2502d57fe264702c2ad5a396ffaf1a3b5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:00:43 +0800 Subject: [PATCH 058/140] Fix test cases for minimal and mainnet configs --- .../test_process_sync_committee.py | 81 +++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 37cd0992b..fb44e855e 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -12,8 +12,10 @@ from eth2spec.test.helpers.sync_committee import ( ) from eth2spec.test.context import ( PHASE0, PHASE1, + MAINNET, MINIMAL, expect_assertion_error, with_all_phases_except, + with_configs, spec_state_test, ) @@ -72,12 +74,18 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, ac @with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MINIMAL], reason="to create nonduplicate committee") @spec_state_test -def test_sync_committee_rewards(spec, state): +def test_sync_committee_rewards_nonduplicate_committee(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + # Preconditions of this test case + # Note that the committee members MAY still be duplicate even with enough active validator count probabilistically. + assert active_validator_count >= spec.SYNC_COMMITTEE_SIZE + assert committee_size == len(set(committee)) + yield 'pre', state pre_balances = state.balances.copy() @@ -114,6 +122,67 @@ def test_sync_committee_rewards(spec, state): assert state.balances[index] == pre_balances[index] + expected_reward +@with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MAINNET], reason="to create duplicate committee") +@spec_state_test +def test_sync_committee_rewards_duplicate_committee(spec, state): + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee_size = len(committee) + active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) + + # Preconditions of this test case + # With mainnet config, where active validators are less than SYNC_COMMITTEE_SIZE, + # the committee members SHOULD be duplicate. + assert active_validator_count < spec.SYNC_COMMITTEE_SIZE + assert committee_size > len(set(committee)) + + yield 'pre', state + + pre_balances = state.balances.copy() + + block = build_empty_block_for_next_slot(spec, state) + block.body.sync_committee_bits = [True] * committee_size + block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( + spec, + state, + block.slot - 1, + committee, + ) + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + duplicate_count = {} + for i, x in enumerate(committee): + if i != committee.index(x): + if x not in duplicate_count: + duplicate_count[x] = 1 + duplicate_count[x] += 1 + + for index in range(len(state.validators)): + expected_reward = 0 + + if index == block.proposer_index: + expected_reward += sum([spec.get_proposer_reward(state, index) for index in committee]) + + if index in committee: + reward = compute_sync_committee_participant_reward( + spec, + state, + index, + active_validator_count, + committee_size, + ) + if index not in duplicate_count: + expected_reward += reward + else: + expected_reward += reward * duplicate_count[index] + + assert state.balances[index] == pre_balances[index] + expected_reward + + @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test def test_invalid_signature_past_block(spec, state): @@ -163,24 +232,26 @@ def test_invalid_signature_previous_committee(spec, state): # To get a distinct committee so we can generate an "old" signature, we need to advance # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) - previous_committee = state.next_sync_committee epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH transition_to(spec, state, slot_in_future_sync_committee_period) + # Create incorrect_committee for generating invalid signature. pubkeys = [validator.pubkey for validator in state.validators] - committee = [pubkeys.index(pubkey) for pubkey in previous_committee.pubkeys] + correct_committee = [pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] + incorrect_committee = [(correct_committee[0] + 1) % len(pubkeys)] + correct_committee[1:] + assert correct_committee != incorrect_committee yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(committee) + block.body.sync_committee_bits = [True] * len(incorrect_committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - committee, + incorrect_committee, ) yield 'blocks', [block] From c877d142bd30fb8325e2f5a99ee9fdaa2aef4111 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 20:24:25 +0800 Subject: [PATCH 059/140] Add duplicate elements warning to the docstring --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..8f0c514f0 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -130,7 +130,8 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Return the sync committee indices for a given state and epoch. - """ + Note that there may be duplicate indices in the resulting list. + """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(state, base_epoch) From 12593e8782ef4d66f557377d537aa843d68f86d0 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 19 Jan 2021 12:52:40 +0000 Subject: [PATCH 060/140] Update comments --- specs/lightclient/beacon-chain.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 8f0c514f0..54006325e 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -129,8 +129,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s ```python def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ - Return the sync committee indices for a given state and epoch. - Note that there may be duplicate indices in the resulting list. + Return the sequence of sync committee indices (which may include duplicate indices) for a given state and epoch. """ MAX_RANDOM_BYTE = 2**8 - 1 base_epoch = Epoch((max(epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD, 1) - 1) * EPOCHS_PER_SYNC_COMMITTEE_PERIOD) @@ -144,7 +143,7 @@ def get_sync_committee_indices(state: BeaconState, epoch: Epoch) -> Sequence[Val candidate_index = active_validator_indices[shuffled_index] random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: # Sample with replacement sync_committee_indices.append(candidate_index) i += 1 return sync_committee_indices From daa47987045f041a09922a3ae1b244c882aaf5aa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:21:45 +0800 Subject: [PATCH 061/140] Break down process_final_updates --- specs/phase0/beacon-chain.md | 52 ++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 08ddc853a..b05ebe6de 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -113,7 +113,12 @@ - [`process_rewards_and_penalties`](#process_rewards_and_penalties) - [Registry updates](#registry-updates) - [Slashings](#slashings) - - [Final updates](#final-updates) + - [Eth1 data votes updates](#eth1-data-votes-updates) + - [Effective balances updates](#effective-balances-updates) + - [Slashings balances updates](#slashings-balances-updates) + - [Randao mixes updates](#randao-mixes-updates) + - [Historical roots updates](#historical-roots-updates) + - [Participation records rotation](#participation-records-rotation) - [Block processing](#block-processing) - [Block header](#block-header) - [RANDAO](#randao) @@ -1250,7 +1255,12 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_final_updates(state) + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) ``` #### Helper functions @@ -1557,15 +1567,20 @@ def process_slashings(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalty) ``` -#### Final updates - +#### Eth1 data votes updates ```python -def process_final_updates(state: BeaconState) -> None: +def process_eth1_data_votes_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = Epoch(current_epoch + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] +``` + +#### Effective balances updates + +```python +def process_effective_balances_updates(state: BeaconState) -> None: # Update effective balances with hysteresis for index, validator in enumerate(state.validators): balance = state.balances[index] @@ -1577,14 +1592,41 @@ def process_final_updates(state: BeaconState) -> None: or validator.effective_balance + UPWARD_THRESHOLD < balance ): validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) +``` + +#### Slashings balances updates + +```python +def process_slashings_updates(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) +``` + +#### Randao mixes updates + +```python +def process_randao_mixes_updates(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) +``` + +#### Historical roots updates +```python +def process_historical_roots_updates(state: BeaconState) -> None: # Set historical root accumulator + next_epoch = Epoch(get_current_epoch(state) + 1) if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) +``` + +#### Participation records rotation + +```python +def process_participation_record_updates(state: BeaconState) -> None: # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations state.current_epoch_attestations = [] From fa6094837bfeab56e35d74d81f3ca15adc9221d1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:41:34 +0800 Subject: [PATCH 062/140] Update lightclient patch and phase1 specs --- specs/lightclient/beacon-chain.md | 26 +++++++++++++++++++------- specs/phase1/beacon-chain.md | 10 +++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2fbe190e4..60526d8c0 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -29,7 +29,7 @@ - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - [Components of attestation deltas](#components-of-attestation-deltas) - - [Final updates](#final-updates) + - [Sync committee updates](#sync-committee-updates) @@ -209,6 +209,22 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ### Epoch processing +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) + # Light client patch + process_sync_committee_updates(state) +``` + #### Components of attestation deltas *Note*: The function `get_inactivity_penalty_deltas` is modified with `BASE_REWARDS_PER_EPOCH` replaced by `BASE_REWARDS_PER_EPOCH - 1`. @@ -235,14 +251,10 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S return rewards, penalties ``` -#### Final updates - -*Note*: The function `process_final_updates` is modified to handle sync committee updates. +#### Sync committee updates ```python -def process_final_updates(state: BeaconState) -> None: - # FIXME: unfold the full `process_final_updates` to avoid side effects. - phase0.process_final_updates(state) +def process_sync_committee_updates(state: BeaconState) -> None: next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index c8f93cc7f..2b8e460ed 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1054,10 +1054,14 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) - process_reveal_deadlines(state) - process_challenge_deadlines(state) process_slashings(state) - process_final_updates(state) # phase 0 final updates + process_eth1_data_votes_updates(state) + process_effective_balances_updates(state) + process_slashings_updates(state) + process_randao_mixes_updates(state) + process_historical_roots_updates(state) + process_participation_record_updates(state) + # Phase 1 process_phase_1_final_updates(state) ``` From 93d19bdf4025c8eb7b2b3ab493de22113d9bae36 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 19 Jan 2021 21:34:48 +0800 Subject: [PATCH 063/140] Update and add tests --- .../eth2spec/test/helpers/epoch_processing.py | 13 ++++- .../test_process_final_updates.py | 2 +- ...est_process_effective_balances_updates.py} | 57 ++----------------- .../test_process_eth1_data_votes_updates.py | 43 ++++++++++++++ .../test_process_historical_roots_updates.py | 20 +++++++ ...st_process_participation_record_updates.py | 21 +++++++ .../test_process_randao_mixes_updates.py | 21 +++++++ .../test_process_slashings_updates.py | 20 +++++++ tests/formats/epoch_processing/README.md | 11 +++- 9 files changed, 150 insertions(+), 58 deletions(-) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_final_updates.py => test_process_effective_balances_updates.py} (52%) create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py create mode 100644 tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index b8692227f..ca1b9f802 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -1,13 +1,22 @@ process_calls = [ + # PHASE0 'process_justification_and_finalization', 'process_rewards_and_penalties', 'process_registry_updates', 'process_reveal_deadlines', 'process_challenge_deadlines', 'process_slashings', - 'process_final_updates', - 'after_process_final_updates', + 'process_eth1_data_votes_updates', + 'process_effective_balances_updates', + 'process_slashings_updates', + 'process_randao_mixes_updates', + 'process_historical_roots_updates', + 'process_participation_record_updates', + # LIGHTCLIENT_PATCH + 'process_sync_committee_updates', + # PHASE1 + 'process_phase_1_final_updates', ] diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py index 012438cd8..06473c645 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py @@ -27,7 +27,7 @@ def test_sync_committees_progress(spec, state): assert state.current_sync_committee == first_sync_committee assert state.next_sync_committee == second_sync_committee - yield from run_epoch_processing_with(spec, state, 'process_final_updates') + yield from run_epoch_processing_with(spec, state, 'process_sync_committee_updates') # Can compute the third committee having computed final balances in the last epoch # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py similarity index 52% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py index f9b5c872b..ce8f0e348 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py @@ -2,45 +2,10 @@ from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) -from eth2spec.test.helpers.state import transition_to -def run_process_final_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_final_updates') - - -@with_all_phases -@spec_state_test -def test_eth1_vote_no_reset(spec, state): - assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1 - # skip ahead to the end of the epoch - transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) - - for i in range(state.slot + 1): # add a vote for each skipped slot. - state.eth1_data_votes.append( - spec.Eth1Data(deposit_root=b'\xaa' * 32, - deposit_count=state.eth1_deposit_index, - block_hash=b'\xbb' * 32)) - - yield from run_process_final_updates(spec, state) - - assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH - - -@with_all_phases -@spec_state_test -def test_eth1_vote_reset(spec, state): - # skip ahead to the end of the voting period - state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1 - for i in range(state.slot + 1): # add a vote for each skipped slot. - state.eth1_data_votes.append( - spec.Eth1Data(deposit_root=b'\xaa' * 32, - deposit_count=state.eth1_deposit_index, - block_hash=b'\xbb' * 32)) - - yield from run_process_final_updates(spec, state) - - assert len(state.eth1_data_votes) == 0 +def run_process_effective_balances_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_effective_balances_updates') @with_all_phases @@ -48,7 +13,7 @@ def test_eth1_vote_reset(spec, state): def test_effective_balance_hysteresis(spec, state): # Prepare state up to the final-updates. # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. - run_epoch_processing_to(spec, state, 'process_final_updates') + run_epoch_processing_to(spec, state, 'process_effective_balances_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE @@ -79,21 +44,7 @@ def test_effective_balance_hysteresis(spec, state): state.validators[i].effective_balance = pre_eff state.balances[i] = bal - yield 'pre', state - spec.process_final_updates(state) - yield 'post', state + yield from run_process_effective_balances_updates(spec, state) for i, (_, _, post_eff, name) in enumerate(cases): assert state.validators[i].effective_balance == post_eff, name - - -@with_all_phases -@spec_state_test -def test_historical_root_accumulator(spec, state): - # skip ahead to near the end of the historical roots period (excl block before epoch processing) - state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 - history_len = len(state.historical_roots) - - yield from run_process_final_updates(spec, state) - - assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py new file mode 100644 index 000000000..c9d7222d7 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py @@ -0,0 +1,43 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with, +) +from eth2spec.test.helpers.state import transition_to + + +def run_process_eth1_data_votes_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_eth1_data_votes_updates') + + +@with_all_phases +@spec_state_test +def test_eth1_vote_no_reset(spec, state): + assert spec.EPOCHS_PER_ETH1_VOTING_PERIOD > 1 + # skip ahead to the end of the epoch + transition_to(spec, state, spec.SLOTS_PER_EPOCH - 1) + + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_eth1_data_votes_updates(spec, state) + + assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH + + +@with_all_phases +@spec_state_test +def test_eth1_vote_reset(spec, state): + # skip ahead to the end of the voting period + state.slot = (spec.EPOCHS_PER_ETH1_VOTING_PERIOD * spec.SLOTS_PER_EPOCH) - 1 + for i in range(state.slot + 1): # add a vote for each skipped slot. + state.eth1_data_votes.append( + spec.Eth1Data(deposit_root=b'\xaa' * 32, + deposit_count=state.eth1_deposit_index, + block_hash=b'\xbb' * 32)) + + yield from run_process_eth1_data_votes_updates(spec, state) + + assert len(state.eth1_data_votes) == 0 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py new file mode 100644 index 000000000..d1ab9524f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py @@ -0,0 +1,20 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_historical_roots_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_historical_roots_updates') + + +@with_all_phases +@spec_state_test +def test_historical_root_accumulator(spec, state): + # skip ahead to near the end of the historical roots period (excl block before epoch processing) + state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 + history_len = len(state.historical_roots) + + yield from run_process_historical_roots_updates(spec, state) + + assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py new file mode 100644 index 000000000..f5e1513e3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py @@ -0,0 +1,21 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_participation_record_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_participation_record_updates') + + +@with_all_phases +@spec_state_test +def test_updated_participation_record(spec, state): + state.previous_epoch_attestations = [spec.PendingAttestation(proposer_index=100)] + current_epoch_attestations = [spec.PendingAttestation(proposer_index=200)] + state.current_epoch_attestations = current_epoch_attestations + + yield from run_process_participation_record_updates(spec, state) + + assert state.previous_epoch_attestations == current_epoch_attestations + assert state.current_epoch_attestations == [] diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py new file mode 100644 index 000000000..b7e4a0b8f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py @@ -0,0 +1,21 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_randao_mixes_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_updates') + + +@with_all_phases +@spec_state_test +def test_updated_randao_mixes(spec, state): + next_epoch = spec.get_current_epoch(state) + 1 + state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] = b'\x56' * 32 + + yield from run_process_randao_mixes_updates(spec, state) + + assert state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == spec.get_randao_mix( + state, spec.get_current_epoch(state) + ) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py new file mode 100644 index 000000000..559b52c09 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py @@ -0,0 +1,20 @@ +from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.helpers.epoch_processing import ( + run_epoch_processing_with +) + + +def run_process_slashings_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings_updates') + + +@with_all_phases +@spec_state_test +def test_flush_slashings(spec, state): + next_epoch = spec.get_current_epoch(state) + 1 + state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] = 100 + assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] != 0 + + yield from run_process_slashings_updates(spec, state) + + assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] == 0 diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 57c9441c8..39e8050d8 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -37,10 +37,17 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: +Sub-transitions: + - `justification_and_finalization` -- `rewards_and_penalties` (limited to `minimal` config) +- `rewards_and_penalties` - `registry_updates` - `slashings` -- `final_updates` +- `eth1_data_votes_updates` +- `effective_balances_updates` +- `slashings_updates` +- `randao_mixes_updates` +- `historical_roots_updates` +- `participation_record_updates` The resulting state should match the expected `post` state. From e53213a585e00851f61d540ec24a3c19a0b30b57 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 20 Jan 2021 13:06:24 +0800 Subject: [PATCH 064/140] Minor refactor --- specs/phase0/beacon-chain.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index b05ebe6de..bfa6bdd35 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1570,8 +1570,7 @@ def process_slashings(state: BeaconState) -> None: #### Eth1 data votes updates ```python def process_eth1_data_votes_updates(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = Epoch(current_epoch + 1) + next_epoch = Epoch(get_current_epoch(state) + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] From 900eb4a83cfb808ab176d35fc3ddd60436dbc2c2 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 20 Jan 2021 19:33:25 -0800 Subject: [PATCH 065/140] Fixed CI errors --- specs/phase0/weak-subjectivity.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index f5dcdba98..2dd28df3d 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -8,7 +8,9 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Custom Types](#custom-types) - [Constants](#constants) +- [Configuration](#configuration) - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) @@ -34,10 +36,22 @@ For more information about weak subjectivity and why it is required, please refe This document uses data structures, constants, functions, and terminology from [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Beacon Chain Fork Choice](./fork-choice.md). +## Custom Types + +| Name | SSZ Equivalent | Description | +|---|---|---| +| `Ether` | `uint64` | an amount in Ether | + ## Constants -| Name | Value | -|----------------|--------------| +| Name | Value | +|---|---| +| `ETH_TO_GWEI` | `uint64(10**9)` | + +## Configuration + +| Name | Value | +|---|---| | `SAFETY_DECAY` | `uint64(10)` | ## Weak Subjectivity Checkpoint @@ -66,11 +80,12 @@ def get_active_validator_count(state: BeaconState) -> uint64: active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) return active_validator_count -def compute_avg_active_validator_balance(state: BeaconState) -> Gwei: +def compute_avg_active_validator_balance(state: BeaconState) -> Ether: total_active_balance = get_total_active_balance(state) active_validator_count = get_active_validator_count(state) - avg_active_validator_balance = total_active_balance // active_validator_count - return avg_active_validator_balance//10**9 + avg_active_validator_balance_gwei = total_active_balance // active_validator_count + avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI + return avg_active_validator_balance_eth def compute_weak_subjectivity_period(state: BeaconState) -> uint64: ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY @@ -103,7 +118,7 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: A brief reference for what these values look like in practice: -| SAFETY_DECAY | validator_count | average_active_validator_balance | weak_subjectivity_period | +| Safety Decay | Validator Count | Average Active Validator Balance | Weak Subjectivity Period | | ---- | ---- | ---- | ---- | | 10 | 8192 | 28 | 318 | | 10 | 8192 | 32 | 358 | From 17a04c2728c67410941b064cbc9897545c3e2acc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 21 Jan 2021 23:02:22 +0800 Subject: [PATCH 066/140] PR feedback from @ralexstokes --- .../test_process_sync_committee.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index fb44e855e..cec3a1a65 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -1,3 +1,4 @@ +from collections import Counter import random from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -154,12 +155,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): yield 'blocks', [signed_block] yield 'post', state - duplicate_count = {} - for i, x in enumerate(committee): - if i != committee.index(x): - if x not in duplicate_count: - duplicate_count[x] = 1 - duplicate_count[x] += 1 + multiplicities = Counter(committee) for index in range(len(state.validators)): expected_reward = 0 @@ -175,10 +171,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): active_validator_count, committee_size, ) - if index not in duplicate_count: - expected_reward += reward - else: - expected_reward += reward * duplicate_count[index] + expected_reward += reward * multiplicities[index] assert state.balances[index] == pre_balances[index] + expected_reward @@ -224,6 +217,7 @@ def test_invalid_signature_past_block(spec, state): @with_all_phases_except([PHASE0, PHASE1]) +@with_configs([MINIMAL], reason="to produce different committee sets") @spec_state_test def test_invalid_signature_previous_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select @@ -232,26 +226,27 @@ def test_invalid_signature_previous_committee(spec, state): # To get a distinct committee so we can generate an "old" signature, we need to advance # 2 EPOCHS_PER_SYNC_COMMITTEE_PERIOD periods. current_epoch = spec.get_current_epoch(state) + old_sync_committee = state.next_sync_committee epoch_in_future_sync_commitee_period = current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD slot_in_future_sync_committee_period = epoch_in_future_sync_commitee_period * spec.SLOTS_PER_EPOCH transition_to(spec, state, slot_in_future_sync_committee_period) - # Create incorrect_committee for generating invalid signature. - pubkeys = [validator.pubkey for validator in state.validators] - correct_committee = [pubkeys.index(pubkey) for pubkey in state.current_sync_committee.pubkeys] - incorrect_committee = [(correct_committee[0] + 1) % len(pubkeys)] + correct_committee[1:] - assert correct_committee != incorrect_committee - yield 'pre', state + # Use the previous sync committee to produce the signature. + pubkeys = [validator.pubkey for validator in state.validators] + # Ensure that the pubkey sets are different. + assert set(old_sync_committee.pubkeys) != set(state.current_sync_committee.pubkeys) + committee = [pubkeys.index(pubkey) for pubkey in old_sync_committee.pubkeys] + block = build_empty_block_for_next_slot(spec, state) - block.body.sync_committee_bits = [True] * len(incorrect_committee) + block.body.sync_committee_bits = [True] * len(committee) block.body.sync_committee_signature = compute_aggregate_sync_committee_signature( spec, state, block.slot - 1, - incorrect_committee, + committee, ) yield 'blocks', [block] From 3847e425b1cae8292a3fb833730068db7c7269e2 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 20 Jan 2021 16:59:27 -0800 Subject: [PATCH 067/140] refactor to use helper for duplicates in light client committees, rather than config changes --- .../test_process_sync_committee.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index fb44e855e..5ef04ef40 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,6 +18,27 @@ from eth2spec.test.context import ( with_configs, spec_state_test, ) +from eth2spec.utils.hash_function import hash + + +def get_committee_indices(spec, state, duplicates=False): + ''' + This utility function allows the caller to ensure there are or are not + duplicate validator indices in the returned committee based on + the boolean ``duplicates``. + ''' + state = state.copy() + current_epoch = spec.get_current_epoch(state) + randao_index = current_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR + while True: + committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + if duplicates: + if len(committee) != len(set(committee)): + return committee + else: + if len(committee) == len(set(committee)): + return committee + state.randao_mixes[randao_index] = hash(state.randao_mixes[randao_index]) @with_all_phases_except([PHASE0, PHASE1]) @@ -77,7 +98,7 @@ def compute_sync_committee_participant_reward(spec, state, participant_index, ac @with_configs([MINIMAL], reason="to create nonduplicate committee") @spec_state_test def test_sync_committee_rewards_nonduplicate_committee(spec, state): - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee = get_committee_indices(spec, state, duplicates=False) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) @@ -126,7 +147,7 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): @with_configs([MAINNET], reason="to create duplicate committee") @spec_state_test def test_sync_committee_rewards_duplicate_committee(spec, state): - committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) + committee = get_committee_indices(spec, state, duplicates=True) committee_size = len(committee) active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) From 0e415fe7c7d04ce1c1bacdee1288d40b6158c0b8 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 21 Jan 2021 15:33:43 -0800 Subject: [PATCH 068/140] comments no longer apply --- .../block_processing/test_process_sync_committee.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index 5ef04ef40..98db7b7ee 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -103,7 +103,6 @@ def test_sync_committee_rewards_nonduplicate_committee(spec, state): active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case - # Note that the committee members MAY still be duplicate even with enough active validator count probabilistically. assert active_validator_count >= spec.SYNC_COMMITTEE_SIZE assert committee_size == len(set(committee)) @@ -152,8 +151,6 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): active_validator_count = len(spec.get_active_validator_indices(state, spec.get_current_epoch(state))) # Preconditions of this test case - # With mainnet config, where active validators are less than SYNC_COMMITTEE_SIZE, - # the committee members SHOULD be duplicate. assert active_validator_count < spec.SYNC_COMMITTEE_SIZE assert committee_size > len(set(committee)) From c932fc279897bcc4a7bdf109a97d70ceac7cd666 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Thu, 21 Jan 2021 17:07:45 -0800 Subject: [PATCH 069/140] Fix linter errors --- specs/phase0/weak-subjectivity.md | 40 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 2dd28df3d..75cdbec40 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -14,9 +14,13 @@ - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) + - [`get_active_validator_count`](#get_active_validator_count) + - [`compute_avg_active_validator_balance`](#compute_avg_active_validator_balance) + - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) - [Weak Subjectivity Sync](#weak-subjectivity-sync) - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) - [Checking for Stale Weak Subjectivity Checkpoint](#checking-for-stale-weak-subjectivity-checkpoint) + - [`is_within_weak_subjectivity_period`](#is_within_weak_subjectivity_period) - [Distributing Weak Subjectivity Checkpoints](#distributing-weak-subjectivity-checkpoints) @@ -75,43 +79,53 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). +#### `get_active_validator_count` + ```python def get_active_validator_count(state: BeaconState) -> uint64: active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) return active_validator_count +``` +#### `compute_avg_active_validator_balance` + +```python def compute_avg_active_validator_balance(state: BeaconState) -> Ether: total_active_balance = get_total_active_balance(state) active_validator_count = get_active_validator_count(state) avg_active_validator_balance_gwei = total_active_balance // active_validator_count avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI return avg_active_validator_balance_eth +``` +#### `compute_weak_subjectivity_period` + +```python def compute_weak_subjectivity_period(state: BeaconState) -> uint64: ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY N = get_active_validator_count(state) t = compute_avg_active_validator_balance(state) - T = MAX_EFFECTIVE_BALANCE//10**9 + T = MAX_EFFECTIVE_BALANCE // 10**9 delta = get_validator_churn_limit(state) Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH D = SAFETY_DECAY case = ( - T*(200+3*D) < t*(200+12*D) + T * (200 + 3 * D) < t * (200 + 12 * D) ) if case == 1: - arg1 = ( - N*(t*(200+12*D) - T*(200+3*D)) // (600*delta*(2*t+T)) - ) - arg2 = ( - N*(200+3*D) // (600*Delta) - ) - ws_period += max(arg1, arg2) + arg1 = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + arg2 = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(arg1, arg2) else: - ws_period += ( - 3*N*D*t // (200*Delta*(T-t)) - ) + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) return ws_period ``` @@ -168,6 +182,8 @@ To support this mechanism, the client needs to take the state at the Weak Subjec a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). The check can be implemented in the following way: +#### `is_within_weak_subjectivity_period` + ```python def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint From 15e48f712bab8f16e0131ebcddb98b043c0c6fba Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 27 Jan 2021 02:55:37 +0800 Subject: [PATCH 070/140] Fix Phase 1 epoch_processing and fix epoch_processing testgen --- specs/phase1/beacon-chain.md | 5 +++-- ...ates.py => test_process_sync_committee_updates.py} | 0 tests/generators/README.md | 2 +- tests/generators/epoch_processing/main.py | 11 ++++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) rename tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/{test_process_final_updates.py => test_process_sync_committee_updates.py} (100%) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 2b8e460ed..274d3f5b6 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1054,6 +1054,8 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) + process_reveal_deadlines(state) # Phase 1 + process_challenge_deadlines(state) # Phase 1 process_slashings(state) process_eth1_data_votes_updates(state) process_effective_balances_updates(state) @@ -1061,8 +1063,7 @@ def process_epoch(state: BeaconState) -> None: process_randao_mixes_updates(state) process_historical_roots_updates(state) process_participation_record_updates(state) - # Phase 1 - process_phase_1_final_updates(state) + process_phase_1_final_updates(state) # Phase 1 ``` #### Phase 1 final updates diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_final_updates.py rename to tests/core/pyspec/eth2spec/test/lightclient_patch/epoch_processing/test_process_sync_committee_updates.py diff --git a/tests/generators/README.md b/tests/generators/README.md index 9446551fb..077a8443c 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -184,7 +184,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('final_updates', test_process_final_updates, 'minimal'), + create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), ... ]) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index fe8e0ee92..ede966465 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -33,16 +33,21 @@ def create_provider(fork_name: str, handler_name: str, if __name__ == "__main__": phase_0_mods = {key: 'eth2spec.test.phase0.epoch_processing.test_process_' + key for key in [ - 'final_updates', 'justification_and_finalization', - 'registry_updates', 'rewards_and_penalties', + 'registry_updates', 'slashings', + 'eth1_data_votes_updates', + 'effective_balances_updates', + 'slashings_updates', + 'randao_mixes_updates', + 'historical_roots_updates', + 'participation_record_updates', ]} phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ + 'reveal_deadlines', 'challenge_deadlines', 'custody_final_updates', - 'reveal_deadlines', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) gen_runner.run_generator(f"epoch_processing", [ From 1b00c10ed337eaeb49618bcdddbc5694604259c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 27 Jan 2021 14:42:50 +0800 Subject: [PATCH 071/140] Apply @michaelsproul's feedback --- specs/lightclient/beacon-chain.md | 10 +++++----- specs/phase0/beacon-chain.md | 20 +++++++++---------- specs/phase1/beacon-chain.md | 10 +++++----- .../eth2spec/test/helpers/epoch_processing.py | 10 +++++----- ...test_process_effective_balance_updates.py} | 8 ++++---- ...tes.py => test_process_eth1_data_reset.py} | 8 ++++---- ...> test_process_historical_roots_update.py} | 6 +++--- ....py => test_process_randao_mixes_reset.py} | 6 +++--- ...tes.py => test_process_slashings_reset.py} | 6 +++--- tests/formats/epoch_processing/README.md | 10 +++++----- tests/generators/epoch_processing/main.py | 10 +++++----- 11 files changed, 52 insertions(+), 52 deletions(-) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_effective_balances_updates.py => test_process_effective_balance_updates.py} (93%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_eth1_data_votes_updates.py => test_process_eth1_data_reset.py} (86%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_historical_roots_updates.py => test_process_historical_roots_update.py} (79%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_randao_mixes_updates.py => test_process_randao_mixes_reset.py} (81%) rename tests/core/pyspec/eth2spec/test/phase0/epoch_processing/{test_process_slashings_updates.py => test_process_slashings_reset.py} (82%) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index e431cf8c8..d27def8d9 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -215,11 +215,11 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) # Light client patch process_sync_committee_updates(state) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index bfa6bdd35..fcee714d9 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1255,11 +1255,11 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) ``` @@ -1569,7 +1569,7 @@ def process_slashings(state: BeaconState) -> None: #### Eth1 data votes updates ```python -def process_eth1_data_votes_updates(state: BeaconState) -> None: +def process_eth1_data_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) # Reset eth1 data votes if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: @@ -1579,7 +1579,7 @@ def process_eth1_data_votes_updates(state: BeaconState) -> None: #### Effective balances updates ```python -def process_effective_balances_updates(state: BeaconState) -> None: +def process_effective_balance_updates(state: BeaconState) -> None: # Update effective balances with hysteresis for index, validator in enumerate(state.validators): balance = state.balances[index] @@ -1596,7 +1596,7 @@ def process_effective_balances_updates(state: BeaconState) -> None: #### Slashings balances updates ```python -def process_slashings_updates(state: BeaconState) -> None: +def process_slashings_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) @@ -1605,7 +1605,7 @@ def process_slashings_updates(state: BeaconState) -> None: #### Randao mixes updates ```python -def process_randao_mixes_updates(state: BeaconState) -> None: +def process_randao_mixes_reset(state: BeaconState) -> None: current_epoch = get_current_epoch(state) next_epoch = Epoch(current_epoch + 1) # Set randao mix @@ -1614,7 +1614,7 @@ def process_randao_mixes_updates(state: BeaconState) -> None: #### Historical roots updates ```python -def process_historical_roots_updates(state: BeaconState) -> None: +def process_historical_roots_update(state: BeaconState) -> None: # Set historical root accumulator next_epoch = Epoch(get_current_epoch(state) + 1) if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 274d3f5b6..21e9751fe 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -1057,11 +1057,11 @@ def process_epoch(state: BeaconState) -> None: process_reveal_deadlines(state) # Phase 1 process_challenge_deadlines(state) # Phase 1 process_slashings(state) - process_eth1_data_votes_updates(state) - process_effective_balances_updates(state) - process_slashings_updates(state) - process_randao_mixes_updates(state) - process_historical_roots_updates(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) process_participation_record_updates(state) process_phase_1_final_updates(state) # Phase 1 ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index ca1b9f802..b1ebb5d49 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -7,11 +7,11 @@ process_calls = [ 'process_reveal_deadlines', 'process_challenge_deadlines', 'process_slashings', - 'process_eth1_data_votes_updates', - 'process_effective_balances_updates', - 'process_slashings_updates', - 'process_randao_mixes_updates', - 'process_historical_roots_updates', + 'process_eth1_data_reset', + 'process_effective_balance_updates', + 'process_slashings_reset', + 'process_randao_mixes_reset', + 'process_historical_roots_update', 'process_participation_record_updates', # LIGHTCLIENT_PATCH 'process_sync_committee_updates', diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py similarity index 93% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py index ce8f0e348..93411f657 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balances_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_effective_balance_updates.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_effective_balances_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_effective_balances_updates') +def run_process_effective_balance_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_effective_balance_updates') @with_all_phases @@ -13,7 +13,7 @@ def run_process_effective_balances_updates(spec, state): def test_effective_balance_hysteresis(spec, state): # Prepare state up to the final-updates. # Then overwrite the balances, we only want to focus to be on the hysteresis based changes. - run_epoch_processing_to(spec, state, 'process_effective_balances_updates') + run_epoch_processing_to(spec, state, 'process_effective_balance_updates') # Set some edge cases for balances max = spec.MAX_EFFECTIVE_BALANCE min = spec.EJECTION_BALANCE @@ -44,7 +44,7 @@ def test_effective_balance_hysteresis(spec, state): state.validators[i].effective_balance = pre_eff state.balances[i] = bal - yield from run_process_effective_balances_updates(spec, state) + yield from run_process_effective_balance_updates(spec, state) for i, (_, _, post_eff, name) in enumerate(cases): assert state.validators[i].effective_balance == post_eff, name diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py similarity index 86% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py index c9d7222d7..71af69f79 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_votes_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_eth1_data_reset.py @@ -5,8 +5,8 @@ from eth2spec.test.helpers.epoch_processing import ( from eth2spec.test.helpers.state import transition_to -def run_process_eth1_data_votes_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_eth1_data_votes_updates') +def run_process_eth1_data_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_eth1_data_reset') @with_all_phases @@ -22,7 +22,7 @@ def test_eth1_vote_no_reset(spec, state): deposit_count=state.eth1_deposit_index, block_hash=b'\xbb' * 32)) - yield from run_process_eth1_data_votes_updates(spec, state) + yield from run_process_eth1_data_reset(spec, state) assert len(state.eth1_data_votes) == spec.SLOTS_PER_EPOCH @@ -38,6 +38,6 @@ def test_eth1_vote_reset(spec, state): deposit_count=state.eth1_deposit_index, block_hash=b'\xbb' * 32)) - yield from run_process_eth1_data_votes_updates(spec, state) + yield from run_process_eth1_data_reset(spec, state) assert len(state.eth1_data_votes) == 0 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py similarity index 79% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py index d1ab9524f..02ce7ccba 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_historical_roots_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_historical_roots_updates') +def run_process_historical_roots_update(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update') @with_all_phases @@ -15,6 +15,6 @@ def test_historical_root_accumulator(spec, state): state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1 history_len = len(state.historical_roots) - yield from run_process_historical_roots_updates(spec, state) + yield from run_process_historical_roots_update(spec, state) assert len(state.historical_roots) == history_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py similarity index 81% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py index b7e4a0b8f..1d35965b5 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_randao_mixes_reset.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_randao_mixes_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_updates') +def run_process_randao_mixes_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_randao_mixes_reset') @with_all_phases @@ -14,7 +14,7 @@ def test_updated_randao_mixes(spec, state): next_epoch = spec.get_current_epoch(state) + 1 state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] = b'\x56' * 32 - yield from run_process_randao_mixes_updates(spec, state) + yield from run_process_randao_mixes_reset(spec, state) assert state.randao_mixes[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == spec.get_randao_mix( state, spec.get_current_epoch(state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py similarity index 82% rename from tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py rename to tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py index 559b52c09..24c350b25 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings_reset.py @@ -4,8 +4,8 @@ from eth2spec.test.helpers.epoch_processing import ( ) -def run_process_slashings_updates(spec, state): - yield from run_epoch_processing_with(spec, state, 'process_slashings_updates') +def run_process_slashings_reset(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_slashings_reset') @with_all_phases @@ -15,6 +15,6 @@ def test_flush_slashings(spec, state): state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] = 100 assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] != 0 - yield from run_process_slashings_updates(spec, state) + yield from run_process_slashings_reset(spec, state) assert state.slashings[next_epoch % spec.EPOCHS_PER_SLASHINGS_VECTOR] == 0 diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 39e8050d8..1f88b4832 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -43,11 +43,11 @@ Sub-transitions: - `rewards_and_penalties` - `registry_updates` - `slashings` -- `eth1_data_votes_updates` -- `effective_balances_updates` -- `slashings_updates` -- `randao_mixes_updates` -- `historical_roots_updates` +- `eth1_data_reset` +- `effective_balance_updates` +- `slashings_reset` +- `randao_mixes_reset` +- `historical_roots_update` - `participation_record_updates` The resulting state should match the expected `post` state. diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index ede966465..ea7639605 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -37,11 +37,11 @@ if __name__ == "__main__": 'rewards_and_penalties', 'registry_updates', 'slashings', - 'eth1_data_votes_updates', - 'effective_balances_updates', - 'slashings_updates', - 'randao_mixes_updates', - 'historical_roots_updates', + 'eth1_data_reset', + 'effective_balance_updates', + 'slashings_reset', + 'randao_mixes_reset', + 'historical_roots_update', 'participation_record_updates', ]} phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ From 742d21e914b247e7621467cfcdb2b3339bbcd14d Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 27 Jan 2021 15:16:15 -0800 Subject: [PATCH 072/140] Updates based on review --- specs/phase0/weak-subjectivity.md | 77 ++++++++++++------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 75cdbec40..42034ed1a 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -14,8 +14,6 @@ - [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) - [Weak Subjectivity Period](#weak-subjectivity-period) - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) - - [`get_active_validator_count`](#get_active_validator_count) - - [`compute_avg_active_validator_balance`](#compute_avg_active_validator_balance) - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) - [Weak Subjectivity Sync](#weak-subjectivity-sync) - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) @@ -77,44 +75,33 @@ a safety margin of at least `1/3 - SAFETY_DECAY/100`. ### Calculating the Weak Subjectivity Period -A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). The expressions in the report use fractions, whereas we only use uint64 arithmetic in eth2.0-specs. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). +A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). -#### `get_active_validator_count` +*Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). -```python -def get_active_validator_count(state: BeaconState) -> uint64: - active_validator_count = len(get_active_validator_indices(state, get_current_epoch(state))) - return active_validator_count -``` - -#### `compute_avg_active_validator_balance` - -```python -def compute_avg_active_validator_balance(state: BeaconState) -> Ether: - total_active_balance = get_total_active_balance(state) - active_validator_count = get_active_validator_count(state) - avg_active_validator_balance_gwei = total_active_balance // active_validator_count - avg_active_validator_balance_eth = avg_active_validator_balance_gwei // ETH_TO_GWEI - return avg_active_validator_balance_eth -``` +*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magintude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. #### `compute_weak_subjectivity_period` ```python def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - N = get_active_validator_count(state) - t = compute_avg_active_validator_balance(state) - T = MAX_EFFECTIVE_BALANCE // 10**9 + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI delta = get_validator_churn_limit(state) Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH D = SAFETY_DECAY - case = ( - T * (200 + 3 * D) < t * (200 + 12 * D) - ) - - if case == 1: + if T * (200 + 3 * D) < t * (200 + 12 * D): arg1 = ( N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) ) @@ -130,28 +117,22 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: return ws_period ``` -A brief reference for what these values look like in practice: +A brief reference for what these values look like in practice ([reference script](https://gist.github.com/adiasg/3aceab409b36aa9a9d9156c1baa3c248)): -| Safety Decay | Validator Count | Average Active Validator Balance | Weak Subjectivity Period | +| Safety Decay | Avg. Val. Balance (ETH) | Val. Count | Weak Sub. Period (Epochs) | | ---- | ---- | ---- | ---- | -| 10 | 8192 | 28 | 318 | -| 10 | 8192 | 32 | 358 | -| 10 | 16384 | 28 | 380 | -| 10 | 16384 | 32 | 460 | -| 10 | 32768 | 28 | 504 | -| 10 | 32768 | 32 | 665 | -| 20 | 8192 | 28 | 411 | -| 20 | 8192 | 32 | 460 | -| 20 | 16384 | 28 | 566 | -| 20 | 16384 | 32 | 665 | -| 20 | 32768 | 28 | 876 | -| 20 | 32768 | 32 | 1075 | -| 33 | 8192 | 28 | 532 | -| 33 | 8192 | 32 | 593 | -| 33 | 16384 | 28 | 808 | -| 33 | 16384 | 32 | 931 | -| 33 | 32768 | 28 | 1360 | -| 33 | 32768 | 32 | 1607 | +| 10 | 28 | 32768 | 504 | +| 10 | 28 | 65536 | 752 | +| 10 | 28 | 131072 | 1248 | +| 10 | 28 | 262144 | 2241 | +| 10 | 28 | 524288 | 2241 | +| 10 | 28 | 1048576 | 2241 | +| 10 | 32 | 32768 | 665 | +| 10 | 32 | 65536 | 1075 | +| 10 | 32 | 131072 | 1894 | +| 10 | 32 | 262144 | 3532 | +| 10 | 32 | 524288 | 3532 | +| 10 | 32 | 1048576 | 3532 | ## Weak Subjectivity Sync From a28f52729dde6f79a87c87eddcfd1f82fbc5e014 Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Wed, 27 Jan 2021 15:58:19 -0800 Subject: [PATCH 073/140] Rename variables for clarity --- specs/phase0/weak-subjectivity.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 42034ed1a..f91a04d7e 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -102,13 +102,13 @@ def compute_weak_subjectivity_period(state: BeaconState) -> uint64: D = SAFETY_DECAY if T * (200 + 3 * D) < t * (200 + 12 * D): - arg1 = ( + epochs_for_validator_set_churn = ( N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) ) - arg2 = ( + epochs_for_balance_top_ups = ( N * (200 + 3 * D) // (600 * Delta) ) - ws_period += max(arg1, arg2) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) else: ws_period += ( 3 * N * D * t // (200 * Delta * (T - t)) From 007a6f0eccfc3a6a90555d222f08d3b77a0d243d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 29 Jan 2021 10:37:19 -0700 Subject: [PATCH 074/140] a couple of hf1 notes --- specs/lightclient/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bbce43ad5..224be971d 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -473,8 +473,8 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) - process_rewards_and_penalties(state) + process_justification_and_finalization(state) # [Updated in HF1] + process_rewards_and_penalties(state) # [Updated in HF1] process_registry_updates(state) process_slashings(state) process_eth1_data_reset(state) From 6ce4b1b0e795aa5233c57cac97ffbd20a112fe2c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 Jan 2021 17:18:49 +0800 Subject: [PATCH 075/140] Fix tests --- .../eth2spec/test/helpers/epoch_processing.py | 47 +++++++++++-------- ...st_process_participation_record_updates.py | 4 +- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index b1ebb5d49..40c7cb8b0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -1,23 +1,30 @@ -process_calls = [ - # PHASE0 - 'process_justification_and_finalization', - 'process_rewards_and_penalties', - 'process_registry_updates', - 'process_reveal_deadlines', - 'process_challenge_deadlines', - 'process_slashings', - 'process_eth1_data_reset', - 'process_effective_balance_updates', - 'process_slashings_reset', - 'process_randao_mixes_reset', - 'process_historical_roots_update', - 'process_participation_record_updates', - # LIGHTCLIENT_PATCH - 'process_sync_committee_updates', - # PHASE1 - 'process_phase_1_final_updates', -] +from eth2spec.test.context import is_post_lightclient_patch + + +def get_process_calls(spec): + return [ + # PHASE0 + 'process_justification_and_finalization', + 'process_rewards_and_penalties', + 'process_registry_updates', + 'process_reveal_deadlines', + 'process_challenge_deadlines', + 'process_slashings', + 'process_eth1_data_reset', + 'process_effective_balance_updates', + 'process_slashings_reset', + 'process_randao_mixes_reset', + 'process_historical_roots_update', + # LIGHTCLIENT_PATCH replaced `process_participation_record_updates` with + # `process_epoch_participation_updates` + 'process_epoch_participation_updates' if is_post_lightclient_patch(spec) else ( + 'process_participation_record_updates' + ), + 'process_sync_committee_updates', + # PHASE1 + 'process_phase_1_final_updates', + ] def run_epoch_processing_to(spec, state, process_name: str): @@ -34,7 +41,7 @@ def run_epoch_processing_to(spec, state, process_name: str): spec.process_slot(state) # process components of epoch transition before final-updates - for name in process_calls: + for name in get_process_calls(spec): if name == process_name: break # only run when present. Later phases introduce more to the epoch-processing. diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py index f5e1513e3..978ef5739 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_participation_record_updates.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import PHASE0, spec_state_test, with_phases from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with ) @@ -8,7 +8,7 @@ def run_process_participation_record_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_participation_record_updates') -@with_all_phases +@with_phases([PHASE0]) @spec_state_test def test_updated_participation_record(spec, state): state.previous_epoch_attestations = [spec.PendingAttestation(proposer_index=100)] From ad01c85ff6497d303b176b1cc3a5a041f58ed6ad Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 07:06:29 -0700 Subject: [PATCH 076/140] minor reorder to process_epoch calls --- specs/lightclient/beacon-chain.md | 2 +- tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 224be971d..bb00a5c59 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -483,8 +483,8 @@ def process_epoch(state: BeaconState) -> None: process_randao_mixes_reset(state) process_historical_roots_update(state) # [Added in HF1] - process_sync_committee_updates(state) process_participation_flag_updates(state) + process_sync_committee_updates(state) # [Removed in HF1] -- process_participation_record_updates(state) ``` diff --git a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py index 40c7cb8b0..52479cfeb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py @@ -16,9 +16,8 @@ def get_process_calls(spec): 'process_slashings_reset', 'process_randao_mixes_reset', 'process_historical_roots_update', - # LIGHTCLIENT_PATCH replaced `process_participation_record_updates` with - # `process_epoch_participation_updates` - 'process_epoch_participation_updates' if is_post_lightclient_patch(spec) else ( + # HF1 replaced `process_participation_record_updates` with `process_participation_flag_updates` + 'process_participation_flag_updates' if is_post_lightclient_patch(spec) else ( 'process_participation_record_updates' ), 'process_sync_committee_updates', From b029c75d88f911902228b92668b7d0d8f17a894e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 07:52:06 -0700 Subject: [PATCH 077/140] must be correct target to get correct head --- specs/lightclient/beacon-chain.md | 2 +- .../pyspec/eth2spec/test/helpers/rewards.py | 19 +++++++++++++------ .../test/phase0/rewards/test_random.py | 6 ++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bb00a5c59..bcb90f756 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -374,7 +374,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Participation flags participation_flags = [] - if is_matching_head and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: + if is_matching_head and is_matching_target and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: participation_flags.append(TIMELY_HEAD_FLAG) if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): participation_flags.append(TIMELY_SOURCE_FLAG) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 770519384..81eb1f955 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -517,11 +517,18 @@ def run_test_full_random(spec, state, rng=Random(8020)): pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) else: for index in range(len(state.validators)): - # ~1/3 have bad target - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.randint(0, 2) != 0 - # ~1/3 have bad head - state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = rng.randint(0, 2) != 0 - # ~50% participation - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) + # ~1/3 have bad head or bad target or not timely enough + is_timely_correct_head = rng.randint(0, 2) != 0 + state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = is_timely_correct_head + if is_timely_correct_head: + # If timely head, then must be timely target + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + # If timely head, then must be timely source + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + else: + # ~50% of remaining have bad target or not timely enough + state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.choice([True, False]) + # ~50% of remaining have bad source or not timely enough + state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py index 83c7f7905..ae44c6640 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py +++ b/tests/core/pyspec/eth2spec/test/phase0/rewards/test_random.py @@ -29,6 +29,12 @@ def test_full_random_2(spec, state): yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) +@with_all_phases +@spec_state_test +def test_full_random_3(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(4040)) + + @with_all_phases @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test From 1ba491711947b4f908bd68b0639584b11c20442c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Feb 2021 08:35:58 -0700 Subject: [PATCH 078/140] add process_attestation tests to cover various timing and correctness scenarios --- .../eth2spec/test/helpers/attestations.py | 7 +- .../test_process_attestation.py | 159 +++++++++++++++++- 2 files changed, 160 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index cf62482da..571e19fef 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -44,11 +44,8 @@ def run_attestation_processing(spec, state, attestation, valid=True): else: assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 else: - for index in spec.get_attesting_indices(state, attestation.data, attestation.aggregation_bits): - if attestation.data.target.epoch == spec.get_current_epoch(state): - assert state.current_epoch_participation[index][spec.TIMELY_TARGET_FLAG] - else: - assert state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] + # After accounting reform, there are cases when processing an attestation does not result in any flag updates + pass # yield post-state yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py index b31cf167c..71000763b 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py @@ -2,10 +2,13 @@ from eth2spec.test.context import ( spec_state_test, always_bls, never_bls, with_all_phases, + with_all_phases_except, spec_test, low_balances, with_custom_state, - single_phase) + single_phase, + PHASE1, +) from eth2spec.test.helpers.attestations import ( run_attestation_processing, get_valid_attestation, @@ -329,3 +332,157 @@ def test_too_few_aggregation_bits(spec, state): attestation.aggregation_bits = attestation.aggregation_bits[:-1] yield from run_attestation_processing(spec, state, attestation, False) + + +# +# Full correct atttestation contents at different slot inclusions +# + +@with_all_phases +@spec_state_test +def test_correct_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_correct_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_correct_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Incorrect head but correct source/target at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_head_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Incorrect head and target but correct source at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_head_and_target_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +# +# Correct head and source but incorrect target at different slot inclusions +# + +@with_all_phases_except([PHASE1]) +@spec_state_test +def test_incorrect_target_min_inclusion_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_sqrt_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.integer_squareroot(spec.SLOTS_PER_EPOCH)) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + next_slots(spec, state, spec.SLOTS_PER_EPOCH) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation) From 3677073812c02c2ed613e14d874e444b97a89210 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:46:27 +0100 Subject: [PATCH 079/140] bitvector[8] -> uint8, for efficient packing in flags merkle tree --- specs/lightclient/beacon-chain.md | 53 +++++++++++++------ specs/lightclient/lightclient-fork.md | 4 +- .../pyspec/eth2spec/test/helpers/rewards.py | 29 ++++++---- ..._process_justification_and_finalization.py | 6 +-- .../test/phase0/sanity/test_blocks.py | 2 +- 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bcb90f756..1ac6d2a2c 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -55,15 +55,26 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice +## Custom types + +| Name | SSZ equivalent | Description | +| - | - | - | +| `ValidatorFlags` | `uint8` | Bitflags to track validator actions with | + ## Constants -### Participation flags +### Validator action flags + +This is formatted as an enum, with values `2**i` that can be combined as bit-flags. +The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. + +**Note**: unlike Phase0, a `TIMELY_TARGET_FLAG` does not imply a `TIMELY_SOURCE_FLAG`. | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `0` | -| `TIMELY_SOURCE_FLAG` | `1` | -| `TIMELY_TARGET_FLAG` | `2` | +| `TIMELY_HEAD_FLAG` | `ValidatorFlags(2**0)` (= 1) | +| `TIMELY_SOURCE_FLAG` | `ValidatorFlags(2**1)` (= 2) | +| `TIMELY_TARGET_FLAG` | `ValidatorFlags(2**2)` (= 4) | ### Participation rewards @@ -80,7 +91,6 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar | Name | Value | | - | - | -| `PARTICIPATION_FLAGS_LENGTH` | `8` | | `G2_POINT_AT_INFINITY` | `BLSSignature(b'\xc0' + b'\x00' * 95)` | ## Configuration @@ -146,8 +156,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[Bitvector[PARTICIPATION_FLAGS_LENGTH], VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -197,7 +207,15 @@ def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: ) ``` +```python +def add_flags(flags: ValidatorFlags, add: ValidatorFlags) -> ValidatorFlags: + return flags | add +``` +```python +def has_flags(flags: ValidatorFlags, has: ValidatorFlags) -> bool: + return flags & has == has +``` ### Beacon state accessors @@ -257,7 +275,10 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: + """ + Retrieves the active validator indices of the given epoch, who are not slashed, and have all of the given flags. + """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation @@ -265,7 +286,7 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: epoch_participation = state.previous_epoch_participation participating_indices = [ index for index in get_active_validator_indices(state, epoch) - if epoch_participation[index][flag] + if has_flags(epoch_participation[index], flags) ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -273,7 +294,9 @@ def get_unslashed_participating_indices(state: BeaconState, flag: uint8, epoch: #### `get_flag_deltas` ```python -def get_flag_deltas(state: BeaconState, flag: uint8, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: +def get_flag_deltas(state: BeaconState, + flag: ValidatorFlags, + numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Computes the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. @@ -385,8 +408,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not epoch_participation[index][flag]: - epoch_participation[index][flag] = True + if flag in participation_flags and not has_flags(epoch_participation[index], flag): + epoch_participation[index] = add_flags(epoch_participation[index], flag) proposer_reward_numerator += get_base_reward(state, index) * numerator # Reward proposer @@ -432,8 +455,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) - state.current_epoch_participation.append(Bitvector[PARTICIPATION_FLAGS_LENGTH]()) + state.previous_epoch_participation.append(ValidatorFlags(0)) + state.current_epoch_participation.append(ValidatorFlags(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -572,5 +595,5 @@ def process_participation_flag_updates(state: BeaconState) -> None: Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 """ state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(state.validators))] + state.current_epoch_participation = [ValidatorFlags(0) for _ in range(len(state.validators))] ``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index a10e8c5f6..f5f3d1b7b 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,8 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - previous_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], - current_epoch_participation=[Bitvector[PARTICIPATION_FLAGS_LENGTH]() for _ in range(len(pre.validators))], + previous_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 81eb1f955..feaac018b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -6,7 +6,7 @@ from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch -from eth2spec.utils.ssz.ssz_typing import Container, uint64, List, Bitvector +from eth2spec.utils.ssz.ssz_typing import Container, uint64, List class Deltas(Container): @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + state.previous_epoch_participation[index] = spec.ValidatorFlags(0) yield from run_deltas(spec, state) @@ -519,16 +519,25 @@ def run_test_full_random(spec, state, rng=Random(8020)): for index in range(len(state.validators)): # ~1/3 have bad head or bad target or not timely enough is_timely_correct_head = rng.randint(0, 2) != 0 - state.previous_epoch_participation[index][spec.TIMELY_HEAD_FLAG] = is_timely_correct_head + flags = state.previous_epoch_participation[index] + + def set_flag(f, v): + nonlocal flags + if v: + flags |= f + else: + flags &= 0xff ^ f + + set_flag(spec.TIMELY_HEAD_FLAG, is_timely_correct_head) if is_timely_correct_head: # If timely head, then must be timely target - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + set_flag(spec.TIMELY_TARGET_FLAG, True) # If timely head, then must be timely source - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + set_flag(spec.TIMELY_SOURCE_FLAG, True) else: # ~50% of remaining have bad target or not timely enough - state.previous_epoch_participation[index][spec.TIMELY_TARGET_FLAG] = rng.choice([True, False]) + set_flag(spec.TIMELY_TARGET_FLAG, rng.choice([True, False])) # ~50% of remaining have bad source or not timely enough - state.previous_epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = rng.choice([True, False]) - + set_flag(spec.TIMELY_SOURCE_FLAG, rng.choice([True, False])) + state.previous_epoch_participation[index] = flags yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index 0d1741efb..89783f987 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -78,10 +78,10 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: for i, index in enumerate(committee): if aggregation_bits[i]: - epoch_participation[index][spec.TIMELY_HEAD_FLAG] = True - epoch_participation[index][spec.TIMELY_SOURCE_FLAG] = True + epoch_participation[index] |= spec.TIMELY_HEAD_FLAG + epoch_participation[index] |= spec.TIMELY_SOURCE_FLAG if not messed_up_target: - epoch_participation[index][spec.TIMELY_TARGET_FLAG] = True + epoch_participation[index] |= spec.TIMELY_TARGET_FLAG def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 9cc8ab721..1834b290f 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -806,7 +806,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root else: for index in range(len(state.validators)): - assert state.current_epoch_participation[index] == spec.Bitvector[spec.PARTICIPATION_FLAGS_LENGTH]() + assert state.current_epoch_participation[index] == 0 assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root From e865670111c2fb89b7031f3ea6d67bf2426b03a9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:47:00 +0100 Subject: [PATCH 080/140] add missing decorators for testruns in no-bls mode --- .../block_processing/test_process_sync_committee.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py index a8be1af63..430c59b10 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/block_processing/test_process_sync_committee.py @@ -18,6 +18,7 @@ from eth2spec.test.context import ( with_all_phases_except, with_configs, spec_state_test, + always_bls, ) from eth2spec.utils.hash_function import hash @@ -196,6 +197,7 @@ def test_sync_committee_rewards_duplicate_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test +@always_bls def test_invalid_signature_past_block(spec, state): committee = spec.get_sync_committee_indices(state, spec.get_current_epoch(state)) @@ -237,6 +239,7 @@ def test_invalid_signature_past_block(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @with_configs([MINIMAL], reason="to produce different committee sets") @spec_state_test +@always_bls def test_invalid_signature_previous_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select # sync committees currently returns the same committee for the first and second From 71c28e67a13469ffc0452687f4ce6fad99a73107 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 21:48:55 +0100 Subject: [PATCH 081/140] toc update --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 1ac6d2a2c..179aec6d1 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -7,8 +7,9 @@ - [Introduction](#introduction) +- [Custom types](#custom-types) - [Constants](#constants) - - [Participation flags](#participation-flags) + - [Validator action flags](#validator-action-flags) - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) From b4ba6c57de0bb2138379a2e1761d5ab2bc3ed90e Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 1 Feb 2021 22:02:12 +0100 Subject: [PATCH 082/140] linter: first tuple element type is ValidatorFlags, not just int --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 179aec6d1..9f0572740 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -200,7 +200,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s #### `flags_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[int, int]]: +def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: return ( (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), From 1c1ba5cba291e725972601c62e97649565d1f299 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Feb 2021 12:35:00 -0700 Subject: [PATCH 083/140] minor PR feedback --- specs/lightclient/beacon-chain.md | 7 ++- .../test_process_attestation.py | 55 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 9f0572740..3508fe991 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -69,7 +69,8 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel This is formatted as an enum, with values `2**i` that can be combined as bit-flags. The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. -**Note**: unlike Phase0, a `TIMELY_TARGET_FLAG` does not imply a `TIMELY_SOURCE_FLAG`. +**Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` +due to the varying slot delay requirements of each. | Name | Value | | - | - | @@ -278,7 +279,7 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ```python def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: """ - Retrieves the active validator indices of the given epoch, who are not slashed, and have all of the given flags. + Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): @@ -299,7 +300,7 @@ def get_flag_deltas(state: BeaconState, flag: ValidatorFlags, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ - Computes the rewards and penalties associated with a particular duty, by scanning through the participation + Compute the rewards and penalties associated with a particular duty, by scanning through the participation flags to determine who participated and who did not and assigning them the appropriate rewards and penalties. """ rewards = [Gwei(0)] * len(state.validators) diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py index 71000763b..99a82879d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attestation.py @@ -365,6 +365,17 @@ def test_correct_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_correct_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=True, on_time=False) + + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Incorrect head but correct source/target at different slot inclusions # @@ -405,10 +416,27 @@ def test_incorrect_head_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_incorrect_head_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.beacon_block_root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Incorrect head and target but correct source at different slot inclusions # +# Note: current phase 1 spec checks +# `assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot))` +# so this test can't pass that until phase 1 refactor is merged @with_all_phases_except([PHASE1]) @spec_state_test def test_incorrect_head_and_target_min_inclusion_delay(spec, state): @@ -448,6 +476,20 @@ def test_incorrect_head_and_target_epoch_delay(spec, state): yield from run_attestation_processing(spec, state, attestation) +@with_all_phases +@spec_state_test +def test_incorrect_head_and_target_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.beacon_block_root = b'\x42' * 32 + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) + + # # Correct head and source but incorrect target at different slot inclusions # @@ -486,3 +528,16 @@ def test_incorrect_target_epoch_delay(spec, state): sign_attestation(spec, state, attestation) yield from run_attestation_processing(spec, state, attestation) + + +@with_all_phases +@spec_state_test +def test_incorrect_target_after_epoch_delay(spec, state): + attestation = get_valid_attestation(spec, state, signed=False, on_time=False) + # increment past latest inclusion slot + next_slots(spec, state, spec.SLOTS_PER_EPOCH + 1) + + attestation.data.target.root = b'\x42' * 32 + sign_attestation(spec, state, attestation) + + yield from run_attestation_processing(spec, state, attestation, False) From 8b217d9277fd87755cca256e0836999793be297d Mon Sep 17 00:00:00 2001 From: Aditya Asgaonkar Date: Tue, 2 Feb 2021 12:17:56 -0800 Subject: [PATCH 084/140] Fix typo Co-authored-by: Danny Ryan --- specs/phase0/weak-subjectivity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index f91a04d7e..4137efade 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -79,7 +79,7 @@ A detailed analysis of the calculation of the weak subjectivity period is made i *Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). -*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magintude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. +*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magnitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magnitude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. #### `compute_weak_subjectivity_period` From 34cea67b91cd0d36504a6376c93e13c589832a7d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 08:45:25 -0700 Subject: [PATCH 085/140] ValidatorFlags -> ValidatorFlag --- specs/lightclient/beacon-chain.md | 30 +++++++++---------- specs/lightclient/lightclient-fork.md | 4 +-- .../pyspec/eth2spec/test/helpers/rewards.py | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 3508fe991..ff69c1312 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -60,23 +60,23 @@ This is a patch implementing the first hard fork to the beacon chain, tentativel | Name | SSZ equivalent | Description | | - | - | - | -| `ValidatorFlags` | `uint8` | Bitflags to track validator actions with | +| `ValidatorFlag` | `uint8` | Bitflags to track validator actions with | ## Constants ### Validator action flags This is formatted as an enum, with values `2**i` that can be combined as bit-flags. -The `0` value is reserved as default. Remaining bits in `ValidatorFlags` may be used in future hardforks. +The `0` value is reserved as default. Remaining bits in `ValidatorFlag` may be used in future hardforks. **Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` due to the varying slot delay requirements of each. | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `ValidatorFlags(2**0)` (= 1) | -| `TIMELY_SOURCE_FLAG` | `ValidatorFlags(2**1)` (= 2) | -| `TIMELY_TARGET_FLAG` | `ValidatorFlags(2**2)` (= 4) | +| `TIMELY_HEAD_FLAG` | `ValidatorFlag(2**0)` (= 1) | +| `TIMELY_SOURCE_FLAG` | `ValidatorFlag(2**1)` (= 2) | +| `TIMELY_TARGET_FLAG` | `ValidatorFlag(2**2)` (= 4) | ### Participation rewards @@ -158,8 +158,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[ValidatorFlags, VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -201,7 +201,7 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s #### `flags_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: +def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: return ( (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), @@ -210,12 +210,12 @@ def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlags, int]]: ``` ```python -def add_flags(flags: ValidatorFlags, add: ValidatorFlags) -> ValidatorFlags: +def add_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: return flags | add ``` ```python -def has_flags(flags: ValidatorFlags, has: ValidatorFlags) -> bool: +def has_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: return flags & has == has ``` @@ -277,7 +277,7 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlags, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag, epoch: Epoch) -> Set[ValidatorIndex]: """ Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. """ @@ -297,7 +297,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag ```python def get_flag_deltas(state: BeaconState, - flag: ValidatorFlags, + flag: ValidatorFlag, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Compute the rewards and penalties associated with a particular duty, by scanning through the participation @@ -457,8 +457,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ValidatorFlags(0)) - state.current_epoch_participation.append(ValidatorFlags(0)) + state.previous_epoch_participation.append(ValidatorFlag(0)) + state.current_epoch_participation.append(ValidatorFlag(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -597,5 +597,5 @@ def process_participation_flag_updates(state: BeaconState) -> None: Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 """ state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [ValidatorFlags(0) for _ in range(len(state.validators))] + state.current_epoch_participation = [ValidatorFlag(0) for _ in range(len(state.validators))] ``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index f5f3d1b7b..aa0171b86 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -67,8 +67,8 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: # Slashings slashings=pre.slashings, # Attestations - previous_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], - current_epoch_participation=[ValidatorFlags(0) for _ in range(len(pre.validators))], + previous_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index feaac018b..2499bcffe 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = spec.ValidatorFlags(0) + state.previous_epoch_participation[index] = spec.ValidatorFlag(0) yield from run_deltas(spec, state) From 9313815976803c2e93c8fbb4b2518340c4ce83c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 08:47:46 -0700 Subject: [PATCH 086/140] put 'validator' in flags methods --- specs/lightclient/beacon-chain.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index ff69c1312..e11f50076 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -210,12 +210,12 @@ def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: ``` ```python -def add_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: +def add_validator_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: return flags | add ``` ```python -def has_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: +def has_validator_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: return flags & has == has ``` @@ -288,7 +288,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag epoch_participation = state.previous_epoch_participation participating_indices = [ index for index in get_active_validator_indices(state, epoch) - if has_flags(epoch_participation[index], flags) + if has_validator_flags(epoch_participation[index], flags) ] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -410,8 +410,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not has_flags(epoch_participation[index], flag): - epoch_participation[index] = add_flags(epoch_participation[index], flag) + if flag in participation_flags and not has_validator_flags(epoch_participation[index], flag): + epoch_participation[index] = add_validator_flags(epoch_participation[index], flag) proposer_reward_numerator += get_base_reward(state, index) * numerator # Reward proposer From b08600156e03c17ab2c6250bb85b7734a10afe94 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 4 Feb 2021 11:05:00 -0600 Subject: [PATCH 087/140] hww feedback Co-authored-by: Hsiao-Wei Wang --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index e11f50076..2773e1760 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -507,10 +507,10 @@ def process_epoch(state: BeaconState) -> None: process_slashings_reset(state) process_randao_mixes_reset(state) process_historical_roots_update(state) + # [Removed in HF1] -- process_participation_record_updates(state) # [Added in HF1] process_participation_flag_updates(state) process_sync_committee_updates(state) - # [Removed in HF1] -- process_participation_record_updates(state) ``` #### New `process_justification_and_finalization` From 0fd0db1ffc219e8686497874d975e18087f1bd40 Mon Sep 17 00:00:00 2001 From: Phong Phan Date: Sat, 6 Feb 2021 20:20:35 +0700 Subject: [PATCH 088/140] Fix some typos (#2195) * Fix typo for P2P Networking document * Fix link typo of P2P networking document * fix typo for light clients beacon chain document --- specs/lightclient/beacon-chain.md | 2 +- specs/phase0/p2p-interface.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index 2773e1760..fff8726bf 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -314,7 +314,7 @@ def get_flag_deltas(state: BeaconState, base_reward = get_base_reward(state, index) if index in unslashed_participating_indices: if is_in_inactivity_leak(state): - # Optimal participatition is fully rewarded to cancel the inactivity penalty + # Optimal participation is fully rewarded to cancel the inactivity penalty rewards[index] = base_reward * numerator // REWARD_DENOMINATOR else: rewards[index] = ( diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 5a0953f76..a08a2d136 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -101,7 +101,7 @@ It consists of four main sections: - [Compression/Encoding](#compressionencoding) - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding) - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression) + - [Why are we using Snappy for compression?](#why-are-we-using-snappy-for-compression) - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) - [libp2p implementations matrix](#libp2p-implementations-matrix) @@ -937,7 +937,7 @@ where the fields of `ENRForkID` are defined as * `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact -*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. +*Note*: `fork_digest` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. One notable exception to this rule is the distribution of bootnode ENRs prior to genesis. In this case, bootnode ENRs SHOULD be initially distributed with `eth2` field set as @@ -1223,7 +1223,7 @@ the node's fork choice prevents integration of these messages into the actual co Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). -The value is currently set to to be equal `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. +The value is currently set to be equal to `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. ### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? @@ -1369,7 +1369,7 @@ Thus, it may happen that we need to transmit an empty list - there are several w Semantically, it is not an error that a block is missing during a slot making option 2 unnatural. -Option 1 allows allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. +Option 1 allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. Under option 0, there is no way for a client to distinguish between a slot without a block and an incomplete response, but given that it already must contain logic to handle the uncertainty of a malicious peer, option 0 was chosen. @@ -1495,7 +1495,7 @@ This looks different depending on the interaction layer: implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. -### Why are using Snappy for compression? +### Why are we using Snappy for compression? Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, and can calculate the size of the uncompressed object without inflating it in memory. From 7d715220bb7128ea98597deedab3ae82ca5cc854 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 11 Feb 2021 06:39:42 +1100 Subject: [PATCH 089/140] Don't propagate blocks with a faulty slot (#2196) --- specs/phase0/p2p-interface.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a08a2d136..249d08799 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -313,6 +313,7 @@ The following validations MUST pass before forwarding the `signed_beacon_block` (via both gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is retrieved). - _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. +- _[REJECT]_ The block is from a higher slot than its parent. - _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. `get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) == store.finalized_checkpoint.root` From 2b8b0d9e2bcd301c012d09a6ce0d4680cd17ab6f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Feb 2021 14:53:29 -0700 Subject: [PATCH 090/140] update penalty config values for hf1 --- configs/mainnet/lightclient_patch.yaml | 10 +++ configs/minimal/lightclient_patch.yaml | 10 +++ specs/lightclient/beacon-chain.md | 77 ++++++++++++++++++- .../test_process_slashings.py | 48 ++++++++++-- 4 files changed, 134 insertions(+), 11 deletions(-) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index 64c05a720..6c5b16edf 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -2,6 +2,16 @@ CONFIG_NAME: "mainnet" +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24) (= 50,331,648) +HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +# 2**6 (= 64) +HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +# 2 +HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 + + # Misc # --------------------------------------------------------------- # 2**10 (=1,024) diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index afe7d897e..7ab5f34ba 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -2,6 +2,16 @@ CONFIG_NAME: "minimal" +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24) (= 50,331,648) +HF1_INACTIVITY_PENALTY_QUOTIENT: 50331648 +# 2**6 (= 64) +HF1_MIN_SLASHING_PENALTY_QUOTIENT: 64 +# 2 +HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 + + # Misc # --------------------------------------------------------------- # [customized] diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index fff8726bf..ccf9b2ebc 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -13,6 +13,7 @@ - [Participation rewards](#participation-rewards) - [Misc](#misc) - [Configuration](#configuration) + - [Updated penalty values](#updated-penalty-values) - [Misc](#misc-1) - [Time parameters](#time-parameters) - [Domain types](#domain-types) @@ -34,6 +35,8 @@ - [`get_unslashed_participating_indices`](#get_unslashed_participating_indices) - [`get_flag_deltas`](#get_flag_deltas) - [New `get_inactivity_penalty_deltas`](#new-get_inactivity_penalty_deltas) + - [Beacon state mutators](#beacon-state-mutators) + - [New `slash_validator`](#new-slash_validator) - [Block processing](#block-processing) - [New `process_attestation`](#new-process_attestation) - [New `process_deposit`](#new-process_deposit) @@ -41,6 +44,7 @@ - [Epoch processing](#epoch-processing) - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) + - [New `process_slashings`](#new-process_slashings) - [Sync committee updates](#sync-committee-updates) - [Participation flags updates](#participation-flags-updates) @@ -49,7 +53,8 @@ ## Introduction -This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. It has three main features: +This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. +It has three main features: * Light client support via sync committees * Incentive accounting reforms, reducing spec complexity @@ -97,6 +102,18 @@ The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewar ## Configuration +### Updated penalty values + +This patch updates a few configuration values to move penalty constants toward their final, maxmium security values. + +*Note*: The spec does *not* override previous configuration values but instead creates new values and replaces usage throughout. + +| Name | Value | +| - | - | +| `HF1_INACTIVITY_PENALTY_QUOTIENT` | `uint64(3 * 2**24)` (= 50,331,648) | +| `HF1_MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**6)` (=64) | +| `HF1_PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(2)` | + ### Misc | Name | Value | @@ -328,7 +345,8 @@ def get_flag_deltas(state: BeaconState, #### New `get_inactivity_penalty_deltas` -*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices and the removal of `BASE_REWARDS_PER_EPOCH`. +*Note*: The function `get_inactivity_penalty_deltas` is modified in the selection of matching target indices +and the removal of `BASE_REWARDS_PER_EPOCH`. ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: @@ -348,12 +366,47 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei( + effective_balance * get_finality_delay(state) + // HF1_INACTIVITY_PENALTY_QUOTIENT + ) rewards = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` +### Beacon state mutators + +#### New `slash_validator` + +*Note*: The function `slash_validator` is modified +with the substitution of `MIN_SLASHING_PENALTY_QUOTIENT` with `HF1_MIN_SLASHING_PENALTY_QUOTIENT`. + +```python +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: + """ + Slash the validator with index ``slashed_index``. + """ + epoch = get_current_epoch(state) + initiate_validator_exit(state, slashed_index) + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // HF1_MIN_SLASHING_PENALTY_QUOTIENT) + + # Apply proposer and whistleblower rewards + proposer_index = get_beacon_proposer_index(state) + if whistleblower_index is None: + whistleblower_index = proposer_index + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) +``` + ### Block processing ```python @@ -576,6 +629,24 @@ def process_rewards_and_penalties(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` +#### New `process_slashings` + +*Note*: The function `process_slashings` is modified +with the substitution of `PROPORTIONAL_SLASHING_MULTIPLIER` with `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. + +```python +def process_slashings(state: BeaconState) -> None: + epoch = get_current_epoch(state) + total_balance = get_total_active_balance(state) + adjusted_total_slashing_balance = min(sum(state.slashings) * HF1_PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + for index, validator in enumerate(state.validators): + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance + penalty = penalty_numerator // total_balance * increment + decrease_balance(state, ValidatorIndex(index), penalty) +``` + #### Sync committee updates ```python diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 2e09f5c8a..8bb4ac218 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_all_phases, is_post_lightclient_patch from eth2spec.test.helpers.epoch_processing import ( run_epoch_processing_with, run_epoch_processing_to ) @@ -23,12 +23,19 @@ def slash_validators(spec, state, indices, out_epochs): ] = total_slashed_balance +def get_slashing_multipler(spec): + if is_post_lightclient_patch(spec): + return spec.HF1_PROPORTIONAL_SLASHING_MULTIPLIER + else: + return spec.PROPORTIONAL_SLASHING_MULTIPLIER + + @with_all_phases @spec_state_test def test_max_penalties(spec, state): # Slashed count to ensure that enough validators are slashed to induce maximum penalties slashed_count = min( - (len(state.validators) // spec.PROPORTIONAL_SLASHING_MULTIPLIER) + 1, + (len(state.validators) // get_slashing_multipler(spec)) + 1, # Can't slash more than validator count! len(state.validators) ) @@ -40,7 +47,7 @@ def test_max_penalties(spec, state): total_balance = spec.get_total_active_balance(state) total_penalties = sum(state.slashings) - assert total_balance // spec.PROPORTIONAL_SLASHING_MULTIPLIER <= total_penalties + assert total_balance // get_slashing_multipler(spec) <= total_penalties yield from run_process_slashings(spec, state) @@ -50,7 +57,30 @@ def test_max_penalties(spec, state): @with_all_phases @spec_state_test -def test_small_penalty(spec, state): +def test_low_penalty(spec, state): + # Slashed count is one tenth of validator set + slashed_count = (len(state.validators) // 10) + 1 + out_epoch = spec.get_current_epoch(state) + (spec.EPOCHS_PER_SLASHINGS_VECTOR // 2) + + slashed_indices = list(range(slashed_count)) + slash_validators(spec, state, slashed_indices, [out_epoch] * slashed_count) + + pre_state = state.copy() + + yield from run_process_slashings(spec, state) + + for i in slashed_indices: + assert 0 < state.balances[i] < pre_state.balances[i] + + +@with_all_phases +@spec_state_test +def test_minimal_penalty(spec, state): + # + # When very few slashings, the resulting slashing penalty gets rounded down + # to zero so the result of `process_slashings` is null + # + # Just the bare minimum for this one validator state.balances[0] = state.validators[0].effective_balance = spec.EJECTION_BALANCE # All the other validators get the maximum. @@ -74,11 +104,13 @@ def test_small_penalty(spec, state): expected_penalty = ( state.validators[0].effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (3 * total_penalties) + * (get_slashing_multipler(spec) * total_penalties) // total_balance * spec.EFFECTIVE_BALANCE_INCREMENT ) - assert state.balances[0] == pre_slash_balances[0] - expected_penalty + + assert expected_penalty == 0 + assert state.balances[0] == pre_slash_balances[0] @with_all_phases @@ -96,7 +128,7 @@ def test_scaled_penalties(spec, state): state.slashings[5] = base + (incr * 6) state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) - slashed_count = len(state.validators) // (spec.PROPORTIONAL_SLASHING_MULTIPLIER + 1) + slashed_count = len(state.validators) // (get_slashing_multipler(spec) + 1) assert slashed_count > 10 @@ -134,7 +166,7 @@ def test_scaled_penalties(spec, state): v = state.validators[i] expected_penalty = ( v.effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (spec.PROPORTIONAL_SLASHING_MULTIPLIER * total_penalties) + * (get_slashing_multipler(spec) * total_penalties) // (total_balance) * spec.EFFECTIVE_BALANCE_INCREMENT ) From 24a244eb9b340fe707888180e02bcb3918cd7998 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Feb 2021 22:48:34 +0800 Subject: [PATCH 091/140] Fix typo: `get_slashing_multipler` -> `get_slashing_multiplier` --- .../epoch_processing/test_process_slashings.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py index 8bb4ac218..34f1e89c6 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_slashings.py @@ -23,7 +23,7 @@ def slash_validators(spec, state, indices, out_epochs): ] = total_slashed_balance -def get_slashing_multipler(spec): +def get_slashing_multiplier(spec): if is_post_lightclient_patch(spec): return spec.HF1_PROPORTIONAL_SLASHING_MULTIPLIER else: @@ -35,7 +35,7 @@ def get_slashing_multipler(spec): def test_max_penalties(spec, state): # Slashed count to ensure that enough validators are slashed to induce maximum penalties slashed_count = min( - (len(state.validators) // get_slashing_multipler(spec)) + 1, + (len(state.validators) // get_slashing_multiplier(spec)) + 1, # Can't slash more than validator count! len(state.validators) ) @@ -47,7 +47,7 @@ def test_max_penalties(spec, state): total_balance = spec.get_total_active_balance(state) total_penalties = sum(state.slashings) - assert total_balance // get_slashing_multipler(spec) <= total_penalties + assert total_balance // get_slashing_multiplier(spec) <= total_penalties yield from run_process_slashings(spec, state) @@ -104,7 +104,7 @@ def test_minimal_penalty(spec, state): expected_penalty = ( state.validators[0].effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (get_slashing_multipler(spec) * total_penalties) + * (get_slashing_multiplier(spec) * total_penalties) // total_balance * spec.EFFECTIVE_BALANCE_INCREMENT ) @@ -128,7 +128,7 @@ def test_scaled_penalties(spec, state): state.slashings[5] = base + (incr * 6) state.slashings[spec.EPOCHS_PER_SLASHINGS_VECTOR - 1] = base + (incr * 7) - slashed_count = len(state.validators) // (get_slashing_multipler(spec) + 1) + slashed_count = len(state.validators) // (get_slashing_multiplier(spec) + 1) assert slashed_count > 10 @@ -166,7 +166,7 @@ def test_scaled_penalties(spec, state): v = state.validators[i] expected_penalty = ( v.effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT - * (get_slashing_multipler(spec) * total_penalties) + * (get_slashing_multiplier(spec) * total_penalties) // (total_balance) * spec.EFFECTIVE_BALANCE_INCREMENT ) From dda7010c0c02f2bd740509e315b5985db7590f84 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 13 Feb 2021 23:02:06 +0800 Subject: [PATCH 092/140] Fix the tests that use `MIN_SLASHING_PENALTY_QUOTIENT`. (The mainnet tests failed before this fix) --- .../pyspec/eth2spec/test/helpers/proposer_slashings.py | 10 +++++++++- .../block_processing/test_process_attester_slashing.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py index 87b4f5ca0..89acb3417 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,8 +1,16 @@ +from eth2spec.test.context import is_post_lightclient_patch from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.state import get_balance +def get_min_slashing_penalty_quotient(spec): + if is_post_lightclient_patch(spec): + return spec.HF1_MIN_SLASHING_PENALTY_QUOTIENT + else: + return spec.MIN_SLASHING_PENALTY_QUOTIENT + + def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): slashed_validator = state.validators[slashed_index] assert slashed_validator.slashed @@ -10,7 +18,7 @@ def check_proposer_slashing_effect(spec, pre_state, state, slashed_index): assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH proposer_index = spec.get_beacon_proposer_index(state) - slash_penalty = state.validators[slashed_index].effective_balance // spec.MIN_SLASHING_PENALTY_QUOTIENT + slash_penalty = state.validators[slashed_index].effective_balance // get_min_slashing_penalty_quotient(spec) whistleblower_reward = state.validators[slashed_index].effective_balance // spec.WHISTLEBLOWER_REWARD_QUOTIENT if proposer_index != slashed_index: # slashed validator lost initial slash penalty diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py index 82d490311..21d9363f7 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_attester_slashing.py @@ -4,6 +4,7 @@ from eth2spec.test.context import ( from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, \ get_indexed_attestation_participants, get_attestation_2_data, get_attestation_1_data +from eth2spec.test.helpers.proposer_slashings import get_min_slashing_penalty_quotient from eth2spec.test.helpers.state import ( get_balance, next_epoch_via_block, @@ -70,7 +71,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) expected_balance = ( pre_proposer_balance + total_proposer_rewards - - pre_slashings[proposer_index] // spec.MIN_SLASHING_PENALTY_QUOTIENT + - pre_slashings[proposer_index] // get_min_slashing_penalty_quotient(spec) ) assert get_balance(state, proposer_index) == expected_balance From 600a4daddf1456778b71a86f9dad7e25d25aaf52 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Feb 2021 18:38:20 +0800 Subject: [PATCH 093/140] Turn off phase1 testgen and turn on lightclient_patch testgen --- tests/core/gen_helpers/gen_from_tests/gen.py | 12 +++++- tests/core/pyspec/eth2spec/test/context.py | 4 ++ .../test/lightclient_patch/sanity/__init__.py | 0 tests/generators/epoch_processing/main.py | 37 +++++++++++------- tests/generators/finality/main.py | 39 ++++++++++++------- tests/generators/operations/main.py | 37 +++++++++++------- tests/generators/rewards/main.py | 34 +++++++++------- tests/generators/sanity/main.py | 34 +++++++++------- 8 files changed, 126 insertions(+), 71 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py index 902b0954a..c09b477bb 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/gen_helpers/gen_from_tests/gen.py @@ -1,5 +1,5 @@ from inspect import getmembers, isfunction -from typing import Any, Iterable +from typing import Any, Iterable, Dict from gen_base.gen_typing import TestCase @@ -38,3 +38,13 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) ) + + +def get_provider(create_provider_fn, config_name, fork_name, all_mods): + for key, mod_name in all_mods[fork_name].items(): + yield create_provider_fn( + fork_name=fork_name, + handler_name=key, + tests_src_mod_name=mod_name, + config_name=config_name, + ) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index d19547477..75832794f 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -37,6 +37,10 @@ ALL_PHASES = (PHASE0, PHASE1, LIGHTCLIENT_PATCH) MAINNET = ConfigName('mainnet') MINIMAL = ConfigName('minimal') +ALL_CONFIGS = (MINIMAL, MAINNET) + +# The forks that output to the test vectors. +TESTGEN_FORKS = (PHASE0, LIGHTCLIENT_PATCH) # TODO: currently phases are defined as python modules. # It would be better if they would be more well-defined interfaces for stronger typing. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index ea7639605..835fe0087 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -44,21 +46,28 @@ if __name__ == "__main__": 'historical_roots_update', 'participation_record_updates', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.epoch_processing.test_process_' + key for key in [ + 'sync_committee_updates', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.epoch_processing.test_process_' + key for key in [ 'reveal_deadlines', 'challenge_deadlines', 'custody_final_updates', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"epoch_processing", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"epoch_processing", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index ef2d8293f..6ea3d82c8 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,27 +1,30 @@ from typing import Iterable -from importlib import reload +from importlib import reload, import_module from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider -from eth2spec.test.context import PHASE0, PHASE1 -from eth2spec.test.phase0.finality import test_finality +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 from eth2spec.utils import bls -def create_provider(fork_name: str, handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(fork_name: str, handler_name: str, + tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: + tests_src = import_module(tests_src_mod_name) return generate_from_tests( runner_name='finality', handler_name=handler_name, @@ -33,11 +36,21 @@ def create_provider(fork_name: str, handler_name: str, tests_src, config_name: s if __name__ == "__main__": - # No additional phase 1 specific rewards tests, yet. - key = 'finality' - gen_runner.run_generator("finality", [ - create_provider(PHASE0, 'finality', test_finality, 'minimal'), - create_provider(PHASE0, 'finality', test_finality, 'mainnet'), - create_provider(PHASE1, 'finality', test_finality, 'minimal'), - create_provider(PHASE1, 'finality', test_finality, 'mainnet'), - ]) + phase_0_mods = {'finality': 'eth2spec.test.phase0.finality.test_finality'} + # No additional lightclient_patch or phase 1 specific finality tests, yet. + lightclient_patch_mods = phase_0_mods + phase_1_mods = phase_0_mods + + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"finality", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 1acf45e47..20643f8d7 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -40,6 +42,12 @@ if __name__ == "__main__": 'proposer_slashing', 'voluntary_exit', ]} + lightclient_patch_mods = { + **{key: 'eth2spec.test.lightclient_patch.block_processing.test_process_' + key for key in [ + 'sync_committee', + ]}, + **phase_0_mods, + } # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.block_processing.test_process_' + key for key in [ 'attestation', 'chunk_challenge', @@ -49,15 +57,16 @@ if __name__ == "__main__": 'shard_transition', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"operations", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"operations", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 23d0633b0..cd3206a3f 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -37,18 +39,20 @@ if __name__ == "__main__": 'leak', 'random', ]} - # No additional phase 1 specific rewards tests, yet. + # No additional lightclient_patch or phase 1 specific rewards tests, yet. + lightclient_patch_mods = phase_0_mods phase_1_mods = phase_0_mods - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"rewards", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"rewards", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 83166f0cf..0aabf5d4c 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,12 +1,13 @@ from typing import Iterable from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from gen_from_tests.gen import generate_from_tests, get_provider from importlib import reload, import_module from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS from eth2spec.utils import bls @@ -15,6 +16,7 @@ def create_provider(fork_name: str, handler_name: str, def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) reload(spec_phase1) bls.use_milagro() return config_name @@ -36,20 +38,24 @@ if __name__ == "__main__": 'blocks', 'slots', ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ 'blocks', # more phase 1 specific block tests 'shard_blocks', ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'minimal') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE0, key, mod_name, 'mainnet') for key, mod_name in phase_0_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'minimal') for key, mod_name in phase_1_mods.items() - ]) - gen_runner.run_generator(f"sanity", [ - create_provider(PHASE1, key, mod_name, 'mainnet') for key, mod_name in phase_1_mods.items() - ]) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(f"sanity", get_provider( + create_provider_fn=create_provider, config_name=config_name, + fork_name=fork_name, all_mods=all_mods, + )) From c7d975981c04626b72cdc3566914af0aa552b7bb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Feb 2021 22:25:58 +0800 Subject: [PATCH 094/140] Refactor state tests generators --- tests/core/gen_helpers/gen_from_tests/gen.py | 53 ++++++++++++++++++-- tests/generators/epoch_processing/main.py | 39 ++------------ tests/generators/finality/main.py | 41 ++------------- tests/generators/operations/main.py | 39 ++------------ tests/generators/rewards/main.py | 39 ++------------ tests/generators/sanity/main.py | 39 ++------------ 6 files changed, 69 insertions(+), 181 deletions(-) diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py index c09b477bb..3ec8904fc 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/gen_helpers/gen_from_tests/gen.py @@ -1,11 +1,17 @@ +from importlib import reload, import_module from inspect import getmembers, isfunction -from typing import Any, Iterable, Dict +from typing import Any, Callable, Dict, Iterable -from gen_base.gen_typing import TestCase +from eth2spec.config import config_util +from eth2spec.utils import bls + +from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName +from gen_base import gen_runner +from gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: str, bls_active: bool = True) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. @@ -40,7 +46,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, ) -def get_provider(create_provider_fn, config_name, fork_name, all_mods): +def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigName], TestProvider], + config_name: ConfigName, + fork_name: SpecForkName, + all_mods: Dict[str, Dict[str, str]]) -> Iterable[TestProvider]: for key, mod_name in all_mods[fork_name].items(): yield create_provider_fn( fork_name=fork_name, @@ -48,3 +57,39 @@ def get_provider(create_provider_fn, config_name, fork_name, all_mods): tests_src_mod_name=mod_name, config_name=config_name, ) + + +def get_create_provider_fn(runner_name: str, config_name: ConfigName, specs: Iterable[Any] + ) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + for spec in specs: + reload(spec) + bls.use_milagro() + return config_name + + def create_provider(fork_name: SpecForkName, handler_name: str, + tests_src_mod_name: str, config_name: ConfigName) -> TestProvider: + def cases_fn() -> Iterable[TestCase]: + tests_src = import_module(tests_src_mod_name) + return generate_from_tests( + runner_name=runner_name, + handler_name=handler_name, + src=tests_src, + fork_name=fork_name, + ) + + return TestProvider(prepare=prepare_fn, make_cases=cases_fn) + return create_provider + + +def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: + for config_name in ALL_CONFIGS: + for fork_name in TESTGEN_FORKS: + if fork_name in all_mods: + gen_runner.run_generator(runner_name, get_provider( + create_provider_fn=get_create_provider_fn(runner_name, config_name, specs), + config_name=config_name, + fork_name=fork_name, + all_mods=all_mods, + )) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 835fe0087..1306da523 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -64,10 +39,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"epoch_processing", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="epoch_processing", specs=specs, all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 6ea3d82c8..0a62b1aa5 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,38 +1,11 @@ -from typing import Iterable -from importlib import reload, import_module - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider - -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='finality', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -47,10 +20,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"finality", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="finality", specs=specs, all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 20643f8d7..d40dbe9cd 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='operations', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -63,10 +38,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"operations", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="operations", specs=specs, all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index cd3206a3f..addb6aef0 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='rewards', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -49,10 +24,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"rewards", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="rewards", specs=specs, all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 0aabf5d4c..fc5227a53 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,36 +1,11 @@ -from typing import Iterable - -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests, get_provider -from importlib import reload, import_module -from eth2spec.config import config_util +from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, ALL_CONFIGS -from eth2spec.utils import bls +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH -def create_provider(fork_name: str, handler_name: str, - tests_src_mod_name: str, config_name: str) -> gen_typing.TestProvider: - def prepare_fn(configs_path: str) -> str: - config_util.prepare_config(configs_path, config_name) - reload(spec_phase0) - reload(spec_lightclient_patch) - reload(spec_phase1) - bls.use_milagro() - return config_name - - def cases_fn() -> Iterable[gen_typing.TestCase]: - tests_src = import_module(tests_src_mod_name) - return generate_from_tests( - runner_name='sanity', - handler_name=handler_name, - src=tests_src, - fork_name=fork_name, - ) - - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": @@ -52,10 +27,4 @@ if __name__ == "__main__": PHASE1: phase_1_mods, } - for config_name in ALL_CONFIGS: - for fork_name in TESTGEN_FORKS: - if fork_name in all_mods: - gen_runner.run_generator(f"sanity", get_provider( - create_provider_fn=create_provider, config_name=config_name, - fork_name=fork_name, all_mods=all_mods, - )) + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) From c9ba641800dfba8a4838d5030260b1b6e5c7efd4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Feb 2021 11:22:11 -0700 Subject: [PATCH 095/140] note penalties in hf1 list --- specs/lightclient/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index ccf9b2ebc..bf9b6a092 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -54,11 +54,12 @@ ## Introduction This is a patch implementing the first hard fork to the beacon chain, tentatively named HF1 pending a permanent name. -It has three main features: +It has four main features: * Light client support via sync committees * Incentive accounting reforms, reducing spec complexity and [TODO] reducing the cost of processing chains that have very little or zero participation for a long span of epochs +* Update penalty configuration values, moving them toward their planned maximally punitive configuration * Fork choice rule changes to address weaknesses recently discovered in the existing fork choice ## Custom types From 7050cb0add082f7a406a8d086f332a369b806fa7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Feb 2021 11:55:01 -0700 Subject: [PATCH 096/140] minor 0x01 PR feedback --- specs/phase0/validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5106c2b73..9e2ee7b1f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -125,14 +125,16 @@ The `withdrawal_credentials` field must be such that: Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. + The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) -triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +After the merge of eth1 into eth2, +withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by a user transaction that will set the gas price and gas limit as well pay fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 9cc8567d682ee831adf4a019de31fd0ebbdd30bb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 Feb 2021 15:17:47 +0800 Subject: [PATCH 097/140] Move `gen_helpers` into a module of `eth2spec` package --- .../{ => pyspec/eth2spec}/gen_helpers/README.md | 0 .../eth2spec/gen_helpers}/__init__.py | 0 .../eth2spec/gen_helpers/gen_base}/__init__.py | 0 .../eth2spec}/gen_helpers/gen_base/gen_runner.py | 13 +++++++------ .../eth2spec}/gen_helpers/gen_base/gen_typing.py | 0 .../eth2spec/gen_helpers/gen_from_tests/__init__.py | 0 .../eth2spec}/gen_helpers/gen_from_tests/gen.py | 13 +++++++------ .../eth2spec}/gen_helpers/requirements.txt | 0 .../core/{ => pyspec/eth2spec}/gen_helpers/setup.py | 0 tests/generators/README.md | 2 +- tests/generators/bls/main.py | 2 +- tests/generators/bls/requirements.txt | 4 +--- tests/generators/epoch_processing/main.py | 2 +- tests/generators/epoch_processing/requirements.txt | 4 ++-- tests/generators/finality/main.py | 2 +- tests/generators/finality/requirements.txt | 4 ++-- tests/generators/genesis/main.py | 4 ++-- tests/generators/genesis/requirements.txt | 4 ++-- tests/generators/operations/main.py | 2 +- tests/generators/operations/requirements.txt | 5 ++--- tests/generators/rewards/main.py | 2 +- tests/generators/rewards/requirements.txt | 4 ++-- tests/generators/sanity/main.py | 3 ++- tests/generators/sanity/requirements.txt | 4 ++-- tests/generators/shuffling/main.py | 2 +- tests/generators/ssz_generic/main.py | 2 +- tests/generators/ssz_generic/requirements.txt | 3 +-- tests/generators/ssz_static/main.py | 2 +- tests/generators/ssz_static/requirements.txt | 4 ++-- 29 files changed, 43 insertions(+), 44 deletions(-) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/README.md (100%) rename tests/core/{gen_helpers/gen_base => pyspec/eth2spec/gen_helpers}/__init__.py (100%) rename tests/core/{gen_helpers/gen_from_tests => pyspec/eth2spec/gen_helpers/gen_base}/__init__.py (100%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_base/gen_runner.py (95%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_base/gen_typing.py (100%) create mode 100644 tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py rename tests/core/{ => pyspec/eth2spec}/gen_helpers/gen_from_tests/gen.py (92%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/requirements.txt (100%) rename tests/core/{ => pyspec/eth2spec}/gen_helpers/setup.py (100%) diff --git a/tests/core/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md similarity index 100% rename from tests/core/gen_helpers/README.md rename to tests/core/pyspec/eth2spec/gen_helpers/README.md diff --git a/tests/core/gen_helpers/gen_base/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_base/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/__init__.py diff --git a/tests/core/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py similarity index 100% rename from tests/core/gen_helpers/gen_from_tests/__init__.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/__init__.py diff --git a/tests/core/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py similarity index 95% rename from tests/core/gen_helpers/gen_base/gen_runner.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index a22073c00..26427c6f8 100644 --- a/tests/core/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -8,11 +8,11 @@ from ruamel.yaml import ( YAML, ) -from gen_base.gen_typing import TestProvider - from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest +from .gen_typing import TestProvider + # Flag that the runner does NOT run test via pytest context.is_pytest = False @@ -119,10 +119,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f"generating tests with config '{config_name}' ...") for test_case in tprov.make_cases(): - case_dir = Path(output_dir) / Path(config_name) / Path(test_case.fork_name) \ - / Path(test_case.runner_name) / Path(test_case.handler_name) \ - / Path(test_case.suite_name) / Path(test_case.case_name) - + case_dir = ( + Path(output_dir) / Path(config_name) / Path(test_case.fork_name) + / Path(test_case.runner_name) / Path(test_case.handler_name) + / Path(test_case.suite_name) / Path(test_case.case_name) + ) if case_dir.exists(): if not args.force: print(f'Skipping already existing test: {case_dir}') diff --git a/tests/core/gen_helpers/gen_base/gen_typing.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py similarity index 100% rename from tests/core/gen_helpers/gen_base/gen_typing.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_typing.py diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py similarity index 92% rename from tests/core/gen_helpers/gen_from_tests/gen.py rename to tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 3ec8904fc..f5a0f378b 100644 --- a/tests/core/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -4,10 +4,10 @@ from typing import Any, Callable, Dict, Iterable from eth2spec.config import config_util from eth2spec.utils import bls - from eth2spec.test.context import ALL_CONFIGS, TESTGEN_FORKS, SpecForkName, ConfigName -from gen_base import gen_runner -from gen_base.gen_typing import TestCase, TestProvider + +from eth2spec.gen_helpers.gen_base import gen_runner +from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, @@ -56,11 +56,12 @@ def get_provider(create_provider_fn: Callable[[SpecForkName, str, str, ConfigNam handler_name=key, tests_src_mod_name=mod_name, config_name=config_name, - ) + ) -def get_create_provider_fn(runner_name: str, config_name: ConfigName, specs: Iterable[Any] - ) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: +def get_create_provider_fn( + runner_name: str, config_name: ConfigName, specs: Iterable[Any] +) -> Callable[[SpecForkName, str, str, ConfigName], TestProvider]: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) for spec in specs: diff --git a/tests/core/gen_helpers/requirements.txt b/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt similarity index 100% rename from tests/core/gen_helpers/requirements.txt rename to tests/core/pyspec/eth2spec/gen_helpers/requirements.txt diff --git a/tests/core/gen_helpers/setup.py b/tests/core/pyspec/eth2spec/gen_helpers/setup.py similarity index 100% rename from tests/core/gen_helpers/setup.py rename to tests/core/pyspec/eth2spec/gen_helpers/setup.py diff --git a/tests/generators/README.md b/tests/generators/README.md index 077a8443c..b819d93b8 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -103,7 +103,7 @@ Write a `main.py` file. The shuffling test generator is a good minimal starting ```python from eth2spec.phase0 import spec as spec from eth_utils import to_tuple -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from preset_loader import loader from typing import Iterable diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 6552b8654..fecc2df7d 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -13,7 +13,7 @@ import milagro_bls_binding as milagro_bls from eth2spec.utils import bls from eth2spec.test.context import PHASE0 -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing def to_bytes(i): diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 5f830773a..df386450b 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,2 @@ -py_ecc==5.1.0 -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 1306da523..50a1e2b57 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/epoch_processing/requirements.txt +++ b/tests/generators/epoch_processing/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 0a62b1aa5..8b961f9f4 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/finality/requirements.txt b/tests/generators/finality/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/finality/requirements.txt +++ b/tests/generators/finality/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index ce055b44a..854af6572 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -3,8 +3,8 @@ from typing import Iterable from eth2spec.test.context import PHASE0 from eth2spec.test.phase0.genesis import test_initialization, test_validity -from gen_base import gen_runner, gen_typing -from gen_from_tests.gen import generate_from_tests +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests from eth2spec.phase0 import spec as spec from importlib import reload from eth2spec.config import config_util diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/genesis/requirements.txt +++ b/tests/generators/genesis/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index d40dbe9cd..00a58288f 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/operations/requirements.txt +++ b/tests/generators/operations/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index addb6aef0..4124587cc 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,4 +1,4 @@ -from gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 diff --git a/tests/generators/rewards/requirements.txt b/tests/generators/rewards/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/rewards/requirements.txt +++ b/tests/generators/rewards/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index fc5227a53..5155798ff 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,9 +1,10 @@ -from gen_from_tests.gen import run_state_test_generators from eth2spec.phase0 import spec as spec_phase0 from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + specs = (spec_phase0, spec_lightclient_patch, spec_phase1) diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/sanity/requirements.txt +++ b/tests/generators/sanity/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index 6069de77a..091207bca 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -2,7 +2,7 @@ from eth_utils import to_tuple from typing import Iterable from importlib import reload -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.config import config_util from eth2spec.phase0 import spec as spec diff --git a/tests/generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py index 8cfb2e3eb..737f8cda6 100644 --- a/tests/generators/ssz_generic/main.py +++ b/tests/generators/ssz_generic/main.py @@ -1,5 +1,5 @@ from typing import Iterable -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing import ssz_basic_vector import ssz_bitlist import ssz_bitvector diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt index af061a3b1..df386450b 100644 --- a/tests/generators/ssz_generic/requirements.txt +++ b/tests/generators/ssz_generic/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers +pytest>=4.4 ../../../ diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 38ff18615..8b18705e7 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -3,7 +3,7 @@ from typing import Iterable from importlib import reload from inspect import getmembers, isclass -from gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.debug import random_value, encode from eth2spec.config import config_util diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt index b82314298..df386450b 100644 --- a/tests/generators/ssz_static/requirements.txt +++ b/tests/generators/ssz_static/requirements.txt @@ -1,2 +1,2 @@ -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ From e58dcb40acc4348047e681fc069b1c7abb33fa16 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 Feb 2021 17:51:01 +0800 Subject: [PATCH 098/140] Clean up and kick the cache --- setup.py | 3 +-- .../core/pyspec/eth2spec/gen_helpers/requirements.txt | 3 --- tests/core/pyspec/eth2spec/gen_helpers/setup.py | 11 ----------- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/gen_helpers/requirements.txt delete mode 100644 tests/core/pyspec/eth2spec/gen_helpers/setup.py diff --git a/setup.py b/setup.py index 6b8520075..9c2696006 100644 --- a/setup.py +++ b/setup.py @@ -562,13 +562,12 @@ setup( url="https://github.com/ethereum/eth2.0-specs", include_package_data=False, package_data={'configs': ['*.yaml'], - 'specs': ['**/*.md'], 'eth2spec': ['VERSION.txt']}, package_dir={ "eth2spec": "tests/core/pyspec/eth2spec", "configs": "configs", - "specs": "specs" + "specs": "specs", }, packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'], py_modules=["eth2spec"], diff --git a/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt b/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt deleted file mode 100644 index e7cdd30ea..000000000 --- a/tests/core/pyspec/eth2spec/gen_helpers/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -ruamel.yaml==0.16.5 -eth-utils==1.6.0 -pytest>=4.4 diff --git a/tests/core/pyspec/eth2spec/gen_helpers/setup.py b/tests/core/pyspec/eth2spec/gen_helpers/setup.py deleted file mode 100644 index e9fc1a787..000000000 --- a/tests/core/pyspec/eth2spec/gen_helpers/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from distutils.core import setup - -setup( - name='gen_helpers', - packages=['gen_base', 'gen_from_tests'], - install_requires=[ - "ruamel.yaml==0.16.5", - "eth-utils==1.6.0", - "pytest>=4.4", - ] -) From a28b2d73493f8a978c6afec967156a752fe87fbd Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 20 Feb 2021 15:29:34 +0000 Subject: [PATCH 099/140] Bump Milagro dependency for M1 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b8520075..df95d99d3 100644 --- a/setup.py +++ b/setup.py @@ -583,7 +583,7 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==5.1.0", - "milagro_bls_binding==1.6.2", + "milagro_bls_binding==1.6.3", "dataclasses==0.6", "remerkleable==0.1.18", "ruamel.yaml==0.16.5", From de4cad5d3596abbcb0f3bb74d9cd478933b89f2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Feb 2021 18:29:31 +0800 Subject: [PATCH 100/140] Update docs --- .../pyspec/eth2spec/gen_helpers/README.md | 2 +- .../gen_helpers/gen_from_tests/gen.py | 3 + tests/generators/README.md | 57 ++++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/README.md b/tests/core/pyspec/eth2spec/gen_helpers/README.md index 20b48db83..d39ee66ae 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/README.md +++ b/tests/core/pyspec/eth2spec/gen_helpers/README.md @@ -4,7 +4,7 @@ A util to quickly write new test suite generators with. -See [Generators documentation](../../generators/README.md) for integration details. +See [Generators documentation](../../../../generators/README.md) for integration details. Options: diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index f5a0f378b..67d29b194 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -85,6 +85,9 @@ def get_create_provider_fn( def run_state_test_generators(runner_name: str, specs: Iterable[Any], all_mods: Dict[str, Dict[str, str]]) -> None: + """ + Generate all available state tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. + """ for config_name in ALL_CONFIGS: for fork_name in TESTGEN_FORKS: if fork_name in all_mods: diff --git a/tests/generators/README.md b/tests/generators/README.md index b819d93b8..26094bbae 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -78,9 +78,8 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -../../core/gen_helpers -../../core/config_helpers -../../core/pyspec +pytest>=4.4 +../../../ ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. @@ -163,35 +162,40 @@ To extend this, one could decide to parametrize the `shuffling_test_cases` funct Another example, to generate tests from pytests: ```python -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH - def prepare_fn(configs_path: str) -> str: - presets = loader.load_presets(configs_path, config_name) - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) - return config_name +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators - def cases_fn() -> Iterable[gen_typing.TestCase]: - return generate_from_tests( - runner_name='epoch_processing', - handler_name=handler_name, - src=tests_src, - fork_name='phase0' - ) - return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) +specs = (spec_phase0, spec_lightclient_patch, spec_phase1) if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('justification_and_finalization', test_process_justification_and_finalization, 'minimal'), - ... - ]) + phase_0_mods = {key: 'eth2spec.test.phase0.sanity.test_' + key for key in [ + 'blocks', + 'slots', + ]} + lightclient_patch_mods = {**{key: 'eth2spec.test.lightclient_patch.sanity.test_' + key for key in [ + 'blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests + phase_1_mods = {**{key: 'eth2spec.test.phase1.sanity.test_' + key for key in [ + 'blocks', # more phase 1 specific block tests + 'shard_blocks', + ]}, **phase_0_mods} # also run the previous phase 0 tests (but against phase 1 spec) + all_mods = { + PHASE0: phase_0_mods, + LIGHTCLIENT_PATCH: lightclient_patch_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="sanity", specs=specs, all_mods=all_mods) ``` -Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `generate_from_tests` utility. - +Here multiple phases load the configuration, and the stream of test cases is derived from a pytest file using the `eth2spec.gen_helpers.gen_from_tests.gen.run_state_test_generators` utility. Note that this helper generates all available tests of `TESTGEN_FORKS` forks of `ALL_CONFIGS` configs of the given runner. Recommendations: - You can have more than just one test provider. @@ -200,8 +204,7 @@ Recommendations: - Use config `minimal` for performance and simplicity, but also implement a suite with the `mainnet` config where necessary. - You may be able to write your test case provider in a way where it does not make assumptions on constants. If so, you can generate test cases with different configurations for the same scenario (see example). -- See [`tests/core/gen_helpers/README.md`](../core/gen_helpers/README.md) for command line options for generators. - +- See [`tests/core/gen_helpers/README.md`](../core/pyspec/eth2spec/gen_helpers/README.md) for command line options for generators. ## How to add a new test generator @@ -216,8 +219,8 @@ To add a new test generator that builds `New Tests`: 4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. The base generator helps you handle this; you only have to define test case providers. 5. Finally, add any linting or testing commands to the - [circleci config file](../.circleci/config.yml) if desired to increase code quality. - Or add it to the [`Makefile`](../Makefile), if it can be run locally. + [circleci config file](../../.circleci/config.yml) if desired to increase code quality. + Or add it to the [`Makefile`](../../Makefile), if it can be run locally. *Note*: You do not have to change the makefile. However, if necessary (e.g. not using Python, or mixing in other languages), submit an issue, and it can be a special case. From f6b8171350f702501c997abf9bd837489b89dfb6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Feb 2021 19:45:10 +0800 Subject: [PATCH 101/140] Update shuffling generator requirements.txt --- tests/generators/shuffling/requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt index a6ea61aea..df386450b 100644 --- a/tests/generators/shuffling/requirements.txt +++ b/tests/generators/shuffling/requirements.txt @@ -1,3 +1,2 @@ -eth-utils==1.6.0 -../../core/gen_helpers -../../../ \ No newline at end of file +pytest>=4.4 +../../../ From 9f634dc6f6d269339d8e5fd7d069286fa75c2687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 09:56:38 +0100 Subject: [PATCH 102/140] Update test generation docs --- tests/generators/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/generators/README.md b/tests/generators/README.md index 26094bbae..7d0b5240c 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -36,7 +36,7 @@ Prerequisites: ### Cleaning -This removes the existing virtual environments (`/test_generators//venv`) and generated tests (`/yaml_tests/`). +This removes the existing virtual environments (`/tests/generators//venv`) and generated tests (`../eth2.0-spec-tests/tests`). ```bash make clean @@ -47,7 +47,7 @@ make clean This runs all of the generators. ```bash -make -j 4 gen_yaml_tests +make -j 4 generate_tests ``` The `-j N` flag makes the generators run in parallel, with `N` being the amount of cores. @@ -55,7 +55,7 @@ The `-j N` flag makes the generators run in parallel, with `N` being the amount ### Running a single generator -The makefile auto-detects generators in the `test_generators` directory and provides a tests-gen target for each generator. See example: +The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target for each generator. See example: ```bash make ./eth2.0-spec-tests/tests/shuffling/ @@ -210,7 +210,7 @@ Recommendations: To add a new test generator that builds `New Tests`: -1. Create a new directory `new_tests` within the `test_generators` directory. +1. Create a new directory `new_tests` within the `tests/generators` directory. Note that `new_tests` is also the name of the directory in which the tests will appear in the tests repository later. 2. Your generator is assumed to have a `requirements.txt` file, with any dependencies it may need. Leave it empty if your generator has none. From e63754e6293b73cf056c3e5edc54569dba35439e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 13:23:32 +0100 Subject: [PATCH 103/140] change how generator targets are invoked --- tests/generators/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/generators/README.md b/tests/generators/README.md index 7d0b5240c..0f1ed478f 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -55,10 +55,10 @@ The `-j N` flag makes the generators run in parallel, with `N` being the amount ### Running a single generator -The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target for each generator. See example: +The makefile auto-detects generators in the `tests/generators` directory and provides a tests-gen target (gen_) for each generator. See example: ```bash -make ./eth2.0-spec-tests/tests/shuffling/ +make gen_ssz_static ``` ## Developing a generator From 1d4f467516436d0eb9d97692aef2fcd891f79a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 13:30:26 +0100 Subject: [PATCH 104/140] disable phase1/enable lightclient ssz static types test --- tests/generators/ssz_static/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 8b18705e7..854961ae4 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -9,7 +9,8 @@ from eth2spec.debug import random_value, encode from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.context import PHASE0, PHASE1 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -64,6 +65,7 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R config_util.prepare_config(configs_path, config_name) reload(spec_phase0) reload(spec_phase1) + reload(spec_lightclient_patch) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: @@ -71,6 +73,8 @@ def create_provider(fork_name, config_name: str, seed: int, mode: random_value.R spec = spec_phase0 if fork_name == PHASE1: spec = spec_phase1 + if fork_name == LIGHTCLIENT_PATCH: + spec = spec_lightclient_patch for (i, (name, ssz_type)) in enumerate(get_spec_ssz_types(spec)): yield from ssz_static_cases(fork_name, seed * 1000 + i, name, ssz_type, mode, chaos, count) @@ -90,7 +94,7 @@ if __name__ == "__main__": settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - for fork in [PHASE0, PHASE1]: + for fork in [PHASE0, LIGHTCLIENT_PATCH]: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From 993bcdf082dbdbf2103bae608d409f54f75b3088 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Feb 2021 21:17:12 +0800 Subject: [PATCH 105/140] Use constants --- tests/generators/ssz_static/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index 854961ae4..fb8635fa0 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -10,7 +10,7 @@ from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.lightclient_patch import spec as spec_lightclient_patch -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, @@ -87,14 +87,14 @@ if __name__ == "__main__": settings = [] seed = 1 for mode in random_value.RandomizationMode: - settings.append((seed, "minimal", mode, False, 30)) + settings.append((seed, MINIMAL, mode, False, 30)) seed += 1 - settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) + settings.append((seed, MINIMAL, random_value.RandomizationMode.mode_random, True, 30)) seed += 1 - settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + settings.append((seed, MAINNET, random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - for fork in [PHASE0, LIGHTCLIENT_PATCH]: + for fork in TESTGEN_FORKS: gen_runner.run_generator("ssz_static", [ create_provider(fork, config_name, seed, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From d28ec2fc75dc744d4d41201c2554b728c8e1d29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marin=20Petruni=C4=87?= Date: Tue, 23 Feb 2021 14:46:53 +0100 Subject: [PATCH 106/140] fix lint --- tests/generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/ssz_static/main.py b/tests/generators/ssz_static/main.py index fb8635fa0..e21fc4141 100644 --- a/tests/generators/ssz_static/main.py +++ b/tests/generators/ssz_static/main.py @@ -10,7 +10,7 @@ from eth2spec.config import config_util from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.lightclient_patch import spec as spec_lightclient_patch -from eth2spec.test.context import PHASE0, PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET +from eth2spec.test.context import PHASE1, LIGHTCLIENT_PATCH, TESTGEN_FORKS, MINIMAL, MAINNET from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, From 7af50cc8273597123ec7d92ac5a3a00a568e4391 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 11 Dec 2020 14:46:16 -0700 Subject: [PATCH 107/140] add eth1 withdrawal credentials to spec --- specs/phase0/beacon-chain.md | 5 +++ specs/phase0/deposit-contract.md | 13 ++++-- specs/phase0/validator.md | 42 +++++++++++++++--- .../block_processing/test_process_deposit.py | 43 +++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index fcd681115..1e9357610 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -14,6 +14,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) + - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -210,7 +211,11 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | + +### Validator withdrawal credential versions + | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | +| `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | ### Time parameters diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 8b3035629..429ebf238 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -59,12 +59,17 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: +One of the `DepositData` fields is `withdrawal_credentials`. -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey +This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). +The first byte of `withdrawal_credentials` is a version number. The remaining +bytes are content specific to the version. -The private key corresponding to `withdrawal_pubkey` will be required to initiate a withdrawal. It can be stored separately until a withdrawal is required, e.g. in cold storage. +Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` +versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). + +*Note*: The deposit contract does *not* validate withdrawal credentials. +Thus, new versions can be added without modifications of the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index c6a91dcfc..32a1d1010 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -16,7 +16,9 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - [BLS public key](#bls-public-key) - - [BLS withdrawal key](#bls-withdrawal-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [BLS key credentials](#bls-key-credentials) + - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) @@ -101,14 +103,42 @@ A validator must initialize many parameters locally before submitting a deposit Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BLS12-381 curve](https://z.cash/blog/new-snark-curve). A private key, `privkey`, must be securely generated along with the resultant `pubkey`. This `privkey` must be "hot", that is, constantly available to sign data throughout the lifetime of the validator. -#### BLS withdrawal key +#### Withdrawal credentials -A secondary withdrawal private key, `withdrawal_privkey`, must also be securely generated along with the resultant `withdrawal_pubkey`. This `withdrawal_privkey` does not have to be available for signing during the normal lifetime of a validator and can live in "cold storage". +The `withdrawal_credentials` ultimately control the deposited ETH once the +validator has exited and is withdrawable. The 0th byte of this 32-byte field, +called the "withdrawal prefix" defines the withdrawal credential version +while the remaining 31 bytes define the version-specific content. -The validator constructs their `withdrawal_credentials` via the following: +The following withdrawal credentials versions are currently supported. The +validator must choose and construct a version for the `withdrawal_credentials` field. -* Set `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]`. +##### BLS key credentials + +BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. +This key is securely generated along with the resultant `bls_withdrawal_pubkey`. +This `withdrawal_privkey` does not have to be available for signing during +the normal lifetime of a validator and can live in "cold storage". + +The validator constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. + +##### Eth1 address credentials + +Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. + +The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) +triggered by an eth1 transaction that will handle gas price/limit and payment of fees. + +As long as such a withdrawal account/contract can receive ETH transfers, +the future withdrawal protocol is agnostic to all other implementation details. + +The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: +* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. + +*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. ### Submit deposit diff --git a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py index b7a0de6c8..36e76f46c 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py +++ b/tests/core/pyspec/eth2spec/test/phase0/block_processing/test_process_deposit.py @@ -94,6 +94,49 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) +@with_all_phases +@spec_state_test +def test_new_deposit_eth1_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + + b'\x00' * 11 # specified 0s + + b'\x59' * 20 # a 20-byte eth1 address + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + +@with_all_phases +@spec_state_test +def test_new_deposit_non_versioned_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + b'\xFF' # Non specified withdrawal credentials version + + b'\x02' * 31 # Garabage bytes + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit( + spec, state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_processing(spec, state, deposit, validator_index) + + @with_all_phases @spec_state_test @always_bls From 128efdd34da71a146c4e366d4b0970a83fb01a09 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:38:40 +0000 Subject: [PATCH 108/140] Fix table for withdrawal credentials prefixes Minor cosmetic fixes (misformated table, section title). --- specs/phase0/beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1e9357610..a51c503eb 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -14,7 +14,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Validator withdrawal credential versions](#validator-withdrawal-credential-versions) + - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -212,8 +212,10 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Validator withdrawal credential versions +### Withdrawal credentials prefixes +| Name | Value | +| - | - | | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | | `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | From 47ebf438b349175a68f26fca7e889643bae818d7 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:40:59 +0000 Subject: [PATCH 109/140] Cleaner section title "Withdrawal prefixes" matches `[BLS]/[ETH1_ADDRESS]_WITHDRAWAL_PREFIX` --- specs/phase0/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index a51c503eb..084107340 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -14,7 +14,7 @@ - [Misc](#misc) - [Gwei values](#gwei-values) - [Initial values](#initial-values) - - [Withdrawal credentials prefixes](#withdrawal-credentials-prefixes) + - [Withdrawal prefixes](#withdrawal-prefixes) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) @@ -212,7 +212,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | -### Withdrawal credentials prefixes +### Withdrawal prefixes | Name | Value | | - | - | From e93f1e1fa8c54bc0f9d7ede5200c92c4a68badf9 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 20:50:10 +0000 Subject: [PATCH 110/140] Copy-edit deposit-contract.md --- specs/phase0/deposit-contract.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 429ebf238..3633ac3ae 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -61,15 +61,12 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac One of the `DepositData` fields is `withdrawal_credentials`. -This field is a commitment to credentials for withdrawing validator balance (e.g. to another validator, or to shards). -The first byte of `withdrawal_credentials` is a version number. The remaining -bytes are content specific to the version. +This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. +The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. -Currently, `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -versioned withdrawal credentials are supported. Read more in the [validator guide](./validator.md). +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). -*Note*: The deposit contract does *not* validate withdrawal credentials. -Thus, new versions can be added without modifications of the deposit contract. +*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. #### `DepositEvent` log From 0f94fa51b894de331a0e4370a692658294517d56 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:46:39 +0000 Subject: [PATCH 111/140] Update validator.md --- specs/phase0/validator.md | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 32a1d1010..e26f51819 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -105,40 +105,28 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` ultimately control the deposited ETH once the -validator has exited and is withdrawable. The 0th byte of this 32-byte field, -called the "withdrawal prefix" defines the withdrawal credential version -while the remaining 31 bytes define the version-specific content. +The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. -The following withdrawal credentials versions are currently supported. The -validator must choose and construct a version for the `withdrawal_credentials` field. +##### `BLS_WITHDRAWAL_PREFIX` -##### BLS key credentials +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: -BLS withdrawal credentials are controlled by a secondary withdrawal BLS private key, `bls_withdrawal_privkey`. -This key is securely generated along with the resultant `bls_withdrawal_pubkey`. -This `withdrawal_privkey` does not have to be available for signing during -the normal lifetime of a validator and can live in "cold storage". +* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -The validator constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = BLS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[1:] = hash(bls_withdrawal_pubkey)[1:]`. +*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. -##### Eth1 address credentials +##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Eth1 address credentials are controlled by an eth1 address. This can be an either an externally owned account or a contract. +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: -The withdrawal to the address specified will be a normal ETH transfer (with no payload other than the validator's ETH) -triggered by an eth1 transaction that will handle gas price/limit and payment of fees. +* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == Bytes32()[1:12]` +* `withdrawal_credentials[12:] = eth1_withdrawal_address` -As long as such a withdrawal account/contract can receive ETH transfers, -the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. -The validator selects a 20-byte eth1 address, `eth1_withdrawal_address`, and constructs `withdrawal_credentials` as the following: -* Set `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX`. -* Set `withdrawal_credentials[12:] = eth1_withdrawal_address`. - -*Note*: Bytes `withdrawal_credentials[1:12]` remain the default `0x00` value. +**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. ### Submit deposit From 80613a99bda9fc87f20da98efc49984afad677ea Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:51:21 +0000 Subject: [PATCH 112/140] Update validator.md --- specs/phase0/validator.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index e26f51819..e2d8967e5 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -15,8 +15,8 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [BLS key credentials](#bls-key-credentials) - [Eth1 address credentials](#eth1-address-credentials) - [Submit deposit](#submit-deposit) @@ -105,7 +105,7 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field specifies how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` @@ -114,7 +114,7 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` -*Note*: The `bls_withdrawal_pubkey` is not required for validating and can be kept in cold storage. +*Note*: The `bls_withdrawal_privkey` is not required for validating and can be kept in cold storage. ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` @@ -124,7 +124,7 @@ Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte * `withdrawal_credentials[1:12] == Bytes32()[1:12]` * `withdrawal_credentials[12:] = eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle gas price/limit and payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. **TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. From 1f8ca7179fd82e18f6644fbdaec5f1f930827e93 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:56:30 +0000 Subject: [PATCH 113/140] Update validator.md --- specs/phase0/validator.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index e2d8967e5..780f313c7 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -105,11 +105,11 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field contrains how a validator's withdrawable balance may be withdrawn. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -118,11 +118,11 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of an externally owned account or of a contract. The `withdrawal_credentials` field must be constructed such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: -* `withdrawal_credentials[:1] = ETH1_ADDRESS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:12] == Bytes32()[1:12]` -* `withdrawal_credentials[12:] = eth1_withdrawal_address` +* `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == b'\x00' * 11` +* `withdrawal_credentials[12:] == eth1_withdrawal_address` Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 809fc7afaffaad7d31c9b3761c0f35226098da59 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 21:57:58 +0000 Subject: [PATCH 114/140] Update validator.md --- specs/phase0/validator.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 780f313c7..0fd254321 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -15,10 +15,10 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - [Misc](#misc) - [Becoming a validator](#becoming-a-validator) - [Initialization](#initialization) - - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) - - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - - [BLS key credentials](#bls-key-credentials) - - [Eth1 address credentials](#eth1-address-credentials) + - [BLS public key](#bls-public-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - [Submit deposit](#submit-deposit) - [Process deposit](#process-deposit) - [Validator index](#validator-index) From 5992e8ff050855b532fa7d2248fc1b825ef65b2e Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 12 Dec 2020 22:02:56 +0000 Subject: [PATCH 115/140] Update deposit-contract.md --- specs/phase0/deposit-contract.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 3633ac3ae..a953b1781 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -59,14 +59,9 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). -This field is a commitment to credentials for withdrawing validator balance, e.g. to another validator, to an Ethereum 1.0 address, or to a shard. -The first byte of `withdrawal_credentials` is a withdrawal prefix which specifies the withdrawal type. The remaining 31 bytes are specific to the withdrawal prefix. - -The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md). - -*Note*: The deposit contract does not validate withdrawal credentials. Support for new withdrawal types can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log From a0ae04839550e84459da083b35a578558f78cf68 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 14 Dec 2020 13:09:25 -0700 Subject: [PATCH 116/140] copy edits --- specs/phase0/deposit-contract.md | 8 ++++++-- specs/phase0/validator.md | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index a953b1781..4c340d688 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -59,9 +59,13 @@ The amount of ETH (rounded down to the closest Gwei) sent to the deposit contrac #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. Read more in the [validator guide](./validator.md#withdrawal-credentials). +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +Read more in the [validator guide](./validator.md#withdrawal-credentials). -*Note*: The deposit contract does not validate the `withdrawal_credentials` field. Support for new withdrawal prefixes can be added without modifying the deposit contract. +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. +Support for new withdrawal prefixes can be added without modifying the deposit contract. #### `DepositEvent` log diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 0fd254321..6f8d36b93 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -105,11 +105,16 @@ Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BL #### Withdrawal credentials -The `withdrawal_credentials` field constrains validator withdrawals. The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. The following withdrawal prefixes are currently supported. +The `withdrawal_credentials` field constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. + +The following withdrawal prefixes are currently supported. ##### `BLS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair +`(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` @@ -118,15 +123,19 @@ Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair `(bls ##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` -Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. The `withdrawal_credentials` field must be such that: +Withdrawal credentials with the Eth1 address withdrawal prefix specify +a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. +The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. +The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. - -**TODO**: Be explicit about the endianness of `eth1_withdrawal_address`. +Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers +the future withdrawal protocol is agnostic to all other implementation details. ### Submit deposit From 1f7e9fabf24f02d5e178b90ab3f9c27e7c57029c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 16 Feb 2021 11:55:01 -0700 Subject: [PATCH 117/140] minor 0x01 PR feedback --- specs/phase0/validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 6f8d36b93..15be3ee70 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -126,14 +126,16 @@ The `withdrawal_credentials` field must be such that: Withdrawal credentials with the Eth1 address withdrawal prefix specify a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. + The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -Withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) -triggered by an Eth1 transaction that will handle the gas price and gas limit, as well the payment of fees. +After the merge of eth1 into eth2, +withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by a user transaction that will set the gas price and gas limit as well pay fees. As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers the future withdrawal protocol is agnostic to all other implementation details. From 192696a6c55c1dc437b90601eb398ed64695c9eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 10:49:12 +0800 Subject: [PATCH 118/140] Set codespell<3.0.0,>=2.0.0 version and add `ether` to whitelist --- .circleci/config.yml | 2 +- .codespell-whitelist | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2767bec70..b57e15458 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,7 +120,7 @@ jobs: - checkout - run: name: Check codespell - command: pip install codespell --user && make codespell + command: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell lint: docker: - image: circleci/python:3.8 diff --git a/.codespell-whitelist b/.codespell-whitelist index ff694e380..6b8bab36f 100644 --- a/.codespell-whitelist +++ b/.codespell-whitelist @@ -1,2 +1,3 @@ uint -byteorder \ No newline at end of file +byteorder +ether From d1401a7f60dafa7b5c8dd5a7a913de6d45457544 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 10:59:05 +0800 Subject: [PATCH 119/140] Fix doctoc@2 version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b57e15458..e00463cc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,7 +111,7 @@ jobs: - checkout - run: name: Check table of contents - command: sudo npm install -g doctoc && make check_toc + command: sudo npm install -g doctoc@2 && make check_toc codespell: docker: - image: circleci/python:3.8 From b789b10397f9a10c45066d5ee4c4f62d6c138cc3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 7 Dec 2020 11:00:29 +0800 Subject: [PATCH 120/140] Minor update ToC --- specs/phase0/beacon-chain.md | 1 - specs/phase0/deposit-contract.md | 1 - specs/phase0/fork-choice.md | 1 - specs/phase0/p2p-interface.md | 1 - specs/phase0/validator.md | 1 - specs/phase0/weak-subjectivity.md | 1 - specs/phase1/beacon-chain.md | 1 - specs/phase1/custody-game.md | 1 - specs/phase1/fork-choice.md | 1 - specs/phase1/light-client-sync.md | 1 - specs/phase1/phase1-fork.md | 1 - specs/phase1/shard-fork-choice.md | 1 - specs/phase1/shard-transition.md | 1 - specs/phase1/validator.md | 1 - ssz/merkle-proofs.md | 1 - ssz/simple-serialize.md | 1 - 16 files changed, 16 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 084107340..682131546 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Notation](#notation) - [Custom types](#custom-types) diff --git a/specs/phase0/deposit-contract.md b/specs/phase0/deposit-contract.md index 4c340d688..02e762dae 100644 --- a/specs/phase0/deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Configuration](#configuration) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 8bc108caa..b5689ecd2 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -5,7 +5,6 @@ - - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Configuration](#configuration) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 43a097f7e..5dc892991 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -14,7 +14,6 @@ It consists of four main sections: - - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - [Encryption and identification](#encryption-and-identification) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 15be3ee70..9e2ee7b1f 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -8,7 +8,6 @@ This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](. - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/specs/phase0/weak-subjectivity.md b/specs/phase0/weak-subjectivity.md index 9d433de8e..b4d78cb12 100644 --- a/specs/phase0/weak-subjectivity.md +++ b/specs/phase0/weak-subjectivity.md @@ -6,7 +6,6 @@ - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 23ce88aa9..c8f93cc7f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Custom types](#custom-types) - [Configuration](#configuration) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 82eb35d86..33c2a0397 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index 3845b2b74..d2e1bfefe 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Updated data structures](#updated-data-structures) - [Extended `Store`](#extended-store) diff --git a/specs/phase1/light-client-sync.md b/specs/phase1/light-client-sync.md index 4d14485dd..107baa0c6 100644 --- a/specs/phase1/light-client-sync.md +++ b/specs/phase1/light-client-sync.md @@ -8,7 +8,6 @@ - - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index b4ace1066..d81ca64b3 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Configuration](#configuration) - [Fork to Phase 1](#fork-to-phase-1) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 380269d13..177c9c18c 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index f3a1f83ce..35d421cdd 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -7,7 +7,6 @@ - - [Introduction](#introduction) - [Helper functions](#helper-functions) - [Shard block verification functions](#shard-block-verification-functions) diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 2bfea017e..c5893fbc6 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -8,7 +8,6 @@ - - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) diff --git a/ssz/merkle-proofs.md b/ssz/merkle-proofs.md index 2f32e43eb..6772026fe 100644 --- a/ssz/merkle-proofs.md +++ b/ssz/merkle-proofs.md @@ -7,7 +7,6 @@ - - [Helper functions](#helper-functions) - [Generalized Merkle tree index](#generalized-merkle-tree-index) - [SSZ object to index](#ssz-object-to-index) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index b0e7ec1e0..078da9eae 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -5,7 +5,6 @@ - - [Constants](#constants) - [Typing](#typing) - [Basic types](#basic-types) From fb974ed37f8a1b91cea92778a3003a94af6d6040 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 Feb 2021 16:35:01 -0700 Subject: [PATCH 121/140] bump version to 1.0.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index afaf360d3..7f207341d 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.0.1 \ No newline at end of file From 396d399129090d6c8973acfa47edc50077b69d0a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 24 Feb 2021 08:53:54 -0600 Subject: [PATCH 122/140] Apply suggestions from code review form @hwwhww Co-authored-by: Hsiao-Wei Wang --- specs/phase0/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 9e2ee7b1f..8a6f221a7 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -132,10 +132,10 @@ The `withdrawal_credentials` field must be such that: * `withdrawal_credentials[1:12] == b'\x00' * 11` * `withdrawal_credentials[12:] == eth1_withdrawal_address` -After the merge of eth1 into eth2, +After the merge of the current Ethereum application layer (Eth1) into the Beacon Chain (Eth2), withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) triggered by a user transaction that will set the gas price and gas limit as well pay fees. -As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers +As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers, the future withdrawal protocol is agnostic to all other implementation details. ### Submit deposit From deace8768e87ccd3cdc64d759c1752f764fdc3b8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 1 Mar 2021 18:19:12 -0700 Subject: [PATCH 123/140] port @justindrake's general cleanups from #2212 --- configs/mainnet/lightclient_patch.yaml | 2 +- configs/minimal/lightclient_patch.yaml | 2 +- specs/lightclient/beacon-chain.md | 227 +++++++++--------- specs/lightclient/lightclient-fork.md | 6 +- .../pyspec/eth2spec/test/helpers/rewards.py | 47 ++-- ..._process_justification_and_finalization.py | 6 +- .../test/phase0/sanity/test_blocks.py | 2 +- 7 files changed, 150 insertions(+), 142 deletions(-) diff --git a/configs/mainnet/lightclient_patch.yaml b/configs/mainnet/lightclient_patch.yaml index 6c5b16edf..a9ddc16f6 100644 --- a/configs/mainnet/lightclient_patch.yaml +++ b/configs/mainnet/lightclient_patch.yaml @@ -17,7 +17,7 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # 2**10 (=1,024) SYNC_COMMITTEE_SIZE: 1024 # 2**6 (=64) -SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 64 +SYNC_SUBCOMMITTEE_SIZE: 64 # Time parameters diff --git a/configs/minimal/lightclient_patch.yaml b/configs/minimal/lightclient_patch.yaml index 7ab5f34ba..56ce34591 100644 --- a/configs/minimal/lightclient_patch.yaml +++ b/configs/minimal/lightclient_patch.yaml @@ -17,7 +17,7 @@ HF1_PROPORTIONAL_SLASHING_MULTIPLIER: 2 # [customized] SYNC_COMMITTEE_SIZE: 32 # [customized] -SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE: 16 +SYNC_SUBCOMMITTEE_SIZE: 16 # Time parameters diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index bf9b6a092..de0fdf8c5 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -9,8 +9,8 @@ - [Introduction](#introduction) - [Custom types](#custom-types) - [Constants](#constants) - - [Validator action flags](#validator-action-flags) - - [Participation rewards](#participation-rewards) + - [Participation flag indices](#participation-flag-indices) + - [Participation flag fractions](#participation-flag-fractions) - [Misc](#misc) - [Configuration](#configuration) - [Updated penalty values](#updated-penalty-values) @@ -27,7 +27,9 @@ - [`Predicates`](#predicates) - [`eth2_fast_aggregate_verify`](#eth2_fast_aggregate_verify) - [Misc](#misc-2) - - [`flags_and_numerators`](#flags_and_numerators) + - [`get_flag_indices_and_numerators`](#get_flag_indices_and_numerators) + - [`add_flag`](#add_flag) + - [`has_flag`](#has_flag) - [Beacon state accessors](#beacon-state-accessors) - [`get_sync_committee_indices`](#get_sync_committee_indices) - [`get_sync_committee`](#get_sync_committee) @@ -38,15 +40,15 @@ - [Beacon state mutators](#beacon-state-mutators) - [New `slash_validator`](#new-slash_validator) - [Block processing](#block-processing) - - [New `process_attestation`](#new-process_attestation) + - [Modified `process_attestation`](#modified-process_attestation) - [New `process_deposit`](#new-process_deposit) - [Sync committee processing](#sync-committee-processing) - [Epoch processing](#epoch-processing) - - [New `process_justification_and_finalization`](#new-process_justification_and_finalization) - - [New `process_rewards_and_penalties`](#new-process_rewards_and_penalties) - - [New `process_slashings`](#new-process_slashings) - - [Sync committee updates](#sync-committee-updates) + - [Justification and finalization](#justification-and-finalization) + - [Rewards and penalties](#rewards-and-penalties) + - [Slashings](#slashings) - [Participation flags updates](#participation-flags-updates) + - [Sync committee updates](#sync-committee-updates) @@ -66,34 +68,29 @@ It has four main features: | Name | SSZ equivalent | Description | | - | - | - | -| `ValidatorFlag` | `uint8` | Bitflags to track validator actions with | +| `ParticipationFlags` | `uint8` | A succinct representation of 8 boolean participation flags | ## Constants -### Validator action flags - -This is formatted as an enum, with values `2**i` that can be combined as bit-flags. -The `0` value is reserved as default. Remaining bits in `ValidatorFlag` may be used in future hardforks. - -**Note**: Unlike Phase0, a `TIMELY_TARGET_FLAG` does not necessarily imply a `TIMELY_SOURCE_FLAG` -due to the varying slot delay requirements of each. +### Participation flag indices | Name | Value | | - | - | -| `TIMELY_HEAD_FLAG` | `ValidatorFlag(2**0)` (= 1) | -| `TIMELY_SOURCE_FLAG` | `ValidatorFlag(2**1)` (= 2) | -| `TIMELY_TARGET_FLAG` | `ValidatorFlag(2**2)` (= 4) | +| `TIMELY_HEAD_FLAG_INDEX` | `0` | +| `TIMELY_SOURCE_FLAG_INDEX` | `1` | +| `TIMELY_TARGET_FLAG_INDEX` | `2` | -### Participation rewards +### Participation flag fractions | Name | Value | | - | - | -| `TIMELY_HEAD_NUMERATOR` | `12` | -| `TIMELY_SOURCE_NUMERATOR` | `12` | -| `TIMELY_TARGET_NUMERATOR` | `32` | -| `REWARD_DENOMINATOR` | `64` | +| `TIMELY_HEAD_FLAG_NUMERATOR` | `12` | +| `TIMELY_SOURCE_FLAG_NUMERATOR` | `12` | +| `TIMELY_TARGET_FLAG_NUMERATOR` | `32` | +| `FLAG_DENOMINATOR` | `64` | -The reward fractions add up to 7/8, leaving the remaining 1/8 for proposer rewards and other future micro-rewards. +**Note**: The participatition flag fractions add up to 7/8. +The remaining 1/8 is for proposer incentives and other future micro-incentives. ### Misc @@ -119,8 +116,8 @@ This patch updates a few configuration values to move penalty constants toward t | Name | Value | | - | - | -| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1024) | -| `SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE` | `uint64(2**6)` (= 64) | +| `SYNC_COMMITTEE_SIZE` | `uint64(2**10)` (= 1,024) | +| `SYNC_SUBCOMMITTEE_SIZE` | `uint64(2**6)` (= 64) | ### Time parameters @@ -144,10 +141,19 @@ order and append any additional fields to the end. #### `BeaconBlockBody` ```python -class BeaconBlockBody(phase0.BeaconBlockBody): +class BeaconBlockBody(Container): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Sync committee aggregate signature - sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] - sync_committee_signature: BLSSignature + sync_committee_bits: Bitvector[SYNC_COMMITTEE_SIZE] # [New in HF1] + sync_committee_signature: BLSSignature # [New in HF1] ``` #### `BeaconState` @@ -176,8 +182,8 @@ class BeaconState(Container): # Slashings slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation: List[ValidatorFlag, VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] + current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT] # Finality justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint @@ -195,7 +201,7 @@ class BeaconState(Container): ```python class SyncCommittee(Container): pubkeys: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE] - pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE] + pubkey_aggregates: Vector[BLSPubkey, SYNC_COMMITTEE_SIZE // SYNC_SUBCOMMITTEE_SIZE] ``` ## Helper functions @@ -216,25 +222,31 @@ def eth2_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, s ### Misc -#### `flags_and_numerators` +#### `get_flag_indices_and_numerators` ```python -def get_flags_and_numerators() -> Sequence[Tuple[ValidatorFlag, int]]: +def get_flag_indices_and_numerators() -> Sequence[Tuple[int, int]]: return ( - (TIMELY_HEAD_FLAG, TIMELY_HEAD_NUMERATOR), - (TIMELY_SOURCE_FLAG, TIMELY_SOURCE_NUMERATOR), - (TIMELY_TARGET_FLAG, TIMELY_TARGET_NUMERATOR) + (TIMELY_HEAD_FLAG_INDEX, TIMELY_HEAD_FLAG_NUMERATOR), + (TIMELY_SOURCE_FLAG_INDEX, TIMELY_SOURCE_FLAG_NUMERATOR), + (TIMELY_TARGET_FLAG_INDEX, TIMELY_TARGET_FLAG_NUMERATOR), ) ``` -```python -def add_validator_flags(flags: ValidatorFlag, add: ValidatorFlag) -> ValidatorFlag: - return flags | add -``` +#### `add_flag` ```python -def has_validator_flags(flags: ValidatorFlag, has: ValidatorFlag) -> bool: - return flags & has == has +def add_flag(flags: ParticipationFlags, flag_index: int) -> ParticipationFlags: + flag = ParticipationFlags(2**flag_index) + return flags | flag +``` + +#### `has_flag` + +```python +def has_flag(flags: ParticipationFlags, flag_index: int) -> bool: + flag = ParticipationFlags(2**flag_index) + return flags & flag == flag ``` ### Beacon state accessors @@ -275,8 +287,8 @@ def get_sync_committee(state: BeaconState, epoch: Epoch) -> SyncCommittee: validators = [state.validators[index] for index in indices] pubkeys = [validator.pubkey for validator in validators] aggregates = [ - bls.AggregatePKs(pubkeys[i:i + SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE]) - for i in range(0, len(pubkeys), SYNC_COMMITTEE_PUBKEY_AGGREGATES_SIZE) + bls.AggregatePKs(pubkeys[i:i + SYNC_SUBCOMMITTEE_SIZE]) + for i in range(0, len(pubkeys), SYNC_SUBCOMMITTEE_SIZE) ] return SyncCommittee(pubkeys=pubkeys, pubkey_aggregates=aggregates) ``` @@ -295,19 +307,17 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: #### `get_unslashed_participating_indices` ```python -def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag, epoch: Epoch) -> Set[ValidatorIndex]: +def get_unslashed_participating_indices(state: BeaconState, flag_index: int, epoch: Epoch) -> Set[ValidatorIndex]: """ - Retrieve the active validator indices of the given epoch, which are not slashed, and have all of the given flags. + Retrieve the active and unslashed validator indices for the given epoch and flag index. """ assert epoch in (get_previous_epoch(state), get_current_epoch(state)) if epoch == get_current_epoch(state): epoch_participation = state.current_epoch_participation else: epoch_participation = state.previous_epoch_participation - participating_indices = [ - index for index in get_active_validator_indices(state, epoch) - if has_validator_flags(epoch_participation[index], flags) - ] + active_validator_indices = get_active_validator_indices(state, epoch) + participating_indices = [i for i in active_validator_indices if has_flag(epoch_participation[i], flag_index)] return set(filter(lambda index: not state.validators[index].slashed, participating_indices)) ``` @@ -315,7 +325,7 @@ def get_unslashed_participating_indices(state: BeaconState, flags: ValidatorFlag ```python def get_flag_deltas(state: BeaconState, - flag: ValidatorFlag, + flag_index: int, numerator: uint64) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ Compute the rewards and penalties associated with a particular duty, by scanning through the participation @@ -324,7 +334,7 @@ def get_flag_deltas(state: BeaconState, rewards = [Gwei(0)] * len(state.validators) penalties = [Gwei(0)] * len(state.validators) - unslashed_participating_indices = get_unslashed_participating_indices(state, flag, get_previous_epoch(state)) + unslashed_participating_indices = get_unslashed_participating_indices(state, flag_index, get_previous_epoch(state)) increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balances to avoid uint64 overflow unslashed_participating_increments = get_total_balance(state, unslashed_participating_indices) // increment active_increments = get_total_active_balance(state) // increment @@ -333,14 +343,14 @@ def get_flag_deltas(state: BeaconState, if index in unslashed_participating_indices: if is_in_inactivity_leak(state): # Optimal participation is fully rewarded to cancel the inactivity penalty - rewards[index] = base_reward * numerator // REWARD_DENOMINATOR + rewards[index] = base_reward * numerator // FLAG_DENOMINATOR else: rewards[index] = ( (base_reward * numerator * unslashed_participating_increments) - // (active_increments * REWARD_DENOMINATOR) + // (active_increments * FLAG_DENOMINATOR) ) else: - penalties[index] = base_reward * numerator // REWARD_DENOMINATOR + penalties[index] = base_reward * numerator // FLAG_DENOMINATOR return rewards, penalties ``` @@ -358,13 +368,13 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S """ penalties = [Gwei(0) for _ in range(len(state.validators))] if is_in_inactivity_leak(state): - reward_numerator_sum = sum(numerator for (_, numerator) in get_flags_and_numerators()) + reward_numerator_sum = sum(numerator for (_, numerator) in get_flag_indices_and_numerators()) matching_target_attesting_indices = get_unslashed_participating_indices( - state, TIMELY_TARGET_FLAG, get_previous_epoch(state) + state, TIMELY_TARGET_FLAG_INDEX, get_previous_epoch(state) ) for index in get_eligible_validator_indices(state): # If validator is performing optimally this cancels all attestation rewards for a neutral balance - penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // REWARD_DENOMINATOR) + penalties[index] += Gwei(get_base_reward(state, index) * reward_numerator_sum // FLAG_DENOMINATOR) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance penalties[index] += Gwei( @@ -415,12 +425,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) - # Light client support - process_sync_committee(state, block.body) + process_operations(state, block.body) # [Modified in HF1] + process_sync_committee(state, block.body) # [New in HF1] ``` -#### New `process_attestation` +#### Modified `process_attestation` *Note*: The function `process_attestation` is modified to do incentive accounting with epoch participation flags. @@ -451,25 +460,25 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Verify signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # Participation flags - participation_flags = [] + # Participation flag indices + participation_flag_indices = [] if is_matching_head and is_matching_target and state.slot <= data.slot + MIN_ATTESTATION_INCLUSION_DELAY: - participation_flags.append(TIMELY_HEAD_FLAG) + participation_flag_indices.append(TIMELY_HEAD_FLAG_INDEX) if is_matching_source and state.slot <= data.slot + integer_squareroot(SLOTS_PER_EPOCH): - participation_flags.append(TIMELY_SOURCE_FLAG) + participation_flag_indices.append(TIMELY_SOURCE_FLAG_INDEX) if is_matching_target and state.slot <= data.slot + SLOTS_PER_EPOCH: - participation_flags.append(TIMELY_TARGET_FLAG) + participation_flag_indices.append(TIMELY_TARGET_FLAG_INDEX) # Update epoch participation flags proposer_reward_numerator = 0 for index in get_attesting_indices(state, data, attestation.aggregation_bits): - for flag, numerator in get_flags_and_numerators(): - if flag in participation_flags and not has_validator_flags(epoch_participation[index], flag): - epoch_participation[index] = add_validator_flags(epoch_participation[index], flag) - proposer_reward_numerator += get_base_reward(state, index) * numerator + for flag_index, flag_numerator in get_flag_indices_and_numerators(): + if flag_index in participation_flag_indices and not has_flag(epoch_participation[index], flag_index): + epoch_participation[index] = add_flag(epoch_participation[index], flag_index) + proposer_reward_numerator += get_base_reward(state, index) * flag_numerator # Reward proposer - proposer_reward = Gwei(proposer_reward_numerator // (REWARD_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) + proposer_reward = Gwei(proposer_reward_numerator // (FLAG_DENOMINATOR * PROPOSER_REWARD_QUOTIENT)) increase_balance(state, get_beacon_proposer_index(state), proposer_reward) ``` @@ -511,8 +520,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) # [Added in hf-1] Initialize empty participation flags for new validator - state.previous_epoch_participation.append(ValidatorFlag(0)) - state.current_epoch_participation.append(ValidatorFlag(0)) + state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) + state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) @@ -534,26 +543,26 @@ def process_sync_committee(state: BeaconState, body: BeaconBlockBody) -> None: assert eth2_fast_aggregate_verify(participant_pubkeys, signing_root, body.sync_committee_signature) # Reward sync committee participants - total_proposer_reward = Gwei(0) + proposer_rewards = Gwei(0) active_validator_count = uint64(len(get_active_validator_indices(state, get_current_epoch(state)))) for participant_index in participant_indices: - base_reward = get_base_reward(state, participant_index) proposer_reward = get_proposer_reward(state, participant_index) + proposer_rewards += proposer_reward + base_reward = get_base_reward(state, participant_index) max_participant_reward = base_reward - proposer_reward - reward = Gwei(max_participant_reward * active_validator_count // len(committee_indices) // SLOTS_PER_EPOCH) + reward = Gwei(max_participant_reward * active_validator_count // (len(committee_indices) * SLOTS_PER_EPOCH)) increase_balance(state, participant_index, reward) - total_proposer_reward += proposer_reward # Reward beacon proposer - increase_balance(state, get_beacon_proposer_index(state), total_proposer_reward) + increase_balance(state, get_beacon_proposer_index(state), proposer_rewards) ``` ### Epoch processing ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) # [Updated in HF1] - process_rewards_and_penalties(state) # [Updated in HF1] + process_justification_and_finalization(state) # [Modified in HF1] + process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) process_slashings(state) process_eth1_data_reset(state) @@ -561,13 +570,11 @@ def process_epoch(state: BeaconState) -> None: process_slashings_reset(state) process_randao_mixes_reset(state) process_historical_roots_update(state) - # [Removed in HF1] -- process_participation_record_updates(state) - # [Added in HF1] - process_participation_flag_updates(state) - process_sync_committee_updates(state) + process_participation_flag_updates(state) # [New in HF1] + process_sync_committee_updates(state) # [New in HF1] ``` -#### New `process_justification_and_finalization` +#### Justification and finalization *Note*: The function `process_justification_and_finalization` is modified with `matching_target_attestations` replaced by `matching_target_indices`. @@ -586,12 +593,12 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_checkpoint = state.current_justified_checkpoint state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, previous_epoch) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, previous_epoch) if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG, current_epoch) + matching_target_indices = get_unslashed_participating_indices(state, TIMELY_TARGET_FLAG_INDEX, current_epoch) if get_total_balance(state, matching_target_indices) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) @@ -613,16 +620,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.finalized_checkpoint = old_current_justified_checkpoint ``` -#### New `process_rewards_and_penalties` +#### Rewards and penalties -*Note*: The function `process_rewards_and_penalties` is modified to use participation flag deltas. +*Note*: The function `process_rewards_and_penalties` is modified to support the incentive reforms. ```python def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch if get_current_epoch(state) == GENESIS_EPOCH: return - flag_deltas = [get_flag_deltas(state, flag, numerator) for (flag, numerator) in get_flags_and_numerators()] + flag_deltas = [ + get_flag_deltas(state, flag_index, flag_numerator) + for (flag_index, flag_numerator) in get_flag_indices_and_numerators() + ] deltas = flag_deltas + [get_inactivity_penalty_deltas(state)] for (rewards, penalties) in deltas: for index in range(len(state.validators)): @@ -630,10 +640,9 @@ def process_rewards_and_penalties(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` -#### New `process_slashings` +#### Slashings -*Note*: The function `process_slashings` is modified -with the substitution of `PROPORTIONAL_SLASHING_MULTIPLIER` with `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. +*Note*: The function `process_slashings` is modified to use `HF1_PROPORTIONAL_SLASHING_MULTIPLIER`. ```python def process_slashings(state: BeaconState) -> None: @@ -648,26 +657,24 @@ def process_slashings(state: BeaconState) -> None: decrease_balance(state, ValidatorIndex(index), penalty) ``` +#### Participation flags updates + +*Note*: The function `process_participation_flag_updates` is new. + +```python +def process_participation_flag_updates(state: BeaconState) -> None: + state.previous_epoch_participation = state.current_epoch_participation + state.current_epoch_participation = [ParticipationFlags(0b0000_0000) for _ in range(len(state.validators))] +``` + #### Sync committee updates +*Note*: The function `process_sync_committee_updates` is new. + ```python def process_sync_committee_updates(state: BeaconState) -> None: - """ - Call to ``proces_sync_committee_updates`` added to ``process_epoch`` in HF1 - """ next_epoch = get_current_epoch(state) + Epoch(1) if next_epoch % EPOCHS_PER_SYNC_COMMITTEE_PERIOD == 0: state.current_sync_committee = state.next_sync_committee state.next_sync_committee = get_sync_committee(state, next_epoch + EPOCHS_PER_SYNC_COMMITTEE_PERIOD) ``` - -#### Participation flags updates - -```python -def process_participation_flag_updates(state: BeaconState) -> None: - """ - Call to ``process_participation_flag_updates`` added to ``process_epoch`` in HF1 - """ - state.previous_epoch_participation = state.current_epoch_participation - state.current_epoch_participation = [ValidatorFlag(0) for _ in range(len(state.validators))] -``` diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/lightclient-fork.md index aa0171b86..157e67dc9 100644 --- a/specs/lightclient/lightclient-fork.md +++ b/specs/lightclient/lightclient-fork.md @@ -66,9 +66,9 @@ def upgrade_to_lightclient_patch(pre: phase0.BeaconState) -> BeaconState: randao_mixes=pre.randao_mixes, # Slashings slashings=pre.slashings, - # Attestations - previous_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], - current_epoch_participation=[ValidatorFlag(0) for _ in range(len(pre.validators))], + # Participation + previous_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], + current_epoch_participation=[ParticipationFlags(0) for _ in range(len(pre.validators))], # Finality justification_bits=pre.justification_bits, previous_justified_checkpoint=pre.previous_justified_checkpoint, diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index 2499bcffe..19ed3f691 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -41,13 +41,13 @@ def run_deltas(spec, state): if is_post_lightclient_patch(spec): def get_source_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG, spec.TIMELY_SOURCE_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_SOURCE_FLAG_INDEX, spec.TIMELY_SOURCE_FLAG_NUMERATOR) def get_head_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG, spec.TIMELY_HEAD_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_HEAD_FLAG_INDEX, spec.TIMELY_HEAD_FLAG_NUMERATOR) def get_target_deltas(state): - return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG, spec.TIMELY_TARGET_NUMERATOR) + return spec.get_flag_deltas(state, spec.TIMELY_TARGET_FLAG_INDEX, spec.TIMELY_TARGET_FLAG_NUMERATOR) yield from run_attestation_component_deltas( spec, @@ -74,13 +74,13 @@ def run_deltas(spec, state): yield from run_get_inactivity_penalty_deltas(spec, state) -def deltas_name_to_flag(spec, deltas_name): +def deltas_name_to_flag_index(spec, deltas_name): if 'source' in deltas_name: - return spec.TIMELY_SOURCE_FLAG + return spec.TIMELY_SOURCE_FLAG_INDEX elif 'head' in deltas_name: - return spec.TIMELY_HEAD_FLAG + return spec.TIMELY_HEAD_FLAG_INDEX elif 'target' in deltas_name: - return spec.TIMELY_TARGET_FLAG + return spec.TIMELY_TARGET_FLAG_INDEX raise ValueError("Wrong deltas_name %s" % deltas_name) @@ -98,7 +98,7 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: matching_indices = spec.get_unslashed_participating_indices( - state, deltas_name_to_flag(spec, deltas_name), spec.get_previous_epoch(state) + state, deltas_name_to_flag_index(spec, deltas_name), spec.get_previous_epoch(state) ) eligible_indices = spec.get_eligible_validator_indices(state) @@ -187,9 +187,9 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) else: matching_attesting_indices = spec.get_unslashed_participating_indices( - state, spec.TIMELY_TARGET_FLAG, spec.get_previous_epoch(state) + state, spec.TIMELY_TARGET_FLAG_INDEX, spec.get_previous_epoch(state) ) - reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flags_and_numerators()) + reward_numerator_sum = sum(numerator for (_, numerator) in spec.get_flag_indices_and_numerators()) eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): @@ -205,7 +205,7 @@ def run_get_inactivity_penalty_deltas(spec, state): base_reward = spec.get_base_reward(state, index) base_penalty = cancel_base_rewards_per_epoch * base_reward - spec.get_proposer_reward(state, index) else: - base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.REWARD_DENOMINATOR + base_penalty = spec.get_base_reward(state, index) * reward_numerator_sum // spec.FLAG_DENOMINATOR if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 @@ -314,7 +314,7 @@ def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): else: for index in range(len(state.validators)): if rng.choice([True, False]): - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -328,7 +328,7 @@ def run_test_partial(spec, state, fraction_filled): state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] else: for index in range(int(len(state.validators) * fraction_filled)): - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -394,7 +394,7 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): else: index = 0 state.validators[index].effective_balance = 1 - state.previous_epoch_participation[index] = spec.ValidatorFlag(0) + state.previous_epoch_participation[index] = spec.ParticipationFlags(0b0000_0000) yield from run_deltas(spec, state) @@ -521,23 +521,24 @@ def run_test_full_random(spec, state, rng=Random(8020)): is_timely_correct_head = rng.randint(0, 2) != 0 flags = state.previous_epoch_participation[index] - def set_flag(f, v): + def set_flag(index, value): nonlocal flags - if v: - flags |= f + flag = spec.ParticipationFlags(2**index) + if value: + flags |= flag else: - flags &= 0xff ^ f + flags &= 0xff ^ flag - set_flag(spec.TIMELY_HEAD_FLAG, is_timely_correct_head) + set_flag(spec.TIMELY_HEAD_FLAG_INDEX, is_timely_correct_head) if is_timely_correct_head: # If timely head, then must be timely target - set_flag(spec.TIMELY_TARGET_FLAG, True) + set_flag(spec.TIMELY_TARGET_FLAG_INDEX, True) # If timely head, then must be timely source - set_flag(spec.TIMELY_SOURCE_FLAG, True) + set_flag(spec.TIMELY_SOURCE_FLAG_INDEX, True) else: # ~50% of remaining have bad target or not timely enough - set_flag(spec.TIMELY_TARGET_FLAG, rng.choice([True, False])) + set_flag(spec.TIMELY_TARGET_FLAG_INDEX, rng.choice([True, False])) # ~50% of remaining have bad source or not timely enough - set_flag(spec.TIMELY_SOURCE_FLAG, rng.choice([True, False])) + set_flag(spec.TIMELY_SOURCE_FLAG_INDEX, rng.choice([True, False])) state.previous_epoch_participation[index] = flags yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py index 89783f987..274d67134 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py +++ b/tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_justification_and_finalization.py @@ -78,10 +78,10 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: for i, index in enumerate(committee): if aggregation_bits[i]: - epoch_participation[index] |= spec.TIMELY_HEAD_FLAG - epoch_participation[index] |= spec.TIMELY_SOURCE_FLAG + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_HEAD_FLAG_INDEX) + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_SOURCE_FLAG_INDEX) if not messed_up_target: - epoch_participation[index] |= spec.TIMELY_TARGET_FLAG + epoch_participation[index] |= spec.ParticipationFlags(2**spec.TIMELY_TARGET_FLAG_INDEX) def get_checkpoints(spec, epoch): diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 1834b290f..98ffbd590 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -806,7 +806,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root else: for index in range(len(state.validators)): - assert state.current_epoch_participation[index] == 0 + assert state.current_epoch_participation[index] == spec.ParticipationFlags(0b0000_0000) assert spec.hash_tree_root(state.previous_epoch_participation) == pre_current_epoch_participation_root From 8ea5e37608bd6734e2764665d35a0433579b5ad0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Mar 2021 07:56:30 -0600 Subject: [PATCH 124/140] add another 'modified' tag Co-authored-by: Hsiao-Wei Wang --- specs/lightclient/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/lightclient/beacon-chain.md b/specs/lightclient/beacon-chain.md index de0fdf8c5..bc070908a 100644 --- a/specs/lightclient/beacon-chain.md +++ b/specs/lightclient/beacon-chain.md @@ -564,7 +564,7 @@ def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) # [Modified in HF1] process_rewards_and_penalties(state) # [Modified in HF1] process_registry_updates(state) - process_slashings(state) + process_slashings(state) # [Modified in HF1] process_eth1_data_reset(state) process_effective_balance_updates(state) process_slashings_reset(state) From e1023f55d1ab36c0c148da991df62f043b3c6188 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 5 Mar 2021 18:05:15 -0800 Subject: [PATCH 125/140] Fix a small typo --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 249d08799..02a578fb8 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -292,7 +292,7 @@ If one or more validations fail while processing the items in order, return eith There are two primary global topics used to propagate beacon blocks (`beacon_block`) and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the network. -There are three additional global topics are used to propagate lower frequency validator messages +There are three additional global topics that are used to propagate lower frequency validator messages (`voluntary_exit`, `proposer_slashing`, and `attester_slashing`). ##### `beacon_block` From 6bd1efc73bf45ae5658e115a8b2956144d4f4559 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 17:16:13 -0700 Subject: [PATCH 126/140] rename fork files --- setup.py | 4 ++-- specs/lightclient/{lightclient-fork.md => fork.md} | 0 specs/phase1/{phase1-fork.md => fork.md} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename specs/lightclient/{lightclient-fork.md => fork.md} (100%) rename specs/phase1/{phase1-fork.md => fork.md} (100%) diff --git a/setup.py b/setup.py index 6cfa0910a..ca85a83ac 100644 --- a/setup.py +++ b/setup.py @@ -449,7 +449,7 @@ class PySpecCommand(Command): specs/phase1/beacon-chain.md specs/phase1/shard-transition.md specs/phase1/fork-choice.md - specs/phase1/phase1-fork.md + specs/phase1/fork.md specs/phase1/shard-fork-choice.md specs/phase1/validator.md """ @@ -460,7 +460,7 @@ class PySpecCommand(Command): specs/phase0/validator.md specs/phase0/weak-subjectivity.md specs/lightclient/beacon-chain.md - specs/lightclient/lightclient-fork.md + specs/lightclient/fork.md """ # TODO: add specs/lightclient/sync-protocol.md back when the GeneralizedIndex helpers are included. else: diff --git a/specs/lightclient/lightclient-fork.md b/specs/lightclient/fork.md similarity index 100% rename from specs/lightclient/lightclient-fork.md rename to specs/lightclient/fork.md diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/fork.md similarity index 100% rename from specs/phase1/phase1-fork.md rename to specs/phase1/fork.md From d6961f636de17b31ecb3350b0a5710a1d102d669 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 17:16:29 -0700 Subject: [PATCH 127/140] add base hf1 fork function tests --- tests/core/pyspec/eth2spec/test/context.py | 10 +- .../test/lightclient_patch/fork/test_fork.py | 104 ++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 5c2a3bf4d..3f247b400 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -346,13 +346,11 @@ def with_phases(phases, other_phases=None): # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 available_phases.add(PHASE0) + # Populate all phases for multi-phase tests phase_dir = {} - if PHASE0 in available_phases: - phase_dir[PHASE0] = spec_phase0 - if PHASE1 in available_phases: - phase_dir[PHASE1] = spec_phase1 - if LIGHTCLIENT_PATCH in available_phases: - phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch + phase_dir[PHASE0] = spec_phase0 + phase_dir[PHASE1] = spec_phase1 + phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py new file mode 100644 index 000000000..08c6e6d29 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -0,0 +1,104 @@ +from operator import attrgetter + +from eth2spec.test.context import ( + PHASE0, LIGHTCLIENT_PATCH, + spec_state_test, with_phases, + with_custom_state, + spec_test, with_state, + low_balances, misc_balances, large_validator_set, +) +from eth2spec.test.helpers.state import ( + next_slots, + next_epoch, + next_epoch_via_block, + transition_to_slot_via_block, +) + + +def run_fork_test(spec, pre_state): + yield 'pre', pre_state + + post_state = spec.upgrade_to_lightclient_patch(pre_state) + + # Stable fields + stable_fields = [ + 'genesis_time', 'genesis_validators_root', 'slot', + # History + 'latest_block_header', 'block_roots', 'state_roots', 'historical_roots', + # Eth1 + 'eth1_data', 'eth1_data_votes', 'eth1_deposit_index', + # Registry + 'validators', 'balances', + # Randomness + 'randao_mixes', + # Slashings + 'slashings', + # Finality + 'justification_bits', 'previous_justified_checkpoint', 'current_justified_checkpoint', 'finalized_checkpoint', + ] + for field in stable_fields: + assert getattr(pre_state, field) == getattr(post_state, field) + + # Modified fields + modified_fields = ['fork'] + for field in modified_fields: + assert getattr(pre_state, field) != getattr(post_state, field) + + assert pre_state.fork.current_version == post_state.fork.previous_version + assert post_state.fork.current_version == spec.LIGHTCLIENT_PATCH_FORK_VERSION + assert post_state.fork.epoch == spec.get_current_epoch(post_state) + + yield 'post', post_state + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_base_state(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_next_epoch(spec, phases, state): + next_epoch(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_next_epoch_with_block(spec, phases, state): + next_epoch_via_block(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_state +@spec_test +def test_fork_many_next_epoch(spec, phases, state): + for _ in range(3): + next_epoch(spec, state) + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_low_balances(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_misc_balances(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) + + +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +def test_fork_random_large_validator_set(spec, phases, state): + yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From 6c406753f150563cac1253cfef8d99d9111bd906 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 8 Mar 2021 19:11:31 -0700 Subject: [PATCH 128/140] working through test gens --- tests/core/pyspec/eth2spec/test/context.py | 18 +++++-- .../test/lightclient_patch/fork/__init__.py | 0 .../test/lightclient_patch/fork/test_fork.py | 44 +++++++++++------ tests/formats/forks/README.md | 47 +++++++++++++++++++ tests/generators/forks/main.py | 40 ++++++++++++++++ tests/generators/forks/requirements.txt | 2 + 6 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py create mode 100644 tests/formats/forks/README.md create mode 100644 tests/generators/forks/main.py create mode 100644 tests/generators/forks/requirements.txt diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 3f247b400..9c45049a2 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -70,7 +70,7 @@ class SpecForks(TypedDict, total=False): def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], - spec: Spec, phases: SpecForks): + spec: Spec, phases: SpecForks, is_fork_test: bool): p0 = phases[PHASE0] balances = balances_fn(p0) @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH: # not generalizing this just yet, unclear final spec fork/patch order. + elif spec.fork == LIGHTCLIENT_PATCH and not fork_test: # do not upgrade if spec ttttest state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -98,10 +98,11 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def entry(*args, spec: Spec, phases: SpecForks, **kw): # make a key for the state # genesis fork version separates configs during test-generation runtime. - key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) + is_fork_test = kw.pop('fork_test') if 'fork_test' in kw else False + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn, is_fork_test) global _custom_state_cache_dict if key not in _custom_state_cache_dict: - state = _prepare_state(balances_fn, threshold_fn, spec, phases) + state = _prepare_state(balances_fn, threshold_fn, spec, phases, is_fork_test) _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -287,6 +288,15 @@ def bls_switch(fn): return entry +def fork_test(fn): + """ + """ + def entry(*args, **kw): + # override fork test setting + kw['fork_test'] = True + return entry + + def disable_process_reveal_deadlines(fn): """ Decorator to make a function execute with `process_reveal_deadlines` OFF. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 08c6e6d29..527ad746c 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,20 +1,22 @@ -from operator import attrgetter - from eth2spec.test.context import ( - PHASE0, LIGHTCLIENT_PATCH, - spec_state_test, with_phases, - with_custom_state, + LIGHTCLIENT_PATCH, + with_phases, + with_custom_state, fork_test, spec_test, with_state, low_balances, misc_balances, large_validator_set, ) +from eth2spec.test.utils import with_meta_tags from eth2spec.test.helpers.state import ( - next_slots, next_epoch, next_epoch_via_block, - transition_to_slot_via_block, ) +HF1_FORK_TEST_META_TAGS = { + 'fork': 'altair', +} + + def run_fork_test(spec, pre_state): yield 'pre', pre_state @@ -51,54 +53,68 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_state @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(([LIGHTCLIENT_PATCH])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test +@fork_test +@with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md new file mode 100644 index 000000000..1a7ee64ad --- /dev/null +++ b/tests/formats/forks/README.md @@ -0,0 +1,47 @@ +# Forks + +The aim of the fork tests is to ensure that a pre-fork state can be transformed + into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec. + +There is only one handler: `core`. Each fork (after genesis) is handled with the same format, + and the particular fork boundary being tested is noted in `meta.yaml`. + +## Test case format + +### `meta.yaml` + +A yaml file to signify which fork boundary is being tested. + +```yaml +fork: str -- Fork being transitioned to +``` + +#### Fork strings + +Key of valid `fork` strings that might be found in `meta.yaml` + +| String ID | Pre-fork | Post-fork | Function | +| - | - | - | - | +| `altair` | Phase 0 | Altair | `upgrade_to_lightclient_patch` | + +### `pre.yaml` + +A YAML-encoded `BeaconState`, the state before running the fork transition. + +Also available as `pre.ssz`. + +### `post.yaml` + +A YAML-encoded `BeaconState`, the state after applying the fork transition. + +Also available as `post.ssz`. + +*Note*: This type is the `BeaconState` after the fork and is *not* the same type as `pre`. + +## Processing + +To process this test, pass `pre` into the upgrade function defined by the `fork` in `meta.yaml`. + +## Condition + +The resulting state should match the expected `post`. diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py new file mode 100644 index 000000000..a32ece712 --- /dev/null +++ b/tests/generators/forks/main.py @@ -0,0 +1,40 @@ +from importlib import reload +from typing import Iterable + +from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.config import config_util +from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks +from eth2spec.phase0 import spec as spec_phase0 + +from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing +from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests + + +pre_specs = { + LIGHTCLIENT_PATCH: spec_phase0, +} + + +def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: + + def prepare_fn(configs_path: str) -> str: + config_util.prepare_config(configs_path, config_name) + reload(pre_specs[fork_name]) + return config_name + + def cases_fn() -> Iterable[gen_typing.TestCase]: + return generate_from_tests( + runner_name='forks', + handler_name='core', + src=tests_src, + fork_name=fork_name, + ) + + return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) + + +if __name__ == "__main__": + gen_runner.run_generator("forks", [ + create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + ]) diff --git a/tests/generators/forks/requirements.txt b/tests/generators/forks/requirements.txt new file mode 100644 index 000000000..816df6e63 --- /dev/null +++ b/tests/generators/forks/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../ \ No newline at end of file From 50fb3da0729429179e6a24abaf1150b6c023a94b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:31:06 +0800 Subject: [PATCH 129/140] Make test_fork.py truly pass --- tests/core/pyspec/eth2spec/test/context.py | 3 +- .../test/lightclient_patch/fork/test_fork.py | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 9c45049a2..f3d8e2a8f 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -294,6 +294,7 @@ def fork_test(fn): def entry(*args, **kw): # override fork test setting kw['fork_test'] = True + return fn(*args, **kw) return entry diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 527ad746c..88ff68a06 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - LIGHTCLIENT_PATCH, + PHASE0, LIGHTCLIENT_PATCH, with_phases, with_custom_state, fork_test, spec_test, with_state, @@ -53,39 +53,39 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch(spec, phases, state): next_epoch(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_next_epoch_with_block(spec, phases, state): next_epoch_via_block(spec, state) yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_state -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_many_next_epoch(spec, phases, state): for _ in range(3): @@ -93,28 +93,29 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@spec_test +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([LIGHTCLIENT_PATCH])) -@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@spec_test @fork_test +@with_phases(([PHASE0])) +@with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f97ea9e172e88a7527e7c7fa97e104a90df99a69 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 20:55:39 +0800 Subject: [PATCH 130/140] Generate basic tests. Still having problem with generating `with_custom_state` tests --- .../gen_helpers/gen_from_tests/gen.py | 11 +++++++--- tests/generators/forks/main.py | 21 +++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 67d29b194..f04bf46a7 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,6 +1,6 @@ from importlib import reload, import_module from inspect import getmembers, isfunction -from typing import Any, Callable, Dict, Iterable +from typing import Any, Callable, Dict, Iterable, Optional from eth2spec.config import config_util from eth2spec.utils import bls @@ -11,7 +11,7 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. @@ -20,12 +20,17 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param fork_name: to run tests against particular phase and/or fork. (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. + :param phase: optional, specific phase name :return: an iterable of test cases. """ fn_names = [ name for (name, _) in getmembers(src, isfunction) if name.startswith('test_') ] + + if phase is None: + phase = fork_name + print("generating test vectors from tests source: %s" % src.__name__) for name in fn_names: tfn = getattr(src, name) @@ -42,7 +47,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, suite_name='pyspec_tests', case_name=case_name, # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent. - case_fn=lambda: tfn(generator_mode=True, phase=fork_name, bls_active=bls_active) + case_fn=lambda: tfn(generator_mode=True, phase=phase, bls_active=bls_active) ) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index a32ece712..d39af802a 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -1,7 +1,7 @@ from importlib import reload from typing import Iterable -from eth2spec.test.context import LIGHTCLIENT_PATCH +from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 @@ -10,24 +10,20 @@ from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -pre_specs = { - LIGHTCLIENT_PATCH: spec_phase0, -} - - -def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) - reload(pre_specs[fork_name]) + reload(spec_phase0) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( - runner_name='forks', - handler_name='core', + runner_name='fork', + handler_name='fork', src=tests_src, - fork_name=fork_name, + fork_name=LIGHTCLIENT_PATCH, + phase=PHASE0, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -35,6 +31,5 @@ def create_provider(fork_name: str, tests_src, config_name: str) -> gen_typing.T if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), - create_provider(LIGHTCLIENT_PATCH, test_altair_forks, 'minimal'), + create_provider(test_altair_forks, MINIMAL), ]) From 7a10c7108ad5b21efac726afe2a5799383457a3f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:17:02 +0800 Subject: [PATCH 131/140] Fix decorator calls --- .../eth2spec/test/lightclient_patch/fork/test_fork.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 88ff68a06..c5b9b8981 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -95,9 +95,8 @@ def test_fork_many_next_epoch(spec, phases, state): @fork_test @with_phases(([PHASE0])) -@spec_test @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -106,7 +105,7 @@ def test_fork_random_low_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) @@ -115,7 +114,7 @@ def test_fork_random_misc_balances(spec, phases, state): @fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) -@with_state +@spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) def test_fork_random_large_validator_set(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) From f025ec40c584e45517aeed1f93ce4fa4db1f0f3d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:21:32 +0800 Subject: [PATCH 132/140] Fix linter error --- tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index f04bf46a7..057aa1a2c 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -11,7 +11,8 @@ from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider def generate_from_tests(runner_name: str, handler_name: str, src: Any, - fork_name: SpecForkName, bls_active: bool = True, phase: Optional[str]=None) -> Iterable[TestCase]: + fork_name: SpecForkName, bls_active: bool = True, + phase: Optional[str]=None) -> Iterable[TestCase]: """ Generate a list of test cases by running tests from the given src in generator-mode. :param runner_name: to categorize the test in general as. From 5e864af67a161878c294a41dce579a20f98b62bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:32:37 +0800 Subject: [PATCH 133/140] Reload Altair spec --- tests/generators/forks/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index d39af802a..dd4c0051c 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -5,6 +5,7 @@ from eth2spec.test.context import PHASE0, LIGHTCLIENT_PATCH, MINIMAL, MAINNET from eth2spec.config import config_util from eth2spec.test.lightclient_patch.fork import test_fork as test_altair_forks from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.lightclient_patch import spec as spec_lightclient_patch from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests @@ -15,6 +16,7 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) reload(spec_phase0) + reload(spec_lightclient_patch) return config_name def cases_fn() -> Iterable[gen_typing.TestCase]: From f71a3c6b22fe23f28eb49216618a20ef194d1025 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Mar 2021 21:34:45 +0800 Subject: [PATCH 134/140] Generate with mainnet config --- tests/generators/forks/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index dd4c0051c..b22707786 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -34,4 +34,5 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": gen_runner.run_generator("forks", [ create_provider(test_altair_forks, MINIMAL), + create_provider(test_altair_forks, MAINNET), ]) From 956a7a2ef1a20da3472307768d41e28b69c80840 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:09:08 -0700 Subject: [PATCH 135/140] Update tests/core/pyspec/eth2spec/test/context.py --- tests/core/pyspec/eth2spec/test/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index f3d8e2a8f..916c74e6a 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec ttttest + elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec test state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state From f9b54ea03ba57ee3335721596513c45f1f2eaa7c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:18:30 -0700 Subject: [PATCH 136/140] remove fork_test --- tests/core/pyspec/eth2spec/test/context.py | 19 ++++--------------- .../test/lightclient_patch/fork/test_fork.py | 9 +-------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 916c74e6a..1272485de 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -70,7 +70,7 @@ class SpecForks(TypedDict, total=False): def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], - spec: Spec, phases: SpecForks, is_fork_test: bool): + spec: Spec, phases: SpecForks): p0 = phases[PHASE0] balances = balances_fn(p0) @@ -82,7 +82,7 @@ def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Ca # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. # Decide based on performance/consistency results later. state = phases[PHASE1].upgrade_to_phase1(state) - elif spec.fork == LIGHTCLIENT_PATCH and not is_fork_test: # do not upgrade if spec test + elif spec.fork == LIGHTCLIENT_PATCH: state = phases[LIGHTCLIENT_PATCH].upgrade_to_lightclient_patch(state) return state @@ -98,11 +98,10 @@ def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], def entry(*args, spec: Spec, phases: SpecForks, **kw): # make a key for the state # genesis fork version separates configs during test-generation runtime. - is_fork_test = kw.pop('fork_test') if 'fork_test' in kw else False - key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn, is_fork_test) + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) global _custom_state_cache_dict if key not in _custom_state_cache_dict: - state = _prepare_state(balances_fn, threshold_fn, spec, phases, is_fork_test) + state = _prepare_state(balances_fn, threshold_fn, spec, phases) _custom_state_cache_dict[key] = state.get_backing() # Take an entry out of the LRU. @@ -288,16 +287,6 @@ def bls_switch(fn): return entry -def fork_test(fn): - """ - """ - def entry(*args, **kw): - # override fork test setting - kw['fork_test'] = True - return fn(*args, **kw) - return entry - - def disable_process_reveal_deadlines(fn): """ Decorator to make a function execute with `process_reveal_deadlines` OFF. diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index c5b9b8981..5904805cf 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -1,7 +1,7 @@ from eth2spec.test.context import ( PHASE0, LIGHTCLIENT_PATCH, with_phases, - with_custom_state, fork_test, + with_custom_state, spec_test, with_state, low_balances, misc_balances, large_validator_set, ) @@ -53,7 +53,6 @@ def run_fork_test(spec, pre_state): yield 'post', post_state -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -62,7 +61,6 @@ def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -72,7 +70,6 @@ def test_fork_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -82,7 +79,6 @@ def test_fork_next_epoch_with_block(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @spec_test @with_state @@ -93,7 +89,6 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @@ -102,7 +97,6 @@ def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @@ -111,7 +105,6 @@ def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@fork_test @with_phases(([PHASE0])) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test From 338be1f636b7eae6a6bb3182d4b6682984e788c7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 9 Mar 2021 16:27:06 -0700 Subject: [PATCH 137/140] clean up fork gens --- tests/formats/forks/README.md | 2 +- tests/generators/forks/main.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/formats/forks/README.md b/tests/formats/forks/README.md index 1a7ee64ad..57cc09227 100644 --- a/tests/formats/forks/README.md +++ b/tests/formats/forks/README.md @@ -3,7 +3,7 @@ The aim of the fork tests is to ensure that a pre-fork state can be transformed into a valid post-fork state, utilizing the `upgrade` function found in the relevant `fork.md` spec. -There is only one handler: `core`. Each fork (after genesis) is handled with the same format, +There is only one handler: `fork`. Each fork (after genesis) is handled with the same format, and the particular fork boundary being tested is noted in `meta.yaml`. ## Test case format diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index b22707786..1a603a0dc 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -11,7 +11,7 @@ from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.gen_helpers.gen_from_tests.gen import generate_from_tests -def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -24,8 +24,9 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: runner_name='fork', handler_name='fork', src=tests_src, - fork_name=LIGHTCLIENT_PATCH, - phase=PHASE0, + fork_name=fork_name, + phase=phase, + ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) @@ -33,6 +34,6 @@ def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: if __name__ == "__main__": gen_runner.run_generator("forks", [ - create_provider(test_altair_forks, MINIMAL), - create_provider(test_altair_forks, MAINNET), + create_provider(test_altair_forks, MINIMAL, PHASE0, LIGHTCLIENT_PATCH), + create_provider(test_altair_forks, MAINNET, PHASE0, LIGHTCLIENT_PATCH), ]) From 91f6956b3a0a6f210ec4d865f47836cde11e24a5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 Mar 2021 21:55:50 +0800 Subject: [PATCH 138/140] Bump py_ecc to 5.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6cfa0910a..efe2f379f 100644 --- a/setup.py +++ b/setup.py @@ -581,7 +581,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==5.1.0", + "py_ecc==5.2.0", "milagro_bls_binding==1.6.3", "dataclasses==0.6", "remerkleable==0.1.18", From e792c27c915f96e9b576b0a21f3c2cab73a93894 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 12:27:50 -0700 Subject: [PATCH 139/140] @hwwhww review Co-authored-by: Hsiao-Wei Wang --- .../pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py | 4 ++-- .../eth2spec/test/lightclient_patch/fork/test_fork.py | 8 ++++---- tests/generators/forks/main.py | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 057aa1a2c..c090869d5 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -18,10 +18,10 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any, :param runner_name: to categorize the test in general as. :param handler_name: to categorize the test specialization as. :param src: to retrieve tests from (discovered using inspect.getmembers). - :param fork_name: to run tests against particular phase and/or fork. + :param fork_name: the folder name for these tests. (if multiple forks are applicable, indicate the last fork) :param bls_active: optional, to override BLS switch preference. Defaults to True. - :param phase: optional, specific phase name + :param phase: optional, to run tests against a particular spec version. Default to `fork_name` value. :return: an iterable of test cases. """ fn_names = [ diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index 5904805cf..dada24af9 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -17,10 +17,10 @@ HF1_FORK_TEST_META_TAGS = { } -def run_fork_test(spec, pre_state): +def run_fork_test(post_spec, pre_state): yield 'pre', pre_state - post_state = spec.upgrade_to_lightclient_patch(pre_state) + post_state = post_spec.upgrade_to_lightclient_patch(pre_state) # Stable fields stable_fields = [ @@ -47,8 +47,8 @@ def run_fork_test(spec, pre_state): assert getattr(pre_state, field) != getattr(post_state, field) assert pre_state.fork.current_version == post_state.fork.previous_version - assert post_state.fork.current_version == spec.LIGHTCLIENT_PATCH_FORK_VERSION - assert post_state.fork.epoch == spec.get_current_epoch(post_state) + assert post_state.fork.current_version == post_spec.LIGHTCLIENT_PATCH_FORK_VERSION + assert post_state.fork.epoch == post_spec.get_current_epoch(post_state) yield 'post', post_state diff --git a/tests/generators/forks/main.py b/tests/generators/forks/main.py index 1a603a0dc..190d4620c 100644 --- a/tests/generators/forks/main.py +++ b/tests/generators/forks/main.py @@ -26,7 +26,6 @@ def create_provider(tests_src, config_name: str, phase: str, fork_name: str) -> src=tests_src, fork_name=fork_name, phase=phase, - ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) From 1f3e73703c10c5e9c8c3dc182632a535222a8d7f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 10 Mar 2021 12:38:30 -0700 Subject: [PATCH 140/140] use 'other_phases' for fork tests --- tests/core/pyspec/eth2spec/test/context.py | 11 +++++++---- .../test/lightclient_patch/fork/test_fork.py | 14 +++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1272485de..e197124c0 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -340,7 +340,7 @@ def with_phases(phases, other_phases=None): available_phases = set(run_phases) if other_phases is not None: - available_phases += set(other_phases) + available_phases |= set(other_phases) # TODO: test state is dependent on phase0 but is immediately transitioned to phase1. # A new state-creation helper for phase 1 may be in place, and then phase1+ tests can run without phase0 @@ -348,9 +348,12 @@ def with_phases(phases, other_phases=None): # Populate all phases for multi-phase tests phase_dir = {} - phase_dir[PHASE0] = spec_phase0 - phase_dir[PHASE1] = spec_phase1 - phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch + if PHASE0 in available_phases: + phase_dir[PHASE0] = spec_phase0 + if PHASE1 in available_phases: + phase_dir[PHASE1] = spec_phase1 + if LIGHTCLIENT_PATCH in available_phases: + phase_dir[LIGHTCLIENT_PATCH] = spec_lightclient_patch # return is ignored whenever multiple phases are ran. If if PHASE0 in run_phases: diff --git a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py index dada24af9..27d181510 100644 --- a/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py +++ b/tests/core/pyspec/eth2spec/test/lightclient_patch/fork/test_fork.py @@ -53,7 +53,7 @@ def run_fork_test(post_spec, pre_state): yield 'post', post_state -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -61,7 +61,7 @@ def test_fork_base_state(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -70,7 +70,7 @@ def test_fork_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -79,7 +79,7 @@ def test_fork_next_epoch_with_block(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @spec_test @with_state @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -89,7 +89,7 @@ def test_fork_many_next_epoch(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -97,7 +97,7 @@ def test_fork_random_low_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS) @@ -105,7 +105,7 @@ def test_fork_random_misc_balances(spec, phases, state): yield from run_fork_test(phases[LIGHTCLIENT_PATCH], state) -@with_phases(([PHASE0])) +@with_phases(phases=[PHASE0], other_phases=[LIGHTCLIENT_PATCH]) @with_custom_state(balances_fn=large_validator_set, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @with_meta_tags(HF1_FORK_TEST_META_TAGS)