From b6be9e1830f9fff278c2d247cfce3d843be8e8f7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 31 Mar 2019 04:55:24 -0500 Subject: [PATCH 01/15] Possible aesthetic rework to get_domain In general I dislike how domains, which should be an unobtrusive out-of-the-way thing that we don't think about much, are taking up so much space in code to express, to the point of them being the single thing preventing `bls_verify` from being expressed in one line of code. Here I reorder arguments and add a default, and make `bls_verify` a one-liner. Not necessarily convinced that exactly this approach is the way to go, but IMO it's worth considering. --- specs/core/0_beacon-chain.md | 60 +++++++++--------------------------- 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d08828692..6e2ebbcba 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1121,13 +1121,14 @@ def get_fork_version(fork: Fork, ### `get_domain` ```python -def get_domain(fork: Fork, - epoch: Epoch, - domain_type: int) -> int: +def get_domain(state: BeaconState, + domain_type: int, + epoch=None: int) -> int: """ Get the domain number that represents the fork meta and signature domain. """ - return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) + epoch_of_message = get_current_epoch(state) if epoch is None else epoch + return bytes_to_int(get_fork_version(fork, epoch_of_message) + int_to_bytes4(domain_type)) ``` ### `get_bitfield_bit` @@ -1210,7 +1211,7 @@ def verify_indexed_attestation(state: BeaconState, indexed_attestation: IndexedA hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], signature=indexed_attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(indexed_attestation.data.slot), DOMAIN_ATTESTATION), + domain=get_domain(state, DOMAIN_ATTESTATION, slot_to_epoch(indexed_attestation.data.slot)), ) ``` @@ -1316,17 +1317,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: if pubkey not in validator_pubkeys: # Verify the proof of possession - proof_is_valid = bls_verify( - pubkey=pubkey, - message_hash=signed_root(deposit.data), - signature=deposit.data.proof_of_possession, - domain=get_domain( - state.fork, - get_current_epoch(state), - DOMAIN_DEPOSIT, - ) - ) - if not proof_is_valid: + if not bls_verify(pubkey, signed_root(deposit.data), deposit.data.proof_of_possession, get_domain(state, DOMAIN_DEPOSIT)): return # Add new validator @@ -2194,12 +2185,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] assert not proposer.slashed # Verify proposer signature - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(block), - signature=block.signature, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) - ) + assert bls_verify(proposer.pubkey, signed_root(block), block.signature, get_domain(state, DOMAIN_BEACON_BLOCK)) ``` #### RANDAO @@ -2208,12 +2194,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] # Verify that the provided randao value is valid - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=hash_tree_root(get_current_epoch(state)), - signature=block.body.randao_reveal, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) - ) + assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), block.body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), @@ -2258,12 +2239,8 @@ def process_proposer_slashing(state: BeaconState, assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for header in (proposer_slashing.header_1, proposer_slashing.header_2): - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(header), - signature=header.signature, - domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) - ) + domain = get_domain(state, DOMAIN_BEACON_BLOCK, slot_to_epoch(header.slot)) + assert bls_verify(proposer.pubkey, signed_root(header), header.signature, domain) slash_validator(state, proposer_slashing.proposer_index) ``` @@ -2382,12 +2359,8 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: # Verify the validator has been active long enough assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD # Verify signature - assert bls_verify( - pubkey=validator.pubkey, - message_hash=signed_root(exit), - signature=exit.signature, - domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) - ) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch) + assert bls_verify(validator.pubkey, signed_root(exit), exit.signature, domain) # Initiate exit initiate_validator_exit(state, exit.validator_index) ``` @@ -2427,12 +2400,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] ) # Verify that the signature is valid - assert bls_verify( - pubkey=transfer.pubkey, - message_hash=signed_root(transfer), - signature=transfer.signature, - domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) - ) + assert bls_verify(transfer.pubkey, signed_root(transfer), transfer.signature, get_domain(state, DOMAIN_TRANSFER)) # Process the transfer decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) increase_balance(state, transfer.recipient, transfer.amount) From 79d1f9fb76ae0c82a9e34159ddca63b59ab00fc6 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 31 Mar 2019 20:45:57 +0400 Subject: [PATCH 02/15] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6e2ebbcba..49bbe4d9a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -396,7 +396,7 @@ The types are defined topologically to aid in facilitating an executable version # Amount in Gwei 'amount': 'uint64', # Container self-signature - 'proof_of_possession': 'bytes96', + 'signature': 'bytes96', } ``` @@ -1123,7 +1123,7 @@ def get_fork_version(fork: Fork, ```python def get_domain(state: BeaconState, domain_type: int, - epoch=None: int) -> int: + epoch: int=None) -> int: """ Get the domain number that represents the fork meta and signature domain. """ @@ -1316,8 +1316,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount if pubkey not in validator_pubkeys: - # Verify the proof of possession - if not bls_verify(pubkey, signed_root(deposit.data), deposit.data.proof_of_possession, get_domain(state, DOMAIN_DEPOSIT)): + # Verify the deposit signature (proof of possession) + if not bls_verify(pubkey, signed_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return # Add new validator From c2edcebee3dedf6f6b143cfce9a8c03f6654b6be Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 31 Mar 2019 20:48:44 +0400 Subject: [PATCH 03/15] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 49bbe4d9a..0330ef1f0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -277,7 +277,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. | Name | Value | | - | - | -| `DOMAIN_BEACON_BLOCK` | `0` | +| `DOMAIN_BEACON_PROPOSER` | `0` | | `DOMAIN_RANDAO` | `1` | | `DOMAIN_ATTESTATION` | `2` | | `DOMAIN_DEPOSIT` | `3` | @@ -2185,7 +2185,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] assert not proposer.slashed # Verify proposer signature - assert bls_verify(proposer.pubkey, signed_root(block), block.signature, get_domain(state, DOMAIN_BEACON_BLOCK)) + assert bls_verify(proposer.pubkey, signed_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER)) ``` #### RANDAO @@ -2239,7 +2239,7 @@ def process_proposer_slashing(state: BeaconState, assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for header in (proposer_slashing.header_1, proposer_slashing.header_2): - domain = get_domain(state, DOMAIN_BEACON_BLOCK, slot_to_epoch(header.slot)) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) assert bls_verify(proposer.pubkey, signed_root(header), header.signature, domain) slash_validator(state, proposer_slashing.proposer_index) ``` From 37fc79cb94cc26959f0cb96691583cf4569f08ea Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 2 Apr 2019 22:30:26 +0400 Subject: [PATCH 04/15] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0330ef1f0..094e7e49d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -83,7 +83,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_effective_balance`](#get_effective_balance) - [`get_total_balance`](#get_total_balance) - - [`get_fork_version`](#get_fork_version) - [`get_domain`](#get_domain) - [`get_bitfield_bit`](#get_bitfield_bit) - [`verify_bitfield`](#verify_bitfield) @@ -1104,31 +1103,18 @@ def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> G return sum([get_effective_balance(state, i) for i in validators]) ``` -### `get_fork_version` - -```python -def get_fork_version(fork: Fork, - epoch: Epoch) -> bytes: - """ - Return the fork version of the given ``epoch``. - """ - if epoch < fork.epoch: - return fork.previous_version - else: - return fork.current_version -``` - ### `get_domain` ```python def get_domain(state: BeaconState, domain_type: int, - epoch: int=None) -> int: + message_epoch: int=None) -> int: """ - Get the domain number that represents the fork meta and signature domain. + Return the signature domain (fork version concatenated with domain type) of a message. """ - epoch_of_message = get_current_epoch(state) if epoch is None else epoch - return bytes_to_int(get_fork_version(fork, epoch_of_message) + int_to_bytes4(domain_type)) + epoch = get_current_epoch(state) if message_epoch is None else message_epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return bytes_to_int(fork_version + int_to_bytes4(domain_type)) ``` ### `get_bitfield_bit` From dd520c6162a3a9efc72923ea7d89bf9acf338ce3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 3 Apr 2019 11:22:14 +1100 Subject: [PATCH 05/15] fix tests for get_domain PR --- tests/phase0/helpers.py | 26 ++++++++++++-------------- tests/phase0/test_sanity.py | 6 ++---- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tests/phase0/helpers.py b/tests/phase0/helpers.py index 33f394def..6a7ffd5dd 100644 --- a/tests/phase0/helpers.py +++ b/tests/phase0/helpers.py @@ -50,7 +50,7 @@ pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkey def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): if not deposit_data_leaves: deposit_data_leaves = [] - proof_of_possession = b'\x33' * 96 + signature = b'\x33' * 96 deposit_data_list = [] for i in range(num_validators): @@ -60,7 +60,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=spec.MAX_DEPOSIT_AMOUNT, - proof_of_possession=proof_of_possession, + signature=signature, ) item = hash(deposit_data.serialize()) deposit_data_leaves.append(item) @@ -120,18 +120,17 @@ def build_deposit_data(state, pubkey, privkey, amount): # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=amount, - proof_of_possession=EMPTY_SIGNATURE, + signature=EMPTY_SIGNATURE, ) - proof_of_possession = bls.sign( + signature = bls.sign( message_hash=signed_root(deposit_data), privkey=privkey, domain=get_domain( - state.fork, - get_current_epoch(state), + state, spec.DOMAIN_DEPOSIT, ) ) - deposit_data.proof_of_possession = proof_of_possession + deposit_data.signature = signature return deposit_data @@ -173,9 +172,9 @@ def build_voluntary_exit(state, epoch, validator_index, privkey): message_hash=signed_root(voluntary_exit), privkey=privkey, domain=get_domain( - fork=state.fork, - epoch=epoch, + state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + message_epoch=epoch, ) ) @@ -224,9 +223,8 @@ def get_valid_proposer_slashing(state): header_2.slot = slot + 1 domain = get_domain( - fork=state.fork, - epoch=get_current_epoch(state), - domain_type=spec.DOMAIN_BEACON_BLOCK, + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, ) header_1.signature = bls.sign( message_hash=signed_root(header_1), @@ -307,8 +305,8 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) message_hash=message_hash, privkey=privkey, domain=get_domain( - fork=state.fork, - epoch=slot_to_epoch(attestation_data.slot), + state=state, domain_type=spec.DOMAIN_ATTESTATION, + message_epoch=slot_to_epoch(attestation_data.slot), ) ) diff --git a/tests/phase0/test_sanity.py b/tests/phase0/test_sanity.py index 90825242f..b86187ec8 100644 --- a/tests/phase0/test_sanity.py +++ b/tests/phase0/test_sanity.py @@ -303,8 +303,7 @@ def test_voluntary_exit(state): message_hash=signed_root(voluntary_exit), privkey=privkeys[validator_index], domain=get_domain( - fork=pre_state.fork, - epoch=get_current_epoch(pre_state), + state=pre_state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, ) ) @@ -390,8 +389,7 @@ def test_transfer(state): message_hash=signed_root(transfer), privkey=transfer_privkey, domain=get_domain( - fork=pre_state.fork, - epoch=get_current_epoch(pre_state), + state=pre_state, domain_type=spec.DOMAIN_TRANSFER, ) ) From 24492aa36f51d2a3ee9fa07a057be5454c3be656 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 17 Apr 2019 10:16:01 +1000 Subject: [PATCH 06/15] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0104a16ae..dc359b056 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -379,7 +379,7 @@ The types are defined topologically to aid in facilitating an executable version # Attestation data 'data': AttestationData, # Aggregate signature - 'aggregate_signature': 'bytes96', + 'signature': 'bytes96', } ``` @@ -495,7 +495,7 @@ The types are defined topologically to aid in facilitating an executable version # Custody bitfield 'custody_bitfield': 'bytes', # BLS aggregate signature - 'aggregate_signature': 'bytes96', + 'signature': 'bytes96', } ``` @@ -1159,7 +1159,7 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation): custody_bit_0_indices=custody_bit_0_indices, custody_bit_1_indices=custody_bit_1_indices, data=attestation.data, - aggregate_signature=attestation.aggregate_signature + signature=attestation.signature ) ``` @@ -1198,7 +1198,7 @@ def verify_indexed_attestation(state: BeaconState, indexed_attestation: IndexedA hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], - signature=indexed_attestation.aggregate_signature, + signature=indexed_attestation.signature, domain=get_domain(state, DOMAIN_ATTESTATION, slot_to_epoch(indexed_attestation.data.slot)), ) ``` From 972168d69521e21b1ab25b2f603cab2cb45058f3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 18 Apr 2019 18:49:38 +1000 Subject: [PATCH 07/15] Make type-inference stable on empty lists/vectors --- test_libs/pyspec/eth2spec/utils/minimal_ssz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 8f42f1f65..ff7ab6027 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -262,7 +262,6 @@ def infer_type(value): else: raise Exception("Failed to infer type") - def hash_tree_root(value, typ=None): if typ is None: typ = infer_type(value) From 23d6b468e39fee37b4567f430185e6283869e756 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 16 Apr 2019 20:50:13 +1000 Subject: [PATCH 08/15] Work towards testing all edge-cases of SSZ, for known (static) object types --- scripts/phase0/function_puller.py | 5 + .../{ssz => ssz_generic}/__init__.py | 0 test_generators/{ssz => ssz_generic}/main.py | 2 +- .../{ssz => ssz_generic}/renderers.py | 0 .../{ssz => ssz_generic}/requirements.txt | 0 .../{ssz => ssz_generic}/uint_test_cases.py | 0 test_generators/ssz_static/README.md | 4 + test_generators/ssz_static/__init__.py | 0 test_generators/ssz_static/main.py | 55 ++++++++ test_generators/ssz_static/requirements.txt | 4 + .../pyspec/eth2spec/debug/random_value.py | 126 ++++++++++++++++++ 11 files changed, 195 insertions(+), 1 deletion(-) rename test_generators/{ssz => ssz_generic}/__init__.py (100%) rename test_generators/{ssz => ssz_generic}/main.py (93%) rename test_generators/{ssz => ssz_generic}/renderers.py (100%) rename test_generators/{ssz => ssz_generic}/requirements.txt (100%) rename test_generators/{ssz => ssz_generic}/uint_test_cases.py (100%) create mode 100644 test_generators/ssz_static/README.md create mode 100644 test_generators/ssz_static/__init__.py create mode 100644 test_generators/ssz_static/main.py create mode 100644 test_generators/ssz_static/requirements.txt create mode 100644 test_libs/pyspec/eth2spec/debug/random_value.py diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 59e5b5e24..635797d39 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -62,4 +62,9 @@ def get_spec(file_name: str) -> List[str]: code_lines.append('') for type_line in ssz_type: code_lines.append(' ' + type_line) + code_lines.append('') + code_lines.append('ssz_types = [' + ', '.join([f'\'{ssz_type_name}\'' for (ssz_type_name, _) in type_defs]) + ']') + code_lines.append('') + code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType: return globals()[name]') + code_lines.append('') return code_lines diff --git a/test_generators/ssz/__init__.py b/test_generators/ssz_generic/__init__.py similarity index 100% rename from test_generators/ssz/__init__.py rename to test_generators/ssz_generic/__init__.py diff --git a/test_generators/ssz/main.py b/test_generators/ssz_generic/main.py similarity index 93% rename from test_generators/ssz/main.py rename to test_generators/ssz_generic/main.py index 1c09d51e7..fe01a68d7 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz_generic/main.py @@ -44,4 +44,4 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": - gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) + gen_runner.run_generator("ssz_generic", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) diff --git a/test_generators/ssz/renderers.py b/test_generators/ssz_generic/renderers.py similarity index 100% rename from test_generators/ssz/renderers.py rename to test_generators/ssz_generic/renderers.py diff --git a/test_generators/ssz/requirements.txt b/test_generators/ssz_generic/requirements.txt similarity index 100% rename from test_generators/ssz/requirements.txt rename to test_generators/ssz_generic/requirements.txt diff --git a/test_generators/ssz/uint_test_cases.py b/test_generators/ssz_generic/uint_test_cases.py similarity index 100% rename from test_generators/ssz/uint_test_cases.py rename to test_generators/ssz_generic/uint_test_cases.py diff --git a/test_generators/ssz_static/README.md b/test_generators/ssz_static/README.md new file mode 100644 index 000000000..014c71517 --- /dev/null +++ b/test_generators/ssz_static/README.md @@ -0,0 +1,4 @@ +# SSZ-static + +The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ: + the serialization and hashing of ETH 2.0 data types diff --git a/test_generators/ssz_static/__init__.py b/test_generators/ssz_static/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py new file mode 100644 index 000000000..19942c0e8 --- /dev/null +++ b/test_generators/ssz_static/main.py @@ -0,0 +1,55 @@ +from eth_utils import ( + to_tuple, to_dict +) +from preset_loader import loader +from eth2spec.phase0 import spec +from eth2spec.utils.minimal_ssz import hash_tree_root, serialize +from eth2spec.debug import random_value, encode + +from gen_base import gen_runner, gen_suite, gen_typing +from random import Random + + +@to_dict +def render_test_case(rng: Random, name): + typ = spec.get_ssz_type_by_name(name) + # TODO: vary randomization args + value = random_value.get_random_ssz_object(rng, typ, 100, 10, random_value.RandomizationMode.mode_random, False) + yield "type_name", name + yield "value", encode.encode(value, typ) + yield "serialized", serialize(value) + yield "root", '0x' + hash_tree_root(value).hex() + + +@to_tuple +def ssz_static_cases(rng: Random): + for type_name in spec.ssz_types: + # TODO more types + for i in range(10): + render_test_case(rng, type_name) + + +def min_ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, 'minimal') + spec.apply_constants_preset(presets) + rng = Random(123) + + return ("ssz_min_values_minimal", "core", gen_suite.render_suite( + title="ssz testing, with minimal config", + summary="Test suite for ssz serialization and hash-tree-root", + forks_timeline="testing", + forks=["phase0"], + config="minimal", + runner="ssz", + handler="static", + test_cases=ssz_static_cases(rng))) + +# TODO more suites + +# Variation in: randomization-mode, chaos mode, configuration + + +if __name__ == "__main__": + gen_runner.run_generator("ssz_static", [ + min_ssz_suite + ]) diff --git a/test_generators/ssz_static/requirements.txt b/test_generators/ssz_static/requirements.txt new file mode 100644 index 000000000..8f9bede8f --- /dev/null +++ b/test_generators/ssz_static/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.4.1 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py new file mode 100644 index 000000000..431a4986a --- /dev/null +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -0,0 +1,126 @@ +from random import Random +from typing import Any +from enum import Enum + +UINT_SIZES = [8, 16, 32, 64, 128, 256] + +basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] + +class RandomizationMode(Enum): + # random content / length + mode_random = 0 + # Zero-value + mode_zero = 2 + # Maximum value, limited to count 1 however + mode_max = 3 + # Return 0 values, i.e. empty + mode_nil_count = 4 + # Return 1 value, random content + mode_one_count = 5 + # Return max amount of values, random content + mode_max_count = 6 + +def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any: + """ + Create an object for a given type, filled with random data. + :param rng: The random number generator to use. + :param typ: The type to instantiate + :param max_bytes_length: the max. length for a random bytes array + :param max_list_length: the max. length for a random list + :param mode: how to randomize + :param chaos: if true, the randomization-mode will be randomly changed + :return: the random object instance, of the given type. + """ + if chaos: + mode = rng.choice(list(RandomizationMode)) + if isinstance(typ, str): + # Bytes array + if typ == 'bytes': + if mode == RandomizationMode.mode_nil_count: + return b'' + if mode == RandomizationMode.mode_max_count: + return get_random_bytes_list(rng, max_bytes_length) + if mode == RandomizationMode.mode_one_count: + return get_random_bytes_list(rng, 1) + if mode == RandomizationMode.mode_zero: + return b'\x00' + if mode == RandomizationMode.mode_max: + return b'\xff' + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + elif typ[:5] == 'bytes' and len(typ) > 5: + length = int(typ[5:]) + # Sanity, don't generate absurdly big random values + # If a client is aiming to performance-test, they should create a benchmark suite. + assert length <= max_bytes_length + if mode == RandomizationMode.mode_zero: + return b'\x00' * length + if mode == RandomizationMode.mode_max: + return b'\xff' * length + return get_random_bytes_list(rng, length) + # Basic types + else: + if mode == RandomizationMode.mode_zero: + return get_min_basic_value(typ) + if mode == RandomizationMode.mode_max: + return get_max_basic_value(typ) + return get_random_basic_value(rng, typ) + # Vector: + elif isinstance(typ, list) and len(typ) == 2: + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(typ[1])] + # List: + elif isinstance(typ, list) and len(typ) == 1: + length = rng.randint(0, max_list_length) + if mode == RandomizationMode.mode_one_count: + length = 1 + if mode == RandomizationMode.mode_max_count: + length = max_list_length + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(length)] + # Container: + elif hasattr(typ, 'fields'): + return typ({field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode) for field, subtype in typ.fields.items()}) + else: + print(typ) + raise Exception("Type not recognized") + + +def get_random_bytes_list(rng: Random, length: int) -> bytes: + return bytes(rng.getrandbits(8) for _ in range(length)) + + +def get_random_basic_value(rng: Random, typ: str) -> Any: + if typ == 'bool': + return rng.choice((True, False)) + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in (8, 16, 32, 64, 128, 256) + return rng.randint(0, 2**size - 1) + if typ == 'byte': + return rng.randint(0, 8) + else: + raise ValueError("Not a basic type") + + +def get_min_basic_value(typ: str) -> Any: + if typ == 'bool': + return False + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in (8, 16, 32, 64, 128, 256) + return 0 + if typ == 'byte': + return 0x00 + else: + raise ValueError("Not a basic type") + + +def get_max_basic_value(typ: str) -> Any: + if typ == 'bool': + return True + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in (8, 16, 32, 64, 128, 256) + return 2**size - 1 + if typ == 'byte': + return 0xff + else: + raise ValueError("Not a basic type") From ad30722420ffd09ff3ea79f9dec01857ae0dc5de Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 17 Apr 2019 13:49:29 +1000 Subject: [PATCH 09/15] ssz-static suite --- test_generators/ssz_static/main.py | 84 ++++++++++++------- .../pyspec/eth2spec/debug/random_value.py | 27 ++++-- 2 files changed, 73 insertions(+), 38 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 19942c0e8..2445e4ab7 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -1,55 +1,79 @@ +from random import Random + +from eth2spec.debug import random_value, encode +from eth2spec.phase0 import spec +from eth2spec.utils.minimal_ssz import hash_tree_root, serialize from eth_utils import ( to_tuple, to_dict ) -from preset_loader import loader -from eth2spec.phase0 import spec -from eth2spec.utils.minimal_ssz import hash_tree_root, serialize -from eth2spec.debug import random_value, encode - from gen_base import gen_runner, gen_suite, gen_typing -from random import Random +from preset_loader import loader + +MAX_BYTES_LENGTH = 100 +MAX_LIST_LENGTH = 10 @to_dict -def render_test_case(rng: Random, name): +def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): typ = spec.get_ssz_type_by_name(name) - # TODO: vary randomization args - value = random_value.get_random_ssz_object(rng, typ, 100, 10, random_value.RandomizationMode.mode_random, False) + value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "type_name", name yield "value", encode.encode(value, typ) - yield "serialized", serialize(value) + yield "serialized", serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() @to_tuple -def ssz_static_cases(rng: Random): +def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int): for type_name in spec.ssz_types: - # TODO more types - for i in range(10): - render_test_case(rng, type_name) + for i in range(count): + yield create_test_case(rng, type_name, mode, chaos) -def min_ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - presets = loader.load_presets(configs_path, 'minimal') - spec.apply_constants_preset(presets) - rng = Random(123) +def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int): + def ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + # Apply changes to presets, this affects some of the vector types. + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) - return ("ssz_min_values_minimal", "core", gen_suite.render_suite( - title="ssz testing, with minimal config", - summary="Test suite for ssz serialization and hash-tree-root", - forks_timeline="testing", - forks=["phase0"], - config="minimal", - runner="ssz", - handler="static", - test_cases=ssz_static_cases(rng))) + # Reproducible RNG + rng = Random(seed) -# TODO more suites + random_mode_name = mode.to_name() -# Variation in: randomization-mode, chaos mode, configuration + suite_name = f"ssz_{config_name}_{random_mode_name}{'_chaos' if chaos else ''}" + + count = cases_if_random if chaos or mode.is_changing() else 1 + print(f"generating SSZ-static suite ({count} cases per ssz type): {suite_name}") + + return (suite_name, "core", gen_suite.render_suite( + title=f"ssz testing, with {config_name} config, randomized with mode {random_mode_name}{' and with chaos applied' if chaos else ''}", + summary="Test suite for ssz serialization and hash-tree-root", + forks_timeline="testing", + forks=["phase0"], + config="minimal", + runner="ssz", + handler="static", + test_cases=ssz_static_cases(rng, mode, chaos, count))) + + return ssz_suite if __name__ == "__main__": + # [(seed, config name, randomization mode, chaos on/off, cases_if_random)] + settings = [] + seed = 1 + for mode in random_value.RandomizationMode: + settings.append((seed, "minimal", mode, False, 30)) + seed += 1 + settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) + seed += 1 + settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + seed += 1 + + print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types))) + gen_runner.run_generator("ssz_static", [ - min_ssz_suite + get_ssz_suite(seed, config_name, mode, chaos, cases_if_random) + for (seed, config_name, mode, chaos, cases_if_random) in settings ]) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 431a4986a..59ed5b54b 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,23 +2,34 @@ from random import Random from typing import Any from enum import Enum + UINT_SIZES = [8, 16, 32, 64, 128, 256] basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] +random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] + + class RandomizationMode(Enum): # random content / length mode_random = 0 # Zero-value - mode_zero = 2 + mode_zero = 1 # Maximum value, limited to count 1 however - mode_max = 3 + mode_max = 2 # Return 0 values, i.e. empty - mode_nil_count = 4 + mode_nil_count = 3 # Return 1 value, random content - mode_one_count = 5 + mode_one_count = 4 # Return max amount of values, random content - mode_max_count = 6 + mode_max_count = 5 + + def to_name(self): + return random_mode_names[self.value] + + def is_changing(self): + return self.value in [0, 4, 5] + def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any: """ @@ -66,7 +77,7 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list return get_random_basic_value(rng, typ) # Vector: elif isinstance(typ, list) and len(typ) == 2: - return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(typ[1])] + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(typ[1])] # List: elif isinstance(typ, list) and len(typ) == 1: length = rng.randint(0, max_list_length) @@ -74,10 +85,10 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list length = 1 if mode == RandomizationMode.mode_max_count: length = max_list_length - return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode) for _ in range(length)] + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(length)] # Container: elif hasattr(typ, 'fields'): - return typ({field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode) for field, subtype in typ.fields.items()}) + return typ(**{field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.fields.items()}) else: print(typ) raise Exception("Type not recognized") From 40cc0420abeab2154dac32b94ccf478543cab792 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 17 Apr 2019 19:31:06 +1000 Subject: [PATCH 10/15] fix test output format --- test_generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 2445e4ab7..149b83626 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -19,7 +19,7 @@ def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMod value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield "type_name", name yield "value", encode.encode(value, typ) - yield "serialized", serialize(value).hex() + yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() From 627c1a347af1ca3d7f4494b198928b58e2dc2823 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 18 Apr 2019 11:37:02 +1000 Subject: [PATCH 11/15] make encoder output large uints as string, and fix ssz suite config setting --- test_generators/ssz_static/main.py | 2 +- test_libs/pyspec/eth2spec/debug/encode.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 149b83626..010ca2735 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -51,7 +51,7 @@ def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationM summary="Test suite for ssz serialization and hash-tree-root", forks_timeline="testing", forks=["phase0"], - config="minimal", + config=config_name, runner="ssz", handler="static", test_cases=ssz_static_cases(rng, mode, chaos, count))) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index f50bc9d5e..d3513e638 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -3,6 +3,8 @@ from eth2spec.utils.minimal_ssz import hash_tree_root def encode(value, typ, include_hash_tree_roots=False): if isinstance(typ, str) and typ[:4] == 'uint': + if typ[4:] == '128' or typ[4:] == '256': + return str(value) return value elif typ == 'bool': assert value in (True, False) From bc685133ece8b4deb94602567a6cef2812b40bec Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 18 Apr 2019 18:40:11 +1000 Subject: [PATCH 12/15] Document SSZ testing, and test-suite running --- specs/test_formats/README.md | 21 +++++++++++++++++ specs/test_formats/ssz/README.md | 15 ------------ specs/test_formats/ssz_generic/README.md | 20 ++++++++++++++++ .../test_formats/{ssz => ssz_generic}/uint.md | 0 specs/test_formats/ssz_static/README.md | 8 +++++++ specs/test_formats/ssz_static/core.md | 23 +++++++++++++++++++ 6 files changed, 72 insertions(+), 15 deletions(-) delete mode 100644 specs/test_formats/ssz/README.md create mode 100644 specs/test_formats/ssz_generic/README.md rename specs/test_formats/{ssz => ssz_generic}/uint.md (100%) create mode 100644 specs/test_formats/ssz_static/README.md create mode 100644 specs/test_formats/ssz_static/core.md diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 6b9533056..da2e38c01 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -175,3 +175,24 @@ To prevent parsing of hundreds of different YAML files to test a specific test t │   ... <--- more handlers ... <--- more test types ``` + + +## Note for implementers + +The basic pattern for test-suite loading and running is: + +Iterate suites for given test-type, or sub-type (e.g. `operations > deposits`): +1. Filter test-suite, options: + - Config: Load first few lines, load into YAML, and check `config`, either: + - Pass the suite to the correct compiled target + - Ignore the suite if running tests as part of a compiled target with different configuration + - Load the correct configuration for the suite dynamically before running the suite + - Select by file name + - Filter for specific suites (e.g. for a specific fork) +2. Load the YAML + - Optionally translate the data into applicable naming, e.g. `snake_case` to `PascalCase` +3. Iterate through the `test_cases` +4. Ask test-runner to allocate a new test-case (i.e. objectify the test-case, generalize it with a `TestCase` interface) + Optionally pass raw test-case data to enable dynamic test-case allocation. + 1. Load test-case data into it. + 2. Make the test-case run. diff --git a/specs/test_formats/ssz/README.md b/specs/test_formats/ssz/README.md deleted file mode 100644 index 72ba7dac1..000000000 --- a/specs/test_formats/ssz/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# SSZ tests - -SSZ has changed throughout the development of ETH 2.0. - -## Contents - -A minimal but useful series of tests covering `uint` encoding and decoding is provided. -This is a direct port of the older SSZ `uint` tests (minus outdated test cases). - -[uint test format](./uint.md). - -Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. -The exact uint lengths to support may be redefined in the future. - -Extension of the SSZ tests collection is planned, see CI/testing issues for progress tracking. diff --git a/specs/test_formats/ssz_generic/README.md b/specs/test_formats/ssz_generic/README.md new file mode 100644 index 000000000..9fda0c368 --- /dev/null +++ b/specs/test_formats/ssz_generic/README.md @@ -0,0 +1,20 @@ +# SSZ, generic tests + +This set of test-suites provides general testing for SSZ: + to instantiate any container/list/vector/other type from binary data. + +Since SSZ is in a development-phase, not the full suite of features is covered yet. +Note that these tests are based on the older SSZ package. +The tests are still relevant, but limited in scope: + more complex object encodings have changed since the original SSZ testing. + +A minimal but useful series of tests covering `uint` encoding and decoding is provided. +This is a direct port of the older SSZ `uint` tests (minus outdated test cases). + +[uint test format](./uint.md). + +Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. +The exact uint lengths to support may be redefined in the future. + +Extension of the SSZ tests collection is planned, with an update to the new spec-maintained `minimal_ssz.py`, + see CI/testing issues for progress tracking. diff --git a/specs/test_formats/ssz/uint.md b/specs/test_formats/ssz_generic/uint.md similarity index 100% rename from specs/test_formats/ssz/uint.md rename to specs/test_formats/ssz_generic/uint.md diff --git a/specs/test_formats/ssz_static/README.md b/specs/test_formats/ssz_static/README.md new file mode 100644 index 000000000..413b00c75 --- /dev/null +++ b/specs/test_formats/ssz_static/README.md @@ -0,0 +1,8 @@ +# SSZ, static tests + +This set of test-suites provides static testing for SSZ: + to instantiate just the known ETH-2.0 SSZ types from binary data. + +This series of tests is based on the spec-maintained `minimal_ssz.py`, i.e. fully consistent with the SSZ spec. + +Test format documentation can be found here: [core test format](./core.md). diff --git a/specs/test_formats/ssz_static/core.md b/specs/test_formats/ssz_static/core.md new file mode 100644 index 000000000..8a5067f03 --- /dev/null +++ b/specs/test_formats/ssz_static/core.md @@ -0,0 +1,23 @@ +# Test format: SSZ static types + +The goal of this type is to provide clients with a solid reference how the known SSZ objects should be encoded. +Each object described in the Phase-0 spec is covered. +This is important, as many of the clients aiming to serialize/deserialize objects directly into structs/classes +do not support (or have alternatives for) generic SSZ encoding/decoding. +This test-format ensures these direct serializations are covered. + +## Test case format + +```yaml +type_name: string -- string, object name, formatted as in spec. E.g. "BeaconBlock" +value: dynamic -- the YAML-encoded value, of the type specified by type_name. +serialized: bytes -- string, SSZ-serialized data, hex encoded, with prefix 0x +root: bytes32 -- string, hash-tree-root of the value, hex encoded, with prefix 0x +``` + +## Condition + +A test-runner can implement the following assertions: +- Serialization: After parsing the `value`, SSZ-serialize it: the output should match `serialized` +- Hash-tree-root: After parsing the `value`, Hash-tree-root it: the output should match `root` +- Deserialization: SSZ-deserialize the `serialized` value, and see if it matches the parsed `value` From 3a5243cc898c42c777060aeab724f1bc09bec230 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 19 Apr 2019 12:09:30 +1000 Subject: [PATCH 13/15] apply PR suggestions from djrtwo --- test_libs/pyspec/eth2spec/debug/random_value.py | 6 +++--- test_libs/pyspec/eth2spec/utils/minimal_ssz.py | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 59ed5b54b..a853d2328 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -103,7 +103,7 @@ def get_random_basic_value(rng: Random, typ: str) -> Any: return rng.choice((True, False)) if typ[:4] == 'uint': size = int(typ[4:]) - assert size in (8, 16, 32, 64, 128, 256) + assert size in UINT_SIZES return rng.randint(0, 2**size - 1) if typ == 'byte': return rng.randint(0, 8) @@ -116,7 +116,7 @@ def get_min_basic_value(typ: str) -> Any: return False if typ[:4] == 'uint': size = int(typ[4:]) - assert size in (8, 16, 32, 64, 128, 256) + assert size in UINT_SIZES return 0 if typ == 'byte': return 0x00 @@ -129,7 +129,7 @@ def get_max_basic_value(typ: str) -> Any: return True if typ[:4] == 'uint': size = int(typ[4:]) - assert size in (8, 16, 32, 64, 128, 256) + assert size in UINT_SIZES return 2**size - 1 if typ == 'byte': return 0xff diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index ff7ab6027..dbe9d1359 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -1,7 +1,7 @@ -from .hash_function import hash - from typing import Any +from .hash_function import hash + BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_PREFIX = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK @@ -17,10 +17,7 @@ def SSZType(fields): setattr(self, f, kwargs[f]) def __eq__(self, other): - return ( - self.fields == other.fields and - self.serialize() == other.serialize() - ) + return self.fields == other.fields and self.serialize() == other.serialize() def __hash__(self): return int.from_bytes(self.hash_tree_root(), byteorder="little") @@ -262,6 +259,7 @@ def infer_type(value): else: raise Exception("Failed to infer type") + def hash_tree_root(value, typ=None): if typ is None: typ = infer_type(value) From 39d082260293ca2afb06042ed0087e3026dd7ebc Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 19 Apr 2019 18:26:54 +1000 Subject: [PATCH 14/15] Sane SSZ object default values (#963) --- specs/core/0_beacon-chain.md | 104 ++----------------------- specs/simple-serialize.md | 5 ++ test_generators/operations/deposits.py | 1 - test_libs/pyspec/tests/helpers.py | 9 +-- test_libs/pyspec/tests/test_sanity.py | 3 - 5 files changed, 14 insertions(+), 108 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f04a04877..624413879 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -51,7 +51,6 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) - - [`get_temporary_block_header`](#get_temporary_block_header) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) - [`get_current_epoch`](#get_current_epoch) @@ -201,13 +200,10 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | -| `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` | | `GENESIS_SLOT` | `0` | | `GENESIS_EPOCH` | `0` | -| `GENESIS_START_SHARD` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `int_to_bytes32(0)` | -| `EMPTY_SIGNATURE` | `int_to_bytes96(0)` | | `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes1(0)` | ### Time parameters @@ -640,23 +636,6 @@ Note: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethere `def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. -### `get_temporary_block_header` - -```python -def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: - """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. - """ - return BeaconBlockHeader( - slot=block.slot, - previous_block_root=block.previous_block_root, - state_root=ZERO_HASH, - block_body_root=hash_tree_root(block.body), - # signing_root(block) is used for block id purposes so signature is a stub - signature=EMPTY_SIGNATURE, - ) -``` - ### `slot_to_epoch` ```python @@ -1345,35 +1324,7 @@ When enough full deposits have been made to the deposit contract, an `Eth2Genesi * `genesis_eth1_data.deposit_count` is the `deposit_count` contained in the `Eth2Genesis` log. * `genesis_eth1_data.block_hash` is the hash of the Ethereum 1.0 block that emitted the `Eth2Genesis` log. * Let `genesis_state = get_genesis_beacon_state(genesis_validator_deposits, genesis_time, genesis_eth1_data)`. -* Let `genesis_block = get_empty_block()`. -* Set `genesis_block.state_root = hash_tree_root(genesis_state)`. - -```python -def get_empty_block() -> BeaconBlock: - """ - Get an empty ``BeaconBlock``. - """ - return BeaconBlock( - slot=GENESIS_SLOT, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - body=BeaconBlockBody( - randao_reveal=EMPTY_SIGNATURE, - eth1_data=Eth1Data( - deposit_root=ZERO_HASH, - deposit_count=0, - block_hash=ZERO_HASH, - ), - proposer_slashings=[], - attester_slashings=[], - attestations=[], - deposits=[], - voluntary_exits=[], - transfers=[], - ), - signature=EMPTY_SIGNATURE, - ) -``` +* Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ```python def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], @@ -1382,50 +1333,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], """ Get the genesis ``BeaconState``. """ - state = BeaconState( - # Misc - slot=GENESIS_SLOT, - genesis_time=genesis_time, - fork=Fork( - previous_version=GENESIS_FORK_VERSION, - current_version=GENESIS_FORK_VERSION, - epoch=GENESIS_EPOCH, - ), - - # Validator registry - validator_registry=[], - balances=[], - - # Randomness and committees - latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), - latest_start_shard=GENESIS_START_SHARD, - - # Finality - previous_epoch_attestations=[], - current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH, - current_justified_epoch=GENESIS_EPOCH, - previous_justified_root=ZERO_HASH, - current_justified_root=ZERO_HASH, - justification_bitfield=0, - finalized_epoch=GENESIS_EPOCH, - finalized_root=ZERO_HASH, - - # Recent state - current_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, previous_crosslink_root=ZERO_HASH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), - previous_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, previous_crosslink_root=ZERO_HASH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), - latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]), - latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]), - latest_block_header=get_temporary_block_header(get_empty_block()), - historical_roots=[], - - # Ethereum 1.0 chain data - latest_eth1_data=genesis_eth1_data, - eth1_data_votes=[], - deposit_index=0, - ) + state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) # Process genesis deposits for deposit in genesis_validator_deposits: @@ -1929,7 +1837,11 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the parent matches assert block.previous_block_root == signing_root(state.latest_block_header) # Save current block as the new latest block - state.latest_block_header = get_temporary_block_header(block) + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + previous_block_root=block.previous_block_root, + block_body_root=hash_tree_root(block.body), + ) # Verify proposer is not slashed proposer = state.validator_registry[get_beacon_proposer_index(state)] assert not proposer.slashed @@ -2132,8 +2044,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - slashed=False, - high_balance=0 ) # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 804c66d70..f30d34709 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -9,6 +9,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Basic types](#basic-types) - [Composite types](#composite-types) - [Aliases](#aliases) + - [Default values](#default-values) - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) @@ -50,6 +51,10 @@ For convenience we alias: * `"bytes"` to `["byte"]` (this is *not* a basic type) * `"bytesN"` to `["byte", N]` (this is *not* a basic type) +### Default values + +The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. + ## Serialization We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 85c93f86b..454c6f22d 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -24,7 +24,6 @@ def build_deposit_data(state, pubkey=pubkey, withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:], amount=amount, - proof_of_possession=spec.EMPTY_SIGNATURE, ) deposit_data.proof_of_possession = bls.sign( message_hash=signing_root(deposit_data), diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index e04409792..be43ac6aa 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -9,13 +9,13 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants - EMPTY_SIGNATURE, ZERO_HASH, # SSZ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, + BeaconBlock, BeaconBlockHeader, Deposit, DepositData, @@ -30,7 +30,6 @@ from eth2spec.phase0.spec import ( get_crosslink_committees_at_slot, get_current_epoch, get_domain, - get_empty_block, get_epoch_start_slot, get_genesis_beacon_state, get_previous_epoch, @@ -115,7 +114,7 @@ def create_genesis_state(num_validators, deposit_data_leaves=None): def build_empty_block_for_next_slot(state): - empty_block = get_empty_block() + empty_block = BeaconBlock() empty_block.slot = state.slot + 1 previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: @@ -130,7 +129,6 @@ def build_deposit_data(state, pubkey, privkey, amount): # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=amount, - signature=EMPTY_SIGNATURE, ) signature = bls.sign( message_hash=signing_root(deposit_data), @@ -185,7 +183,6 @@ def build_voluntary_exit(state, epoch, validator_index, privkey): voluntary_exit = VoluntaryExit( epoch=epoch, validator_index=validator_index, - signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), @@ -235,7 +232,6 @@ def get_valid_proposer_slashing(state): previous_block_root=ZERO_HASH, state_root=ZERO_HASH, block_body_root=ZERO_HASH, - signature=EMPTY_SIGNATURE, ) header_2 = deepcopy(header_1) header_2.previous_block_root = b'\x02' * 32 @@ -304,7 +300,6 @@ def get_valid_attestation(state, slot=None): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=EMPTY_SIGNATURE, ) participants = get_attesting_indices( state, diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 7ddd4d386..ba9dc6df6 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -8,7 +8,6 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants - EMPTY_SIGNATURE, ZERO_HASH, # SSZ Deposit, @@ -350,7 +349,6 @@ def test_voluntary_exit(state): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), @@ -398,7 +396,6 @@ def test_transfer(state): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=EMPTY_SIGNATURE, ) transfer.signature = bls.sign( message_hash=signing_root(transfer), From 8c59bfd9be703a6b31c969e46a2a26d4c0c8b51f Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Sat, 20 Apr 2019 00:18:14 -0500 Subject: [PATCH 15/15] Update simple-serialize.md (#969) --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index f30d34709..6ccb8f22d 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -34,11 +34,11 @@ This is a **work in progress** describing typing, serialization and Merkleizatio ### Composite types -* **container**: ordered heterogenous collection of values +* **container**: ordered heterogeneous collection of values * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` * **vector**: ordered fixed-length homogeneous collection of values * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogenous collection of values +* **list**: ordered variable-length homogeneous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size".