diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 59e5b5e24..635797d39 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -62,4 +62,9 @@ def get_spec(file_name: str) -> List[str]: code_lines.append('') for type_line in ssz_type: code_lines.append(' ' + type_line) + code_lines.append('') + code_lines.append('ssz_types = [' + ', '.join([f'\'{ssz_type_name}\'' for (ssz_type_name, _) in type_defs]) + ']') + code_lines.append('') + code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType: return globals()[name]') + code_lines.append('') return code_lines diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ba17691bc..27e3b8663 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -51,7 +51,6 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) - - [`get_temporary_block_header`](#get_temporary_block_header) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) - [`get_current_epoch`](#get_current_epoch) @@ -79,7 +78,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_effective_balance`](#get_effective_balance) - [`get_total_balance`](#get_total_balance) - - [`get_fork_version`](#get_fork_version) - [`get_domain`](#get_domain) - [`get_bitfield_bit`](#get_bitfield_bit) - [`verify_bitfield`](#verify_bitfield) @@ -201,13 +199,10 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | -| `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` | | `GENESIS_SLOT` | `0` | | `GENESIS_EPOCH` | `0` | -| `GENESIS_START_SHARD` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `int_to_bytes32(0)` | -| `EMPTY_SIGNATURE` | `int_to_bytes96(0)` | | `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes1(0)` | ### Time parameters @@ -264,7 +259,7 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | -| `DOMAIN_BEACON_BLOCK` | `0` | +| `DOMAIN_BEACON_PROPOSER` | `0` | | `DOMAIN_RANDAO` | `1` | | `DOMAIN_ATTESTATION` | `2` | | `DOMAIN_DEPOSIT` | `3` | @@ -359,7 +354,7 @@ The types are defined topologically to aid in facilitating an executable version # Attestation data 'data': AttestationData, # Aggregate signature - 'aggregate_signature': 'bytes96', + 'signature': 'bytes96', } ``` @@ -374,7 +369,7 @@ The types are defined topologically to aid in facilitating an executable version # Amount in Gwei 'amount': 'uint64', # Container self-signature - 'proof_of_possession': 'bytes96', + 'signature': 'bytes96', } ``` @@ -473,7 +468,7 @@ The types are defined topologically to aid in facilitating an executable version # Custody bitfield 'custody_bitfield': 'bytes', # BLS aggregate signature - 'aggregate_signature': 'bytes96', + 'signature': 'bytes96', } ``` @@ -641,23 +636,6 @@ Note: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethere `def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. -### `get_temporary_block_header` - -```python -def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: - """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. - """ - return BeaconBlockHeader( - slot=block.slot, - previous_block_root=block.previous_block_root, - state_root=ZERO_HASH, - block_body_root=hash_tree_root(block.body), - # signing_root(block) is used for block id purposes so signature is a stub - signature=EMPTY_SIGNATURE, - ) -``` - ### `slot_to_epoch` ```python @@ -900,7 +878,7 @@ def get_block_root(state: BeaconState, return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` -`get_block_root(_, s)` should always return `signed_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. +`get_block_root(_, s)` should always return `signing_root` of the block in the beacon chain at slot `s`, and `get_crosslink_committees_at_slot(_, s)` should not change unless the [validator](#dfn-validator) registry changes. ### `get_state_root` @@ -1033,30 +1011,18 @@ def get_total_balance(state: BeaconState, validators: List[ValidatorIndex], epoc return sum([get_effective_balance(state, i, epoch) for i in validators]) ``` -### `get_fork_version` - -```python -def get_fork_version(fork: Fork, - epoch: Epoch) -> bytes: - """ - Return the fork version of the given ``epoch``. - """ - if epoch < fork.epoch: - return fork.previous_version - else: - return fork.current_version -``` - ### `get_domain` ```python -def get_domain(fork: Fork, - epoch: Epoch, - domain_type: int) -> int: +def get_domain(state: BeaconState, + domain_type: int, + message_epoch: int=None) -> int: """ - Get the domain number that represents the fork meta and signature domain. + Return the signature domain (fork version concatenated with domain type) of a message. """ - return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) + epoch = get_current_epoch(state) if message_epoch is None else message_epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return bytes_to_int(fork_version + int_to_bytes4(domain_type)) ``` ### `get_bitfield_bit` @@ -1102,7 +1068,7 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA custody_bit_0_indices=custody_bit_0_indices, custody_bit_1_indices=custody_bit_1_indices, data=attestation.data, - aggregate_signature=attestation.aggregate_signature, + signature=attestation.signature, ) ``` @@ -1140,8 +1106,8 @@ def verify_indexed_attestation(state: BeaconState, indexed_attestation: IndexedA hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), ], - signature=indexed_attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(indexed_attestation.data.slot), DOMAIN_ATTESTATION), + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, slot_to_epoch(indexed_attestation.data.slot)), ) ``` @@ -1346,35 +1312,7 @@ When enough full deposits have been made to the deposit contract, an `Eth2Genesi * `genesis_eth1_data.deposit_count` is the `deposit_count` contained in the `Eth2Genesis` log. * `genesis_eth1_data.block_hash` is the hash of the Ethereum 1.0 block that emitted the `Eth2Genesis` log. * Let `genesis_state = get_genesis_beacon_state(genesis_validator_deposits, genesis_time, genesis_eth1_data)`. -* Let `genesis_block = get_empty_block()`. -* Set `genesis_block.state_root = hash_tree_root(genesis_state)`. - -```python -def get_empty_block() -> BeaconBlock: - """ - Get an empty ``BeaconBlock``. - """ - return BeaconBlock( - slot=GENESIS_SLOT, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - body=BeaconBlockBody( - randao_reveal=EMPTY_SIGNATURE, - eth1_data=Eth1Data( - deposit_root=ZERO_HASH, - deposit_count=0, - block_hash=ZERO_HASH, - ), - proposer_slashings=[], - attester_slashings=[], - attestations=[], - deposits=[], - voluntary_exits=[], - transfers=[], - ), - signature=EMPTY_SIGNATURE, - ) -``` +* Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ```python def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], @@ -1383,50 +1321,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], """ Get the genesis ``BeaconState``. """ - state = BeaconState( - # Misc - slot=GENESIS_SLOT, - genesis_time=genesis_time, - fork=Fork( - previous_version=GENESIS_FORK_VERSION, - current_version=GENESIS_FORK_VERSION, - epoch=GENESIS_EPOCH, - ), - - # Validator registry - validator_registry=[], - balances=[], - - # Randomness and committees - latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), - latest_start_shard=GENESIS_START_SHARD, - - # Finality - previous_epoch_attestations=[], - current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH, - current_justified_epoch=GENESIS_EPOCH, - previous_justified_root=ZERO_HASH, - current_justified_root=ZERO_HASH, - justification_bitfield=0, - finalized_epoch=GENESIS_EPOCH, - finalized_root=ZERO_HASH, - - # Recent state - current_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, previous_crosslink_root=ZERO_HASH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), - previous_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, previous_crosslink_root=ZERO_HASH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), - latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]), - latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]), - latest_block_header=get_temporary_block_header(get_empty_block()), - historical_roots=[], - - # Ethereum 1.0 chain data - latest_eth1_data=genesis_eth1_data, - eth1_data_votes=[], - deposit_index=0, - ) + state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) # Process genesis deposits for deposit in genesis_validator_deposits: @@ -1908,17 +1803,16 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the parent matches assert block.previous_block_root == signing_root(state.latest_block_header) # Save current block as the new latest block - state.latest_block_header = get_temporary_block_header(block) + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + previous_block_root=block.previous_block_root, + block_body_root=hash_tree_root(block.body), + ) # Verify proposer is not slashed proposer = state.validator_registry[get_beacon_proposer_index(state)] assert not proposer.slashed # Verify proposer signature - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signing_root(block), - signature=block.signature, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) - ) + assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER)) ``` #### RANDAO @@ -1927,12 +1821,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=hash_tree_root(get_current_epoch(state)), - signature=block.body.randao_reveal, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) - ) + assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), block.body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), @@ -1973,12 +1862,9 @@ def process_proposer_slashing(state: BeaconState, assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for header in (proposer_slashing.header_1, proposer_slashing.header_2): - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signing_root(header), - signature=header.signature, - domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) - ) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot)) + assert bls_verify(proposer.pubkey, signing_root(header), header.signature, domain) + slash_validator(state, proposer_slashing.proposer_index) ``` @@ -2100,18 +1986,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount if pubkey not in validator_pubkeys: - # Verify the proof of possession - proof_is_valid = bls_verify( - pubkey=pubkey, - message_hash=signing_root(deposit.data), - signature=deposit.data.proof_of_possession, - domain=get_domain( - state.fork, - get_current_epoch(state), - DOMAIN_DEPOSIT, - ) - ) - if not proof_is_valid: + # Verify the deposit signature (proof of possession) + if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return # Add new validator @@ -2122,7 +1998,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - slashed=False, effective_balance=amount - amount % HIGH_BALANCE_INCREMENT, ) @@ -2156,12 +2031,8 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: # Verify the validator has been active long enough assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD # Verify signature - assert bls_verify( - pubkey=validator.pubkey, - message_hash=signing_root(exit), - signature=exit.signature, - domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) - ) + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch) + assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain) # Initiate exit initiate_validator_exit(state, exit.validator_index) ``` @@ -2195,12 +2066,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] ) # Verify that the signature is valid - assert bls_verify( - pubkey=transfer.pubkey, - message_hash=signing_root(transfer), - signature=transfer.signature, - domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) - ) + assert bls_verify(transfer.pubkey, signing_root(transfer), transfer.signature, get_domain(state, DOMAIN_TRANSFER)) # Process the transfer decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) increase_balance(state, transfer.recipient, transfer.amount) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 804c66d70..6ccb8f22d 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -9,6 +9,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Basic types](#basic-types) - [Composite types](#composite-types) - [Aliases](#aliases) + - [Default values](#default-values) - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) @@ -33,11 +34,11 @@ This is a **work in progress** describing typing, serialization and Merkleizatio ### Composite types -* **container**: ordered heterogenous collection of values +* **container**: ordered heterogeneous collection of values * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` * **vector**: ordered fixed-length homogeneous collection of values * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogenous collection of values +* **list**: ordered variable-length homogeneous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size". @@ -50,6 +51,10 @@ For convenience we alias: * `"bytes"` to `["byte"]` (this is *not* a basic type) * `"bytesN"` to `["byte", N]` (this is *not* a basic type) +### Default values + +The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. + ## Serialization We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 6b9533056..da2e38c01 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -175,3 +175,24 @@ To prevent parsing of hundreds of different YAML files to test a specific test t │   ... <--- more handlers ... <--- more test types ``` + + +## Note for implementers + +The basic pattern for test-suite loading and running is: + +Iterate suites for given test-type, or sub-type (e.g. `operations > deposits`): +1. Filter test-suite, options: + - Config: Load first few lines, load into YAML, and check `config`, either: + - Pass the suite to the correct compiled target + - Ignore the suite if running tests as part of a compiled target with different configuration + - Load the correct configuration for the suite dynamically before running the suite + - Select by file name + - Filter for specific suites (e.g. for a specific fork) +2. Load the YAML + - Optionally translate the data into applicable naming, e.g. `snake_case` to `PascalCase` +3. Iterate through the `test_cases` +4. Ask test-runner to allocate a new test-case (i.e. objectify the test-case, generalize it with a `TestCase` interface) + Optionally pass raw test-case data to enable dynamic test-case allocation. + 1. Load test-case data into it. + 2. Make the test-case run. diff --git a/specs/test_formats/ssz/README.md b/specs/test_formats/ssz/README.md deleted file mode 100644 index 72ba7dac1..000000000 --- a/specs/test_formats/ssz/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# SSZ tests - -SSZ has changed throughout the development of ETH 2.0. - -## Contents - -A minimal but useful series of tests covering `uint` encoding and decoding is provided. -This is a direct port of the older SSZ `uint` tests (minus outdated test cases). - -[uint test format](./uint.md). - -Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. -The exact uint lengths to support may be redefined in the future. - -Extension of the SSZ tests collection is planned, see CI/testing issues for progress tracking. diff --git a/specs/test_formats/ssz_generic/README.md b/specs/test_formats/ssz_generic/README.md new file mode 100644 index 000000000..9fda0c368 --- /dev/null +++ b/specs/test_formats/ssz_generic/README.md @@ -0,0 +1,20 @@ +# SSZ, generic tests + +This set of test-suites provides general testing for SSZ: + to instantiate any container/list/vector/other type from binary data. + +Since SSZ is in a development-phase, not the full suite of features is covered yet. +Note that these tests are based on the older SSZ package. +The tests are still relevant, but limited in scope: + more complex object encodings have changed since the original SSZ testing. + +A minimal but useful series of tests covering `uint` encoding and decoding is provided. +This is a direct port of the older SSZ `uint` tests (minus outdated test cases). + +[uint test format](./uint.md). + +Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. +The exact uint lengths to support may be redefined in the future. + +Extension of the SSZ tests collection is planned, with an update to the new spec-maintained `minimal_ssz.py`, + see CI/testing issues for progress tracking. diff --git a/specs/test_formats/ssz/uint.md b/specs/test_formats/ssz_generic/uint.md similarity index 100% rename from specs/test_formats/ssz/uint.md rename to specs/test_formats/ssz_generic/uint.md diff --git a/specs/test_formats/ssz_static/README.md b/specs/test_formats/ssz_static/README.md new file mode 100644 index 000000000..413b00c75 --- /dev/null +++ b/specs/test_formats/ssz_static/README.md @@ -0,0 +1,8 @@ +# SSZ, static tests + +This set of test-suites provides static testing for SSZ: + to instantiate just the known ETH-2.0 SSZ types from binary data. + +This series of tests is based on the spec-maintained `minimal_ssz.py`, i.e. fully consistent with the SSZ spec. + +Test format documentation can be found here: [core test format](./core.md). diff --git a/specs/test_formats/ssz_static/core.md b/specs/test_formats/ssz_static/core.md new file mode 100644 index 000000000..8a5067f03 --- /dev/null +++ b/specs/test_formats/ssz_static/core.md @@ -0,0 +1,23 @@ +# Test format: SSZ static types + +The goal of this type is to provide clients with a solid reference how the known SSZ objects should be encoded. +Each object described in the Phase-0 spec is covered. +This is important, as many of the clients aiming to serialize/deserialize objects directly into structs/classes +do not support (or have alternatives for) generic SSZ encoding/decoding. +This test-format ensures these direct serializations are covered. + +## Test case format + +```yaml +type_name: string -- string, object name, formatted as in spec. E.g. "BeaconBlock" +value: dynamic -- the YAML-encoded value, of the type specified by type_name. +serialized: bytes -- string, SSZ-serialized data, hex encoded, with prefix 0x +root: bytes32 -- string, hash-tree-root of the value, hex encoded, with prefix 0x +``` + +## Condition + +A test-runner can implement the following assertions: +- Serialization: After parsing the `value`, SSZ-serialize it: the output should match `serialized` +- Hash-tree-root: After parsing the `value`, Hash-tree-root it: the output should match `root` +- Deserialization: SSZ-deserialize the `serialized` value, and see if it matches the parsed `value` diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 85c93f86b..454c6f22d 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -24,7 +24,6 @@ def build_deposit_data(state, pubkey=pubkey, withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:], amount=amount, - proof_of_possession=spec.EMPTY_SIGNATURE, ) deposit_data.proof_of_possession = bls.sign( message_hash=signing_root(deposit_data), diff --git a/test_generators/ssz/__init__.py b/test_generators/ssz_generic/__init__.py similarity index 100% rename from test_generators/ssz/__init__.py rename to test_generators/ssz_generic/__init__.py diff --git a/test_generators/ssz/main.py b/test_generators/ssz_generic/main.py similarity index 93% rename from test_generators/ssz/main.py rename to test_generators/ssz_generic/main.py index 1c09d51e7..fe01a68d7 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz_generic/main.py @@ -44,4 +44,4 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": - gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) + gen_runner.run_generator("ssz_generic", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) diff --git a/test_generators/ssz/renderers.py b/test_generators/ssz_generic/renderers.py similarity index 100% rename from test_generators/ssz/renderers.py rename to test_generators/ssz_generic/renderers.py diff --git a/test_generators/ssz/requirements.txt b/test_generators/ssz_generic/requirements.txt similarity index 100% rename from test_generators/ssz/requirements.txt rename to test_generators/ssz_generic/requirements.txt diff --git a/test_generators/ssz/uint_test_cases.py b/test_generators/ssz_generic/uint_test_cases.py similarity index 100% rename from test_generators/ssz/uint_test_cases.py rename to test_generators/ssz_generic/uint_test_cases.py diff --git a/test_generators/ssz_static/README.md b/test_generators/ssz_static/README.md new file mode 100644 index 000000000..014c71517 --- /dev/null +++ b/test_generators/ssz_static/README.md @@ -0,0 +1,4 @@ +# SSZ-static + +The purpose of this test-generator is to provide test-vectors for the most important applications of SSZ: + the serialization and hashing of ETH 2.0 data types diff --git a/test_generators/ssz_static/__init__.py b/test_generators/ssz_static/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py new file mode 100644 index 000000000..010ca2735 --- /dev/null +++ b/test_generators/ssz_static/main.py @@ -0,0 +1,79 @@ +from random import Random + +from eth2spec.debug import random_value, encode +from eth2spec.phase0 import spec +from eth2spec.utils.minimal_ssz import hash_tree_root, serialize +from eth_utils import ( + to_tuple, to_dict +) +from gen_base import gen_runner, gen_suite, gen_typing +from preset_loader import loader + +MAX_BYTES_LENGTH = 100 +MAX_LIST_LENGTH = 10 + + +@to_dict +def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): + typ = spec.get_ssz_type_by_name(name) + value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) + yield "type_name", name + yield "value", encode.encode(value, typ) + yield "serialized", '0x' + serialize(value).hex() + yield "root", '0x' + hash_tree_root(value).hex() + + +@to_tuple +def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int): + for type_name in spec.ssz_types: + for i in range(count): + yield create_test_case(rng, type_name, mode, chaos) + + +def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int): + def ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + # Apply changes to presets, this affects some of the vector types. + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + # Reproducible RNG + rng = Random(seed) + + random_mode_name = mode.to_name() + + suite_name = f"ssz_{config_name}_{random_mode_name}{'_chaos' if chaos else ''}" + + count = cases_if_random if chaos or mode.is_changing() else 1 + print(f"generating SSZ-static suite ({count} cases per ssz type): {suite_name}") + + return (suite_name, "core", gen_suite.render_suite( + title=f"ssz testing, with {config_name} config, randomized with mode {random_mode_name}{' and with chaos applied' if chaos else ''}", + summary="Test suite for ssz serialization and hash-tree-root", + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="ssz", + handler="static", + test_cases=ssz_static_cases(rng, mode, chaos, count))) + + return ssz_suite + + +if __name__ == "__main__": + # [(seed, config name, randomization mode, chaos on/off, cases_if_random)] + settings = [] + seed = 1 + for mode in random_value.RandomizationMode: + settings.append((seed, "minimal", mode, False, 30)) + seed += 1 + settings.append((seed, "minimal", random_value.RandomizationMode.mode_random, True, 30)) + seed += 1 + settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) + seed += 1 + + print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types))) + + gen_runner.run_generator("ssz_static", [ + get_ssz_suite(seed, config_name, mode, chaos, cases_if_random) + for (seed, config_name, mode, chaos, cases_if_random) in settings + ]) diff --git a/test_generators/ssz_static/requirements.txt b/test_generators/ssz_static/requirements.txt new file mode 100644 index 000000000..8f9bede8f --- /dev/null +++ b/test_generators/ssz_static/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.4.1 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index f50bc9d5e..d3513e638 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -3,6 +3,8 @@ from eth2spec.utils.minimal_ssz import hash_tree_root def encode(value, typ, include_hash_tree_roots=False): if isinstance(typ, str) and typ[:4] == 'uint': + if typ[4:] == '128' or typ[4:] == '256': + return str(value) return value elif typ == 'bool': assert value in (True, False) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py new file mode 100644 index 000000000..a853d2328 --- /dev/null +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -0,0 +1,137 @@ +from random import Random +from typing import Any +from enum import Enum + + +UINT_SIZES = [8, 16, 32, 64, 128, 256] + +basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] + +random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] + + +class RandomizationMode(Enum): + # random content / length + mode_random = 0 + # Zero-value + mode_zero = 1 + # Maximum value, limited to count 1 however + mode_max = 2 + # Return 0 values, i.e. empty + mode_nil_count = 3 + # Return 1 value, random content + mode_one_count = 4 + # Return max amount of values, random content + mode_max_count = 5 + + def to_name(self): + return random_mode_names[self.value] + + def is_changing(self): + return self.value in [0, 4, 5] + + +def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any: + """ + Create an object for a given type, filled with random data. + :param rng: The random number generator to use. + :param typ: The type to instantiate + :param max_bytes_length: the max. length for a random bytes array + :param max_list_length: the max. length for a random list + :param mode: how to randomize + :param chaos: if true, the randomization-mode will be randomly changed + :return: the random object instance, of the given type. + """ + if chaos: + mode = rng.choice(list(RandomizationMode)) + if isinstance(typ, str): + # Bytes array + if typ == 'bytes': + if mode == RandomizationMode.mode_nil_count: + return b'' + if mode == RandomizationMode.mode_max_count: + return get_random_bytes_list(rng, max_bytes_length) + if mode == RandomizationMode.mode_one_count: + return get_random_bytes_list(rng, 1) + if mode == RandomizationMode.mode_zero: + return b'\x00' + if mode == RandomizationMode.mode_max: + return b'\xff' + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + elif typ[:5] == 'bytes' and len(typ) > 5: + length = int(typ[5:]) + # Sanity, don't generate absurdly big random values + # If a client is aiming to performance-test, they should create a benchmark suite. + assert length <= max_bytes_length + if mode == RandomizationMode.mode_zero: + return b'\x00' * length + if mode == RandomizationMode.mode_max: + return b'\xff' * length + return get_random_bytes_list(rng, length) + # Basic types + else: + if mode == RandomizationMode.mode_zero: + return get_min_basic_value(typ) + if mode == RandomizationMode.mode_max: + return get_max_basic_value(typ) + return get_random_basic_value(rng, typ) + # Vector: + elif isinstance(typ, list) and len(typ) == 2: + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(typ[1])] + # List: + elif isinstance(typ, list) and len(typ) == 1: + length = rng.randint(0, max_list_length) + if mode == RandomizationMode.mode_one_count: + length = 1 + if mode == RandomizationMode.mode_max_count: + length = max_list_length + return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(length)] + # Container: + elif hasattr(typ, 'fields'): + return typ(**{field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.fields.items()}) + else: + print(typ) + raise Exception("Type not recognized") + + +def get_random_bytes_list(rng: Random, length: int) -> bytes: + return bytes(rng.getrandbits(8) for _ in range(length)) + + +def get_random_basic_value(rng: Random, typ: str) -> Any: + if typ == 'bool': + return rng.choice((True, False)) + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in UINT_SIZES + return rng.randint(0, 2**size - 1) + if typ == 'byte': + return rng.randint(0, 8) + else: + raise ValueError("Not a basic type") + + +def get_min_basic_value(typ: str) -> Any: + if typ == 'bool': + return False + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in UINT_SIZES + return 0 + if typ == 'byte': + return 0x00 + else: + raise ValueError("Not a basic type") + + +def get_max_basic_value(typ: str) -> Any: + if typ == 'bool': + return True + if typ[:4] == 'uint': + size = int(typ[4:]) + assert size in UINT_SIZES + return 2**size - 1 + if typ == 'byte': + return 0xff + else: + raise ValueError("Not a basic type") diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 8f42f1f65..dbe9d1359 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -1,7 +1,7 @@ -from .hash_function import hash - from typing import Any +from .hash_function import hash + BYTES_PER_CHUNK = 32 BYTES_PER_LENGTH_PREFIX = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK @@ -17,10 +17,7 @@ def SSZType(fields): setattr(self, f, kwargs[f]) def __eq__(self, other): - return ( - self.fields == other.fields and - self.serialize() == other.serialize() - ) + return self.fields == other.fields and self.serialize() == other.serialize() def __hash__(self): return int.from_bytes(self.hash_tree_root(), byteorder="little") diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 616f3b797..249dcbb52 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -9,13 +9,13 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants - EMPTY_SIGNATURE, ZERO_HASH, # SSZ Attestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, + BeaconBlock, BeaconBlockHeader, Deposit, DepositData, @@ -30,7 +30,6 @@ from eth2spec.phase0.spec import ( get_crosslink_committees_at_slot, get_current_epoch, get_domain, - get_empty_block, get_epoch_start_slot, get_genesis_beacon_state, get_previous_epoch, @@ -70,7 +69,7 @@ def set_bitfield_bit(bitfield, i): def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): if not deposit_data_leaves: deposit_data_leaves = [] - proof_of_possession = b'\x33' * 96 + signature = b'\x33' * 96 deposit_data_list = [] for i in range(num_validators): @@ -80,7 +79,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=spec.MAX_DEPOSIT_AMOUNT, - proof_of_possession=proof_of_possession, + signature=signature, ) item = hash(deposit_data.serialize()) deposit_data_leaves.append(item) @@ -117,7 +116,7 @@ def create_genesis_state(num_validators, deposit_data_leaves=None): def build_empty_block_for_next_slot(state): - empty_block = get_empty_block() + empty_block = BeaconBlock() empty_block.slot = state.slot + 1 previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: @@ -132,18 +131,16 @@ def build_deposit_data(state, pubkey, privkey, amount): # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=amount, - proof_of_possession=EMPTY_SIGNATURE, ) - proof_of_possession = bls.sign( + signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, domain=get_domain( - state.fork, - get_current_epoch(state), + state, spec.DOMAIN_DEPOSIT, ) ) - deposit_data.proof_of_possession = proof_of_possession + deposit_data.signature = signature return deposit_data @@ -188,15 +185,14 @@ def build_voluntary_exit(state, epoch, validator_index, privkey): voluntary_exit = VoluntaryExit( epoch=epoch, validator_index=validator_index, - signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), privkey=privkey, domain=get_domain( - fork=state.fork, - epoch=epoch, + state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + message_epoch=epoch, ) ) @@ -238,16 +234,14 @@ def get_valid_proposer_slashing(state): previous_block_root=ZERO_HASH, state_root=ZERO_HASH, block_body_root=ZERO_HASH, - signature=EMPTY_SIGNATURE, ) header_2 = deepcopy(header_1) header_2.previous_block_root = b'\x02' * 32 header_2.slot = slot + 1 domain = get_domain( - fork=state.fork, - epoch=get_current_epoch(state), - domain_type=spec.DOMAIN_BEACON_BLOCK, + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, ) header_1.signature = bls.sign( message_hash=signing_root(header_1), @@ -308,7 +302,6 @@ def get_valid_attestation(state, slot=None): aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, - aggregate_signature=EMPTY_SIGNATURE, ) participants = get_attesting_indices( state, @@ -342,9 +335,9 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) message_hash=message_hash, privkey=privkey, domain=get_domain( - fork=state.fork, - epoch=slot_to_epoch(attestation_data.slot), + state=state, domain_type=spec.DOMAIN_ATTESTATION, + message_epoch=slot_to_epoch(attestation_data.slot), ) ) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 508b07905..f87a7c808 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -8,7 +8,6 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants - EMPTY_SIGNATURE, ZERO_HASH, # SSZ Deposit, @@ -348,14 +347,12 @@ def test_voluntary_exit(state): voluntary_exit = VoluntaryExit( epoch=get_current_epoch(pre_state), validator_index=validator_index, - signature=EMPTY_SIGNATURE, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), privkey=privkeys[validator_index], domain=get_domain( - fork=pre_state.fork, - epoch=get_current_epoch(pre_state), + state=pre_state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, ) ) @@ -397,14 +394,12 @@ def test_transfer(state): fee=0, slot=pre_state.slot + 1, pubkey=transfer_pubkey, - signature=EMPTY_SIGNATURE, ) transfer.signature = bls.sign( message_hash=signing_root(transfer), privkey=transfer_privkey, domain=get_domain( - fork=pre_state.fork, - epoch=get_current_epoch(pre_state), + state=pre_state, domain_type=spec.DOMAIN_TRANSFER, ) )