From c4bf772d30b4a557c484e9527c2c670cdbacfe89 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Jun 2019 17:26:39 -0600 Subject: [PATCH 01/12] demo phase restricted generators for epoch processing --- test_generators/epoch_processing/main.py | 10 +++++----- test_libs/gen_helpers/gen_from_tests/gen.py | 7 ++++--- test_libs/pyspec/eth2spec/test/context.py | 13 +++++++++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 2ce895fc5..679ef4c6b 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -2,7 +2,7 @@ from typing import Callable, Iterable from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.epoch_processing import ( +from eth2spec.test.phase_0.epoch_processing import ( test_process_crosslinks, test_process_registry_updates ) @@ -33,8 +33,8 @@ def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), - create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), - create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)), - create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)), + create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), + create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), + create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), ]) diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py index e7d801131..3810c385e 100644 --- a/test_libs/gen_helpers/gen_from_tests/gen.py +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -1,9 +1,10 @@ from inspect import getmembers, isfunction -def generate_from_tests(src, bls_active=True): +def generate_from_tests(src, phase, bls_active=True): """ Generate a list of test cases by running tests from the given src in generator-mode. - :param src: to retrieve tests from (discovered using inspect.getmembers) + :param src: to retrieve tests from (discovered using inspect.getmembers). + :param phase: to run tests against particular phase. :param bls_active: optional, to override BLS switch preference. Defaults to True. :return: the list of test cases. """ @@ -16,7 +17,7 @@ def generate_from_tests(src, bls_active=True): for name in fn_names: tfn = getattr(src, name) try: - test_case = tfn(generator_mode=True, bls_active=bls_active) + test_case = tfn(generator_mode=True, phase=phase, bls_active=bls_active) # If no test case data is returned, the test is ignored. if test_case is not None: out.append(test_case) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index cbc594cd8..520eb9745 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -119,9 +119,18 @@ def with_phases(phases): fn(*args, **kw) def wrapper(*args, **kw): - if 'phase0' in phases: + run_phases = phases + + # limit phases if one explicitly specified + if 'phase' in kw: + phase = kw.pop('phase') + if phase not in phases: + return + run_phases = [phase] + + if 'phase0' in run_phases: run_with_spec_version(spec_phase0, *args, **kw) - if 'phase1' in phases: + if 'phase1' in run_phases: run_with_spec_version(spec_phase1, *args, **kw) return wrapper return decorator From 94d4e3a94402f65a58a23d434f40d021c4f80259 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Jun 2019 20:05:43 -0600 Subject: [PATCH 02/12] operations suite generator for phase0 --- test_generators/epoch_processing/main.py | 6 +++-- test_generators/operations/main.py | 32 +++++++++++------------ test_libs/pyspec/eth2spec/test/context.py | 7 ++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 679ef4c6b..846f463a1 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -35,6 +35,8 @@ if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), - create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), - create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'minimal', + lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'mainnet', + lambda: generate_from_tests(test_process_registry_updates, 'phase0')), ]) diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 82e05b307..38fa42f68 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,13 +1,13 @@ from typing import Callable, Iterable -from eth2spec.test.block_processing import ( +from eth2spec.test.phase_0.block_processing import ( test_process_attestation, test_process_attester_slashing, test_process_block_header, test_process_deposit, test_process_proposer_slashing, test_process_transfer, - test_process_voluntary_exit + test_process_voluntary_exit, ) from gen_base import gen_runner, gen_suite, gen_typing @@ -38,18 +38,18 @@ def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], if __name__ == "__main__": gen_runner.run_generator("operations", [ - create_suite('attestation', 'minimal', lambda: generate_from_tests(test_process_attestation)), - create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation)), - create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), - create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), - create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit)), - create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit)), - create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), - create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing)), - create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer)), - create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer)), - create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit)), - create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit)), + create_suite('attestation', 'minimal', lambda: generate_from_tests(test_process_attestation, 'phase0')), + create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation, 'phase0')), + create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing, 'phase0')), + create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing, 'phase0')), + create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header, 'phase0')), + create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header, 'phase0')), + create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit, 'phase0')), + create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit, 'phase0')), + create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), + create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), + create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer, 'phase0')), + create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer, 'phase0')), + create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), + create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), ]) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 520eb9745..97266acf2 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -116,7 +116,7 @@ def with_phases(phases): def decorator(fn): def run_with_spec_version(spec, *args, **kw): kw['spec'] = spec - fn(*args, **kw) + return fn(*args, **kw) def wrapper(*args, **kw): run_phases = phases @@ -129,8 +129,9 @@ def with_phases(phases): run_phases = [phase] if 'phase0' in run_phases: - run_with_spec_version(spec_phase0, *args, **kw) + ret = run_with_spec_version(spec_phase0, *args, **kw) if 'phase1' in run_phases: - run_with_spec_version(spec_phase1, *args, **kw) + ret = run_with_spec_version(spec_phase1, *args, **kw) + return ret return wrapper return decorator From 46d8422510e5235cecb9da08982dc5c8c77f8920 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 16:42:50 +0200 Subject: [PATCH 03/12] fix ssz container recognition for generators --- test_libs/pyspec/eth2spec/test/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index b61801c3d..817c952b7 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,5 +1,6 @@ from typing import Dict, Any, Callable, Iterable from eth2spec.debug.encode import encode +from eth2spec.utils.ssz.ssz_typing import Container def spectest(description: str = None): @@ -30,9 +31,13 @@ def spectest(description: str = None): else: # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ container. (key, value) = data - if hasattr(value.__class__, 'fields'): + if isinstance(value, Container): out[key] = encode(value, value.__class__) else: + # not a ssz value. + # It could be vector or bytes still, but it is a rare case, + # and lists can't be inferred fully (generics lose element type). + # In such cases, explicitly state the type of the yielded value as a third yielded object. out[key] = value if has_contents: return out From 98a798bd54d7d1969cf84d32a2e6de073e4a68e0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:10:09 -0600 Subject: [PATCH 04/12] sanity tests gens to use explicit phase0 --- test_generators/sanity/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index a9c0fe160..805ee6139 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -30,8 +30,8 @@ def create_suite(handler_name: str, config_name: str, get_cases: Callable[[], It if __name__ == "__main__": gen_runner.run_generator("sanity", [ - create_suite('blocks', 'minimal', lambda: generate_from_tests(test_blocks)), - create_suite('blocks', 'mainnet', lambda: generate_from_tests(test_blocks)), - create_suite('slots', 'minimal', lambda: generate_from_tests(test_slots)), - create_suite('slots', 'mainnet', lambda: generate_from_tests(test_slots)), + create_suite('blocks', 'minimal', lambda: generate_from_tests(test_blocks, 'phase0')), + create_suite('blocks', 'mainnet', lambda: generate_from_tests(test_blocks, 'phase0')), + create_suite('slots', 'minimal', lambda: generate_from_tests(test_slots, 'phase0')), + create_suite('slots', 'mainnet', lambda: generate_from_tests(test_slots, 'phase0')), ]) From ef1ef753a3ecff39b8a85cca5c407e7ff17a359d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:16:37 -0600 Subject: [PATCH 05/12] comment out transfer test from sanity tests --- .../eth2spec/test/sanity/test_blocks.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 587c37742..593202dab 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -303,38 +303,38 @@ def test_voluntary_exit(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases -@spec_state_test -def test_transfer(spec, state): +# @with_all_phases +# @spec_state_test +# def test_transfer(spec, state): # overwrite default 0 to test - spec.MAX_TRANSFERS = 1 + # spec.MAX_TRANSFERS = 1 - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = get_balance(state, sender_index) + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # amount = get_balance(state, sender_index) - transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) - recipient_index = transfer.recipient - pre_transfer_recipient_balance = get_balance(state, recipient_index) + # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) + # recipient_index = transfer.recipient + # pre_transfer_recipient_balance = get_balance(state, recipient_index) # un-activate so validator can transfer - state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + # state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - yield 'pre', state + # yield 'pre', state # Add to state via block transition - block = build_empty_block_for_next_slot(spec, state) - block.body.transfers.append(transfer) - sign_block(spec, state, block) + # block = build_empty_block_for_next_slot(spec, state) + # block.body.transfers.append(transfer) + # sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + # yield 'blocks', [block], List[spec.BeaconBlock] - spec.state_transition(state, block) - yield 'post', state + # spec.state_transition(state, block) + # yield 'post', state - sender_balance = get_balance(state, sender_index) - recipient_balance = get_balance(state, recipient_index) - assert sender_balance == 0 - assert recipient_balance == pre_transfer_recipient_balance + amount + # sender_balance = get_balance(state, sender_index) + # recipient_balance = get_balance(state, recipient_index) + # assert sender_balance == 0 + # assert recipient_balance == pre_transfer_recipient_balance + amount @with_all_phases From cc625c7d6d0c25664b46b119e5389dc7aee3bda9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:26:23 -0600 Subject: [PATCH 06/12] lint --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 593202dab..2275161c8 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -5,7 +5,7 @@ from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.bls import bls_sign from eth2spec.test.helpers.state import get_balance -from eth2spec.test.helpers.transfers import get_valid_transfer +# from eth2spec.test.helpers.transfers import get_valid_transfer from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing From 85b0c237ca2f58e210d0aa410e4892e4c9be6671 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:58:13 +0200 Subject: [PATCH 07/12] fix formatting sanity generator file names --- test_generators/sanity/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index 805ee6139..fbef4da96 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -16,7 +16,7 @@ def create_suite(handler_name: str, config_name: str, get_cases: Callable[[], It spec_phase0.apply_constants_preset(presets) spec_phase1.apply_constants_preset(presets) - return ("%sanity_s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( + return ("sanity_%s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( title="sanity testing", summary="Sanity test suite, %s type, generated from pytests" % handler_name, forks_timeline="testing", From 22f624e12e5b7765c052b8a82b34b050e65fdada Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:08:27 +0200 Subject: [PATCH 08/12] re-order type check for more speed --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 368041f90..832fd5ae2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -513,13 +513,11 @@ def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: def read_elem_type(typ): - if typ == bytes: + if issubclass(typ, bytes): # bytes or bytesN return byte elif is_list_type(typ): return read_list_elem_type(typ) elif is_vector_type(typ): return read_vector_elem_type(typ) - elif issubclass(typ, bytes): # bytes or bytesN - return byte else: raise TypeError("Unexpected type: {}".format(typ)) From d6fd19ce44f96dc7976ca9f48c43f93048a4478a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:30:50 +0200 Subject: [PATCH 09/12] fix ssz-static generator --- test_generators/ssz_static/main.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 7de5237d1..9d9af8c5e 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -1,7 +1,10 @@ from random import Random +from inspect import getmembers, isclass + from eth2spec.debug import random_value, encode from eth2spec.phase0 import spec +from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -27,17 +30,23 @@ def create_test_case_contents(value, typ): @to_dict -def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): - typ = spec.get_ssz_type_by_name(name) +def create_test_case(rng: Random, name: str, typ, mode: random_value.RandomizationMode, chaos: bool): value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield name, create_test_case_contents(value, typ) +def get_spec_ssz_types(): + return [ + (name, value) for (name, value) in getmembers(spec, isclass) + if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class + ] + + @to_tuple def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int): - for type_name in spec.ssz_types: + for (name, ssz_type) in get_spec_ssz_types(): for i in range(count): - yield create_test_case(rng, type_name, mode, chaos) + yield create_test_case(rng, name, ssz_type, mode, chaos) def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int): @@ -81,8 +90,6 @@ if __name__ == "__main__": settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types))) - gen_runner.run_generator("ssz_static", [ get_ssz_suite(seed, config_name, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From 1157d9b8d8f17c475b4bdb34d60c598075e478be Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:34:49 +0200 Subject: [PATCH 10/12] fix shuffling generator --- test_generators/shuffling/main.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 862c4d910..291aa2c47 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -1,5 +1,4 @@ -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.phase0 import spec as spec from eth_utils import ( to_dict, to_tuple ) @@ -8,7 +7,7 @@ from preset_loader import loader @to_dict -def shuffling_case(seed: spec.Bytes32, count: int): +def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count yield 'shuffled', [spec.get_shuffled_index(i, count, seed) for i in range(count)] @@ -23,8 +22,7 @@ def shuffling_test_cases(): def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'minimal') - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_minimal", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with minimal config", @@ -39,8 +37,7 @@ def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'mainnet') - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_full", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with mainnet config", From f4814862fe3245f43bbdb3570fe6eda88207e217 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 19:23:45 +0200 Subject: [PATCH 11/12] fix typing check, add zero-hash cache to hash function --- test_libs/pyspec/eth2spec/utils/hash_function.py | 16 +++++++++++++++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index f965827d0..4aecc57f8 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -1,5 +1,19 @@ from hashlib import sha256 +ZERO_BYTES32 = b'\x00' * 32 + +def _hash(x): + return sha256(x).digest() + +zerohashes = [(None, ZERO_BYTES32)] +for layer in range(1, 32): + k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] + zerohashes.append((k, _hash(k))) +zerohashes = zerohashes[1:] + def hash(x): - return sha256(x).digest() + for (k, h) in zerohashes: + if x == k: + return h + return _hash(x) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 832fd5ae2..55ced4ee2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -513,7 +513,7 @@ def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: def read_elem_type(typ): - if issubclass(typ, bytes): # bytes or bytesN + if typ == bytes or (isinstance(typ, type) and issubclass(typ, bytes)): # bytes or bytesN return byte elif is_list_type(typ): return read_list_elem_type(typ) From 9ec395c04fedf9572b1bfd815c6da1f9bd4f4b1c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 21:53:38 +0200 Subject: [PATCH 12/12] fig linting + improve docs + structure of hash optimization --- .../pyspec/eth2spec/utils/hash_function.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index 4aecc57f8..2c9b5a579 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -2,18 +2,27 @@ from hashlib import sha256 ZERO_BYTES32 = b'\x00' * 32 + def _hash(x): return sha256(x).digest() -zerohashes = [(None, ZERO_BYTES32)] -for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) -zerohashes = zerohashes[1:] + +# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. +# Key = the hash input +# Value = the hash output +hash_cache = [] + + +def add_zero_hashes_to_cache(): + zerohashes = [(None, ZERO_BYTES32)] + for layer in range(1, 32): + k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] + zerohashes.append((k, _hash(k))) + hash_cache.extend(zerohashes[1:]) def hash(x): - for (k, h) in zerohashes: + for (k, h) in hash_cache: if x == k: return h return _hash(x)