From 6f8d011044137bf5b441eaa83fcb20cf280f0626 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 5 May 2019 13:49:59 +0200 Subject: [PATCH 001/118] update shuffling func name --- test_generators/shuffling/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 2c4faeb8f..bb14520e1 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -10,7 +10,7 @@ from preset_loader import loader def shuffling_case(seed: spec.Bytes32, count: int): yield 'seed', '0x' + seed.hex() yield 'count', count - yield 'shuffled', [spec.get_permuted_index(i, count, seed) for i in range(count)] + yield 'shuffled', [spec.get_shuffled_index(i, count, seed) for i in range(count)] @to_tuple From 8b24abde3103d07b7858786a08847b40e10f8b0a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 00:31:57 +0200 Subject: [PATCH 002/118] implement spectest decorator, update attestation tests --- .../test_process_attestation.py | 93 ++++++++++--------- test_libs/pyspec/tests/conftest.py | 19 ---- test_libs/pyspec/tests/context.py | 10 ++ test_libs/pyspec/tests/utils.py | 46 +++++++++ 4 files changed, 104 insertions(+), 64 deletions(-) create mode 100644 test_libs/pyspec/tests/context.py create mode 100644 test_libs/pyspec/tests/utils.py diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index bcf71376c..c90d13fd5 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -8,8 +8,7 @@ from eth2spec.phase0.state_transition import ( ) from eth2spec.phase0.spec import ( get_current_epoch, - process_attestation, - slot_to_epoch, + process_attestation ) from tests.helpers import ( build_empty_block_for_next_slot, @@ -18,63 +17,75 @@ from tests.helpers import ( next_slot, ) - -# mark entire file as 'attestations' -pytestmark = pytest.mark.attestations +from tests.utils import spectest +from tests.context import with_state def run_attestation_processing(state, attestation, valid=True): """ - Run ``process_attestation`` returning the pre and post state. + Run ``process_attestation``, yielding pre-state ('pre'), attestation ('attestation'), and post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - post_state = deepcopy(state) + # yield pre-state + yield 'pre', state + yield 'attestation', attestation + + # If the attestation is invalid, processing is aborted, and there is no post-state. if not valid: with pytest.raises(AssertionError): - process_attestation(post_state, attestation) - return state, None + process_attestation(state, attestation) + yield 'post', None + return - process_attestation(post_state, attestation) + current_epoch_count = len(state.current_epoch_attestations) + previous_epoch_count = len(state.previous_epoch_attestations) - current_epoch = get_current_epoch(state) - if attestation.data.target_epoch == current_epoch: - assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 + # process attestation + process_attestation(state, attestation) + + # Make sure the attestation has been processed + if attestation.data.target_epoch == get_current_epoch(state): + assert len(state.current_epoch_attestations) == current_epoch_count + 1 else: - assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1 + assert len(state.previous_epoch_attestations) == previous_epoch_count + 1 - return state, post_state + # yield post-state + yield 'post', state +# shorthand for decorating @with_state @spectest() +def attestation_test(fn): + return with_state(spectest()(fn)) + + +@attestation_test def test_success(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation) +@attestation_test def test_success_prevous_epoch(state): attestation = get_valid_attestation(state) block = build_empty_block_for_next_slot(state) block.slot = state.slot + spec.SLOTS_PER_EPOCH state_transition(state, block) - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation) +@attestation_test def test_before_inclusion_delay(state): attestation = get_valid_attestation(state) # do not increment slot to allow for inclusion delay - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_after_epoch_slots(state): attestation = get_valid_attestation(state) block = build_empty_block_for_next_slot(state) @@ -82,44 +93,40 @@ def test_after_epoch_slots(state): block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 state_transition(state, block) - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_bad_source_epoch(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 10 - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_bad_source_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_non_zero_crosslink_data_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink_data_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_bad_previous_crosslink(state): next_epoch(state) attestation = get_valid_attestation(state) @@ -128,28 +135,24 @@ def test_bad_previous_crosslink(state): state.current_crosslinks[attestation.data.shard].epoch += 10 - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) +@attestation_test def test_empty_aggregation_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state + yield from run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py index 9840dc7b2..8b90ce3b0 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -3,10 +3,6 @@ import pytest from eth2spec.phase0 import spec from preset_loader import loader -from .helpers import ( - create_genesis_state, -) - def pytest_addoption(parser): parser.addoption( @@ -19,18 +15,3 @@ def config(request): config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) spec.apply_constants_preset(presets) - - -@pytest.fixture -def num_validators(config): - return spec.SLOTS_PER_EPOCH * 8 - - -@pytest.fixture -def deposit_data_leaves(): - return list() - - -@pytest.fixture -def state(num_validators, deposit_data_leaves): - return create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/context.py b/test_libs/pyspec/tests/context.py new file mode 100644 index 000000000..27a91a031 --- /dev/null +++ b/test_libs/pyspec/tests/context.py @@ -0,0 +1,10 @@ + +from eth2spec.phase0 import spec +from tests.utils import with_args + +from .helpers import ( + create_genesis_state, +) + +# Provides a genesis state as first argument to the function decorated with this +with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())]) diff --git a/test_libs/pyspec/tests/utils.py b/test_libs/pyspec/tests/utils.py new file mode 100644 index 000000000..f13eede65 --- /dev/null +++ b/test_libs/pyspec/tests/utils.py @@ -0,0 +1,46 @@ +from eth2spec.debug.encode import encode + + +def spectest(description: str = None): + def runner(fn): + # this wraps the function, to hide that the function actually yielding data. + def entry(*args, **kw): + # check generator mode, may be None/else. + # "pop" removes it, so it is not passed to the inner function. + if kw.pop('generator_mode', False) is True: + out = {} + if description is None: + # fall back on function name for test description + name = fn.__name__ + if name.startswith('test_'): + name = name[5:] + out['description'] = name + else: + # description can be explicit + out['description'] = description + # put all generated data into a dict. + for data in fn(*args, **kw): + # If there is a type argument, encode it as that type. + if len(data) == 3: + (key, value, typ) = data + out[key] = encode(value, typ) + else: + # Otherwise, just put the raw value. + (key, value) = data + out[key] = value + return out + else: + # just complete the function, ignore all yielded data, we are not using it + for _ in fn(*args, **kw): + continue + return entry + return runner + + +def with_args(create_args): + def runner(fn): + # this wraps the function, to hide that the function actually yielding data. + def entry(*args, **kw): + return fn(*(create_args() + list(args)), **kw) + return entry + return runner From a8d8da25fdd56958320f5dccae44548463c6c550 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 15:40:18 +0200 Subject: [PATCH 003/118] fix wording and improve encoding logic --- test_libs/pyspec/tests/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/tests/utils.py b/test_libs/pyspec/tests/utils.py index f13eede65..b19d4df59 100644 --- a/test_libs/pyspec/tests/utils.py +++ b/test_libs/pyspec/tests/utils.py @@ -3,7 +3,7 @@ from eth2spec.debug.encode import encode def spectest(description: str = None): def runner(fn): - # this wraps the function, to hide that the function actually yielding data. + # this wraps the function, to hide that the function actually is yielding data, instead of returning once. def entry(*args, **kw): # check generator mode, may be None/else. # "pop" removes it, so it is not passed to the inner function. @@ -25,9 +25,12 @@ def spectest(description: str = None): (key, value, typ) = data out[key] = encode(value, typ) else: - # Otherwise, just put the raw value. + # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ container. (key, value) = data - out[key] = value + if hasattr(value.__class__, 'fields'): + out[key] = encode(value, value.__class__) + else: + out[key] = value return out else: # just complete the function, ignore all yielded data, we are not using it From dcedfc350a54e0cad920f1df5dd7ff0f69e4425a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:22:51 +0200 Subject: [PATCH 004/118] move out spec state test deco --- .../pyspec/tests/block_processing/block_test_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 test_libs/pyspec/tests/block_processing/block_test_helpers.py diff --git a/test_libs/pyspec/tests/block_processing/block_test_helpers.py b/test_libs/pyspec/tests/block_processing/block_test_helpers.py new file mode 100644 index 000000000..71b9fc250 --- /dev/null +++ b/test_libs/pyspec/tests/block_processing/block_test_helpers.py @@ -0,0 +1,8 @@ + +from tests.utils import spectest +from tests.context import with_state + + +# shorthand for decorating @with_state @spectest() +def spec_state_test(fn): + return with_state(spectest()(fn)) From 7bbf9ed3fc1609070ca3d7f1d937013b56a298d6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:23:15 +0200 Subject: [PATCH 005/118] update attestation testing --- .../test_process_attestation.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index c90d13fd5..41e9419b3 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -17,13 +17,15 @@ from tests.helpers import ( next_slot, ) -from tests.utils import spectest -from tests.context import with_state +from .block_test_helpers import spec_state_test def run_attestation_processing(state, attestation, valid=True): """ - Run ``process_attestation``, yielding pre-state ('pre'), attestation ('attestation'), and post-state ('post'). + Run ``process_attestation``, yielding: + - pre-state ('pre') + - attestation ('attestation') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ # yield pre-state @@ -54,12 +56,7 @@ def run_attestation_processing(state, attestation, valid=True): yield 'post', state -# shorthand for decorating @with_state @spectest() -def attestation_test(fn): - return with_state(spectest()(fn)) - - -@attestation_test +@spec_state_test def test_success(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -67,7 +64,7 @@ def test_success(state): yield from run_attestation_processing(state, attestation) -@attestation_test +@spec_state_test def test_success_prevous_epoch(state): attestation = get_valid_attestation(state) block = build_empty_block_for_next_slot(state) @@ -77,7 +74,7 @@ def test_success_prevous_epoch(state): yield from run_attestation_processing(state, attestation) -@attestation_test +@spec_state_test def test_before_inclusion_delay(state): attestation = get_valid_attestation(state) # do not increment slot to allow for inclusion delay @@ -85,7 +82,7 @@ def test_before_inclusion_delay(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_after_epoch_slots(state): attestation = get_valid_attestation(state) block = build_empty_block_for_next_slot(state) @@ -96,7 +93,7 @@ def test_after_epoch_slots(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_bad_source_epoch(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -106,7 +103,7 @@ def test_bad_source_epoch(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_bad_source_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -116,7 +113,7 @@ def test_bad_source_root(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_non_zero_crosslink_data_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -126,7 +123,7 @@ def test_non_zero_crosslink_data_root(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_bad_previous_crosslink(state): next_epoch(state) attestation = get_valid_attestation(state) @@ -138,7 +135,7 @@ def test_bad_previous_crosslink(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -148,7 +145,7 @@ def test_non_empty_custody_bitfield(state): yield from run_attestation_processing(state, attestation, False) -@attestation_test +@spec_state_test def test_empty_aggregation_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 61c0ddbcbbea2f09d57076a51c28a4c0cebd0837 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:23:33 +0200 Subject: [PATCH 006/118] update attester slashing testing --- .../test_process_attester_slashing.py | 77 +++++++++++-------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 2ea16f13d..631484b12 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -1,4 +1,3 @@ -from copy import deepcopy import pytest import eth2spec.phase0.spec as spec @@ -12,52 +11,69 @@ from tests.helpers import ( next_epoch, ) -# mark entire file as 'attester_slashing' -pytestmark = pytest.mark.attester_slashings +from tests.utils import spectest +from tests.context import with_state + +from .block_test_helpers import spec_state_test def run_attester_slashing_processing(state, attester_slashing, valid=True): """ - Run ``process_attester_slashing`` returning the pre and post state. + Run ``process_attester_slashing``, yielding: + - pre-state ('pre') + - attester_slashing ('attester_slashing') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - post_state = deepcopy(state) + + yield 'pre', state + yield 'attester_slashing', attester_slashing if not valid: with pytest.raises(AssertionError): - process_attester_slashing(post_state, attester_slashing) - return state, None - - process_attester_slashing(post_state, attester_slashing) + process_attester_slashing(state, attester_slashing) + yield 'post', None + return slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - slashed_validator = post_state.validator_registry[slashed_index] + pre_slashed_balance = get_balance(state, slashed_index) + + proposer_index = get_beacon_proposer_index(state) + pre_proposer_balance = get_balance(state, proposer_index) + + # Process slashing + process_attester_slashing(state, attester_slashing) + + slashed_validator = state.validator_registry[slashed_index] + + # Check slashing assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward assert ( - get_balance(post_state, slashed_index) < - get_balance(state, slashed_index) + get_balance(state, slashed_index) < + pre_slashed_balance ) - proposer_index = get_beacon_proposer_index(state) + # gained whistleblower reward assert ( - get_balance(post_state, proposer_index) > - get_balance(state, proposer_index) + get_balance(state, proposer_index) > + pre_proposer_balance ) - return state, post_state + yield 'post', state +@spec_state_test def test_success_double(state): attester_slashing = get_valid_attester_slashing(state) - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) - - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing) +@spec_state_test def test_success_surround(state): next_epoch(state) state.current_justified_epoch += 1 @@ -67,31 +83,28 @@ def test_success_surround(state): attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) - - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing) +@spec_state_test def test_same_data(state): attester_slashing = get_valid_attester_slashing(state) attester_slashing.attestation_1.data = attester_slashing.attestation_2.data - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing, False) +@spec_state_test def test_no_double_or_surround(state): attester_slashing = get_valid_attester_slashing(state) attester_slashing.attestation_1.data.target_epoch += 1 - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing, False) +@spec_state_test def test_participants_already_slashed(state): attester_slashing = get_valid_attester_slashing(state) @@ -101,17 +114,15 @@ def test_participants_already_slashed(state): for index in validator_indices: state.validator_registry[index].slashed = True - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing, False) +@spec_state_test def test_custody_bit_0_and_1(state): attester_slashing = get_valid_attester_slashing(state) attester_slashing.attestation_1.custody_bit_1_indices = ( attester_slashing.attestation_1.custody_bit_0_indices ) - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - return pre_state, attester_slashing, post_state + yield from run_attester_slashing_processing(state, attester_slashing, False) From 9ff52193b171d411435293b43a62143931a1d136 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:23:53 +0200 Subject: [PATCH 007/118] update block header testing --- .../test_process_block_header.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c1..27058d28c 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -13,8 +13,7 @@ from tests.helpers import ( next_slot, ) -# mark entire file as 'header' -pytestmark = pytest.mark.header +from .block_test_helpers import spec_state_test def prepare_state_for_header_processing(state): @@ -24,43 +23,50 @@ def prepare_state_for_header_processing(state): def run_block_header_processing(state, block, valid=True): """ - Run ``process_block_header`` returning the pre and post state. + Run ``process_block_header``, yielding: + - pre-state ('pre') + - block ('block') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ prepare_state_for_header_processing(state) - post_state = deepcopy(state) + + yield 'pre', state + yield 'block', block if not valid: with pytest.raises(AssertionError): - process_block_header(post_state, block) - return state, None + process_block_header(state, block) + yield 'post', None + return - process_block_header(post_state, block) - return state, post_state + process_block_header(state, block) + yield 'post', state +@spec_state_test def test_success(state): block = build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block) - return state, block, post_state + yield from run_block_header_processing(state, block) +@spec_state_test def test_invalid_slot(state): block = build_empty_block_for_next_slot(state) block.slot = state.slot + 2 # invalid slot - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None + yield from run_block_header_processing(state, block, valid=False) +@spec_state_test def test_invalid_previous_block_root(state): block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None + yield from run_block_header_processing(state, block, valid=False) +@spec_state_test def test_proposer_slashed(state): # use stub state to get proposer index of next slot stub_state = deepcopy(state) @@ -72,5 +78,4 @@ def test_proposer_slashed(state): block = build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None + yield from run_block_header_processing(state, block, valid=False) From 90a56e2f5b2b6360d99ea82d4cbe17f7d2c99a84 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:24:11 +0200 Subject: [PATCH 008/118] update deposit testing --- .../block_processing/test_process_deposit.py | 170 ++++++++---------- 1 file changed, 71 insertions(+), 99 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index bbfb390ef..aee9789cd 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -14,128 +14,100 @@ from tests.helpers import ( pubkeys, ) - -# mark entire file as 'deposits' -pytestmark = pytest.mark.deposits +from .block_test_helpers import spec_state_test -def test_success(state): - pre_state = deepcopy(state) +def prepare_state_and_deposit(state, validator_index, amount): + """ + Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. + """ + pre_validator_count = len(state.validator_registry) # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + deposit_data_leaves = [ZERO_HASH] * pre_validator_count - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) + 1 - assert len(post_state.balances) == len(state.balances) + 1 - assert post_state.validator_registry[index].pubkey == pubkeys[index] - assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - - return pre_state, deposit, post_state - - -def test_success_top_up(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - validator_index = 0 - amount = spec.MAX_EFFECTIVE_BALANCE // 4 pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] deposit, root, deposit_data_leaves = build_deposit( - pre_state, + state, deposit_data_leaves, pubkey, privkey, amount, ) - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - pre_balance = get_balance(pre_state, validator_index) - - post_state = deepcopy(pre_state) - - process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) - assert len(post_state.balances) == len(state.balances) - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - assert get_balance(post_state, validator_index) == pre_balance + amount - - return pre_state, deposit, post_state + state.latest_eth1_data.deposit_root = root + state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + return deposit +def run_deposit_processing(state, deposit, validator_index, valid=True): + """ + Run ``process_deposit``, yielding: + - pre-state ('pre') + - deposit ('deposit') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + pre_balance = get_balance(state, validator_index) + pre_validator_count = len(state.validator_registry) + + yield 'pre', state + yield 'deposit', deposit + + if not valid: + with pytest.raises(AssertionError): + process_deposit(state, deposit) + yield 'post', None + return + + process_deposit(state, deposit) + + yield 'post', state + + assert len(state.validator_registry) == pre_validator_count + assert len(state.balances) == pre_validator_count + assert state.deposit_index == state.latest_eth1_data.deposit_count + assert get_balance(state, validator_index) == pre_balance + deposit.amount + + +@spec_state_test +def test_success(state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount) + + yield from run_deposit_processing(state, deposit, validator_index) + + +@spec_state_test +def test_success_top_up(state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(state, validator_index, amount) + + yield from run_deposit_processing(state, deposit, validator_index) + + +@spec_state_test def test_wrong_index(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount) # mess up deposit_index - deposit.index = pre_state.deposit_index + 1 + deposit.index = state.deposit_index + 1 - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - process_deposit(post_state, deposit) - - return pre_state, deposit, None + yield from run_deposit_processing(state, deposit, validator_index, valid=False) +@spec_state_test def test_bad_merkle_proof(state): - pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount) # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - process_deposit(post_state, deposit) - - return pre_state, deposit, None + yield from run_deposit_processing(state, deposit, validator_index, valid=False) From 802f2710c320b532b89d382b7032ecbee8eed0d3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:30:13 +0200 Subject: [PATCH 009/118] clean import, update proposer slashing test --- .../block_processing/test_process_deposit.py | 1 - .../test_process_proposer_slashing.py | 57 ++++++++++--------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index aee9789cd..c45191020 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -1,4 +1,3 @@ -from copy import deepcopy import pytest import eth2spec.phase0.spec as spec diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index 475221036..fbf4a1959 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -1,4 +1,3 @@ -from copy import deepcopy import pytest import eth2spec.phase0.spec as spec @@ -11,78 +10,82 @@ from tests.helpers import ( get_valid_proposer_slashing, ) -# mark entire file as 'proposer_slashings' -pytestmark = pytest.mark.proposer_slashings +from .block_test_helpers import spec_state_test def run_proposer_slashing_processing(state, proposer_slashing, valid=True): """ - Run ``process_proposer_slashing`` returning the pre and post state. + Run ``process_proposer_slashing``, yielding: + - pre-state ('pre') + - proposer_slashing ('proposer_slashing') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - post_state = deepcopy(state) + pre_proposer_balance = get_balance(state, proposer_slashing.proposer_index) + + yield 'pre', state + yield 'proposer_slashing', proposer_slashing if not valid: with pytest.raises(AssertionError): - process_proposer_slashing(post_state, proposer_slashing) - return state, None + process_proposer_slashing(state, proposer_slashing) + yield 'post', None + return - process_proposer_slashing(post_state, proposer_slashing) + process_proposer_slashing(state, proposer_slashing) + yield 'post', state - slashed_validator = post_state.validator_registry[proposer_slashing.proposer_index] + # check if slashed + slashed_validator = state.validator_registry[proposer_slashing.proposer_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward assert ( - get_balance(post_state, proposer_slashing.proposer_index) < - get_balance(state, proposer_slashing.proposer_index) + get_balance(state, proposer_slashing.proposer_index) < + pre_proposer_balance ) - return state, post_state - +@spec_state_test def test_success(state): proposer_slashing = get_valid_proposer_slashing(state) - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing) - - return pre_state, proposer_slashing, post_state + yield from run_proposer_slashing_processing(state, proposer_slashing) +@spec_state_test def test_epochs_are_different(state): proposer_slashing = get_valid_proposer_slashing(state) # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state + yield from run_proposer_slashing_processing(state, proposer_slashing, False) +@spec_state_test def test_headers_are_same(state): proposer_slashing = get_valid_proposer_slashing(state) # set headers to be the same proposer_slashing.header_2 = proposer_slashing.header_1 - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state + yield from run_proposer_slashing_processing(state, proposer_slashing, False) +@spec_state_test def test_proposer_is_slashed(state): proposer_slashing = get_valid_proposer_slashing(state) # set proposer to slashed state.validator_registry[proposer_slashing.proposer_index].slashed = True - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state + yield from run_proposer_slashing_processing(state, proposer_slashing, False) +@spec_state_test def test_proposer_is_withdrawn(state): proposer_slashing = get_valid_proposer_slashing(state) @@ -91,6 +94,4 @@ def test_proposer_is_withdrawn(state): proposer_index = proposer_slashing.proposer_index state.validator_registry[proposer_index].withdrawable_epoch = current_epoch - 1 - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state + yield from run_proposer_slashing_processing(state, proposer_slashing, False) From 4820ac9d1ae2e2102a658073c28eeec4f5e488ba Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:35:01 +0200 Subject: [PATCH 010/118] update transfer tests --- .../block_processing/test_process_transfer.py | 80 +++++++++---------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 0eeaa7792..920c26f59 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -1,4 +1,3 @@ -from copy import deepcopy import pytest import eth2spec.phase0.spec as spec @@ -14,49 +13,52 @@ from tests.helpers import ( next_epoch, ) - -# mark entire file as 'transfers' -pytestmark = pytest.mark.transfers +from .block_test_helpers import spec_state_test def run_transfer_processing(state, transfer, valid=True): """ - Run ``process_transfer`` returning the pre and post state. + Run ``process_transfer``, yielding: + - pre-state ('pre') + - transfer ('transfer') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - process_transfer(post_state, transfer) - return state, None - - - process_transfer(post_state, transfer) proposer_index = get_beacon_proposer_index(state) pre_transfer_sender_balance = state.balances[transfer.sender] pre_transfer_recipient_balance = state.balances[transfer.recipient] pre_transfer_proposer_balance = state.balances[proposer_index] - sender_balance = post_state.balances[transfer.sender] - recipient_balance = post_state.balances[transfer.recipient] + + yield 'pre', state + yield 'transfer', transfer + + if not valid: + with pytest.raises(AssertionError): + process_transfer(state, transfer) + yield 'post', None + return + + process_transfer(state, transfer) + yield 'post', state + + sender_balance = state.balances[transfer.sender] + recipient_balance = state.balances[transfer.recipient] assert sender_balance == pre_transfer_sender_balance - transfer.amount - transfer.fee assert recipient_balance == pre_transfer_recipient_balance + transfer.amount - assert post_state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee - - return state, post_state + assert state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee +@spec_state_test def test_success_non_activated(state): transfer = get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer) +@spec_state_test def test_success_withdrawable(state): next_epoch(state) @@ -65,43 +67,39 @@ def test_success_withdrawable(state): # withdrawable_epoch in past so can transfer state.validator_registry[transfer.sender].withdrawable_epoch = get_current_epoch(state) - 1 - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer) +@spec_state_test def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer) +@spec_state_test def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer, False) +@spec_state_test def test_incorrect_slot(state): transfer = get_valid_transfer(state, slot=state.slot+1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer, False) +@spec_state_test def test_insufficient_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE @@ -111,11 +109,10 @@ def test_insufficient_balance(state): # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer, False) +@spec_state_test def test_no_dust(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] @@ -124,11 +121,10 @@ def test_no_dust(state): # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer, False) +@spec_state_test def test_invalid_pubkey(state): transfer = get_valid_transfer(state) state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH @@ -136,6 +132,4 @@ def test_invalid_pubkey(state): # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state + yield from run_transfer_processing(state, transfer, False) From 922a30a61911c014004c4f1053ac64964acf316e Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 16:51:46 +0200 Subject: [PATCH 011/118] update voluntary exit tests --- .../block_processing/test_voluntary_exit.py | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py index c58c5238a..3e1ae4f7f 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py @@ -14,32 +14,38 @@ from tests.helpers import ( pubkey_to_privkey, ) - -# mark entire file as 'voluntary_exits' -pytestmark = pytest.mark.voluntary_exits +from .block_test_helpers import spec_state_test def run_voluntary_exit_processing(state, voluntary_exit, valid=True): """ - Run ``process_voluntary_exit`` returning the pre and post state. + Run ``process_voluntary_exit``, yielding: + - pre-state ('pre') + - voluntary_exit ('voluntary_exit') + - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - post_state = deepcopy(state) + validator_index = voluntary_exit.validator_index + pre_exit_epoch = state.validator_registry[validator_index].exit_epoch + + yield 'pre', state + yield 'voluntary_exit', voluntary_exit if not valid: with pytest.raises(AssertionError): - process_voluntary_exit(post_state, voluntary_exit) - return state, None + process_voluntary_exit(state, voluntary_exit) + yield 'post', None + return - process_voluntary_exit(post_state, voluntary_exit) + process_voluntary_exit(state, voluntary_exit) - validator_index = voluntary_exit.validator_index - assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + yield 'post', state - return state, post_state + assert pre_exit_epoch == spec.FAR_FUTURE_EPOCH + assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH +@spec_state_test def test_success(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -55,10 +61,10 @@ def test_success(state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit) - return pre_state, voluntary_exit, post_state + yield from run_voluntary_exit_processing(state, voluntary_exit) +@spec_state_test def test_success_exit_queue(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -67,17 +73,23 @@ def test_success_exit_queue(state): # exit `MAX_EXITS_PER_EPOCH` initial_indices = get_active_validator_indices(state, current_epoch)[:get_churn_limit(state)] - post_state = state + + # Prepare a bunch of exits, based on the current state + exit_queue = [] for index in initial_indices: privkey = pubkey_to_privkey[state.validator_registry[index].pubkey] - voluntary_exit = build_voluntary_exit( + exit_queue.append(build_voluntary_exit( state, current_epoch, index, privkey, - ) + )) - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) + # Now run all the exits + for voluntary_exit in exit_queue: + # the function yields data, but we are just interested in running it here, ignore yields. + for _ in run_voluntary_exit_processing(state, voluntary_exit): + continue # exit an additional validator validator_index = get_active_validator_indices(state, current_epoch)[-1] @@ -89,16 +101,17 @@ def test_success_exit_queue(state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) + # This is the interesting part of the test: on a pre-state with a full exit queue, + # when processing an additional exit, it results in an exit in a later epoch + yield from run_voluntary_exit_processing(state, voluntary_exit) assert ( - post_state.validator_registry[validator_index].exit_epoch == - post_state.validator_registry[initial_indices[0]].exit_epoch + 1 + state.validator_registry[validator_index].exit_epoch == + state.validator_registry[initial_indices[0]].exit_epoch + 1 ) - return pre_state, voluntary_exit, post_state - +@spec_state_test def test_validator_not_active(state): current_epoch = get_current_epoch(state) validator_index = get_active_validator_indices(state, current_epoch)[0] @@ -106,9 +119,7 @@ def test_validator_not_active(state): state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH - # # build and test voluntary exit - # voluntary_exit = build_voluntary_exit( state, current_epoch, @@ -116,10 +127,10 @@ def test_validator_not_active(state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state + yield from run_voluntary_exit_processing(state, voluntary_exit, False) +@spec_state_test def test_validator_already_exited(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -138,10 +149,10 @@ def test_validator_already_exited(state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state + yield from run_voluntary_exit_processing(state, voluntary_exit, False) +@spec_state_test def test_validator_not_active_long_enough(state): current_epoch = get_current_epoch(state) validator_index = get_active_validator_indices(state, current_epoch)[0] @@ -159,5 +170,4 @@ def test_validator_not_active_long_enough(state): spec.PERSISTENT_COMMITTEE_PERIOD ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state + yield from run_voluntary_exit_processing(state, voluntary_exit, False) From 1722d5806763ccd286b640250cc57de4b417e0de Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 17:10:43 +0200 Subject: [PATCH 012/118] updat epoch processing tests --- .../block_processing/block_test_helpers.py | 8 --- .../test_process_attestation.py | 2 +- .../test_process_attester_slashing.py | 5 +- .../test_process_block_header.py | 2 +- .../block_processing/test_process_deposit.py | 2 +- .../test_process_proposer_slashing.py | 2 +- .../block_processing/test_process_transfer.py | 2 +- .../block_processing/test_voluntary_exit.py | 3 +- test_libs/pyspec/tests/context.py | 7 +++ .../test_process_crosslinks.py | 60 ++++++++++--------- .../test_process_registry_updates.py | 27 +++++---- 11 files changed, 61 insertions(+), 59 deletions(-) delete mode 100644 test_libs/pyspec/tests/block_processing/block_test_helpers.py diff --git a/test_libs/pyspec/tests/block_processing/block_test_helpers.py b/test_libs/pyspec/tests/block_processing/block_test_helpers.py deleted file mode 100644 index 71b9fc250..000000000 --- a/test_libs/pyspec/tests/block_processing/block_test_helpers.py +++ /dev/null @@ -1,8 +0,0 @@ - -from tests.utils import spectest -from tests.context import with_state - - -# shorthand for decorating @with_state @spectest() -def spec_state_test(fn): - return with_state(spectest()(fn)) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 41e9419b3..8b43416a6 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -17,7 +17,7 @@ from tests.helpers import ( next_slot, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def run_attestation_processing(state, attestation, valid=True): diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 631484b12..4a19fccb3 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -11,10 +11,7 @@ from tests.helpers import ( next_epoch, ) -from tests.utils import spectest -from tests.context import with_state - -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def run_attester_slashing_processing(state, attester_slashing, valid=True): diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index 27058d28c..199f2bda4 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -13,7 +13,7 @@ from tests.helpers import ( next_slot, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def prepare_state_for_header_processing(state): diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index c45191020..db1738acc 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -13,7 +13,7 @@ from tests.helpers import ( pubkeys, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def prepare_state_and_deposit(state, validator_index, amount): diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index fbf4a1959..5a16c7d6c 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -10,7 +10,7 @@ from tests.helpers import ( get_valid_proposer_slashing, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def run_proposer_slashing_processing(state, proposer_slashing, valid=True): diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 920c26f59..8fc4d1fc4 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -13,7 +13,7 @@ from tests.helpers import ( next_epoch, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def run_transfer_processing(state, transfer, valid=True): diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py index 3e1ae4f7f..566ebf028 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py @@ -1,4 +1,3 @@ -from copy import deepcopy import pytest import eth2spec.phase0.spec as spec @@ -14,7 +13,7 @@ from tests.helpers import ( pubkey_to_privkey, ) -from .block_test_helpers import spec_state_test +from tests.context import spec_state_test def run_voluntary_exit_processing(state, voluntary_exit, valid=True): diff --git a/test_libs/pyspec/tests/context.py b/test_libs/pyspec/tests/context.py index 27a91a031..d78acdc0b 100644 --- a/test_libs/pyspec/tests/context.py +++ b/test_libs/pyspec/tests/context.py @@ -6,5 +6,12 @@ from .helpers import ( create_genesis_state, ) +from tests.utils import spectest + # Provides a genesis state as first argument to the function decorated with this with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())]) + + +# shorthand for decorating @with_state @spectest() +def spec_state_test(fn): + return with_state(spectest()(fn)) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index d6765e3a7..10de064ed 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -1,5 +1,4 @@ from copy import deepcopy -import pytest import eth2spec.phase0.spec as spec @@ -19,15 +18,18 @@ from tests.helpers import ( get_valid_attestation, next_epoch, next_slot, - set_bitfield_bit, ) - -# mark entire file as 'crosslinks' -pytestmark = pytest.mark.crosslinks +from tests.context import spec_state_test def run_process_crosslinks(state, valid=True): + """ + Run ``process_crosslinks``, yielding: + - pre-state ('pre') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 block = build_empty_block_for_next_slot(state) @@ -37,21 +39,20 @@ def run_process_crosslinks(state, valid=True): # cache state before epoch transition cache_state(state) - post_state = deepcopy(state) - process_crosslinks(post_state) - - return state, post_state + yield 'pre', state + process_crosslinks(state) + yield 'post', state +@spec_state_test def test_no_attestations(state): - pre_state, post_state = run_process_crosslinks(state) + yield from run_process_crosslinks(state) for shard in range(spec.SHARD_COUNT): - assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] - - return pre_state, post_state + assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] +@spec_state_test def test_single_crosslink_update_from_current_epoch(state): next_epoch(state) @@ -62,15 +63,16 @@ def test_single_crosslink_update_from_current_epoch(state): assert len(state.current_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(state) - shard = attestation.data.shard - assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] - assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] + pre_crosslink = deepcopy(state.current_crosslinks[shard]) - return pre_state, post_state + yield from run_process_crosslinks(state) + + assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] + assert pre_crosslink != state.current_crosslinks[shard] +@spec_state_test def test_single_crosslink_update_from_previous_epoch(state): next_epoch(state) @@ -81,20 +83,23 @@ def test_single_crosslink_update_from_previous_epoch(state): assert len(state.previous_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(state) + shard = attestation.data.shard + pre_crosslink = deepcopy(state.current_crosslinks[shard]) + crosslink_deltas = get_crosslink_deltas(state) - shard = attestation.data.shard - assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] - assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] + yield from run_process_crosslinks(state) + + assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] + assert pre_crosslink != state.current_crosslinks[shard] + # ensure rewarded for index in get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 - return pre_state, post_state - +@spec_state_test def test_double_late_crosslink(state): next_epoch(state) state.slot += 4 @@ -121,16 +126,15 @@ def test_double_late_crosslink(state): assert len(state.previous_epoch_attestations) == 1 assert len(state.current_epoch_attestations) == 0 - pre_state, post_state = run_process_crosslinks(state) crosslink_deltas = get_crosslink_deltas(state) + + yield from run_process_crosslinks(state) shard = attestation_2.data.shard # ensure that the current crosslinks were not updated by the second attestation - assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] + assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink for index in get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 - - return pre_state, post_state diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py index 11f5de2ad..85d0d07f4 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py @@ -1,7 +1,3 @@ -from copy import deepcopy - -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( @@ -12,10 +8,10 @@ from tests.helpers import ( next_epoch, ) -# mark entire file as 'state' -pytestmark = pytest.mark.state +from tests.context import spec_state_test +@spec_state_test def test_activation(state): index = 0 assert is_active_validator(state.validator_registry[index], get_current_epoch(state)) @@ -26,13 +22,18 @@ def test_activation(state): state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE assert not is_active_validator(state.validator_registry[index], get_current_epoch(state)) - pre_state = deepcopy(state) + yield 'pre', state blocks = [] for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): block = next_epoch(state) blocks.append(block) + # provide extra type hinting here, since it is wrapped in a list. + yield 'blocks', blocks, [spec.BeaconBlock] + + yield 'post', state + assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH assert is_active_validator( @@ -40,9 +41,8 @@ def test_activation(state): get_current_epoch(state), ) - return pre_state, blocks, state - +@spec_state_test def test_ejection(state): index = 0 assert is_active_validator(state.validator_registry[index], get_current_epoch(state)) @@ -51,17 +51,20 @@ def test_ejection(state): # Mock an ejection state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE - pre_state = deepcopy(state) + yield 'pre', state blocks = [] for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): block = next_epoch(state) blocks.append(block) + # provide extra type hinting here, since it is wrapped in a list. + yield 'blocks', blocks, [spec.BeaconBlock] + + yield 'post', state + assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH assert not is_active_validator( state.validator_registry[index], get_current_epoch(state), ) - - return pre_state, blocks, state From 89a2bd2bea0cf0f1c934ec2800ceb57c13ead45c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 17:43:02 +0200 Subject: [PATCH 013/118] work on sanity tests --- test_libs/pyspec/tests/test_sanity.py | 153 +++++++++++++++----------- 1 file changed, 90 insertions(+), 63 deletions(-) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1b4d20f4c..1f608f615 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -1,7 +1,5 @@ from copy import deepcopy -import pytest - from py_ecc import bls import eth2spec.phase0.spec as spec @@ -9,7 +7,6 @@ from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, - SLOTS_PER_HISTORICAL_ROOT, # SSZ Deposit, Transfer, @@ -37,139 +34,169 @@ from .helpers import ( get_balance, build_deposit_data, build_empty_block_for_next_slot, - fill_aggregate_attestation, get_state_root, get_valid_attestation, get_valid_attester_slashing, get_valid_proposer_slashing, - next_slot, privkeys, pubkeys, ) - -# mark entire file as 'sanity' -pytestmark = pytest.mark.sanity +from .context import spec_state_test +@spec_state_test def test_slot_transition(state): - test_state = deepcopy(state) - cache_state(test_state) - advance_slot(test_state) - assert test_state.slot == state.slot + 1 - assert get_state_root(test_state, state.slot) == state.hash_tree_root() - return test_state + pre_slot = state.slot + pre_root = state.hash_tree_root() + yield 'pre', state + + cache_state(state) + advance_slot(state) + yield 'post', state + + assert state.slot == pre_slot + 1 + assert get_state_root(state, pre_slot) == pre_root +@spec_state_test def test_empty_block_transition(state): - test_state = deepcopy(state) + pre_slot = state.slot + pre_eth1_votes = len(state.eth1_data_votes) - block = build_empty_block_for_next_slot(test_state) - state_transition(test_state, block) + yield 'pre', state - assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert get_block_root_at_slot(test_state, state.slot) == block.previous_block_root + block = build_empty_block_for_next_slot(state) + yield 'blocks', [block], [spec.BeaconBlock] - return state, [block], test_state + state_transition(state, block) + yield 'post', state + + assert len(state.eth1_data_votes) == pre_eth1_votes + 1 + assert get_block_root_at_slot(state, pre_slot) == block.previous_block_root +@spec_state_test def test_skipped_slots(state): - test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + pre_slot = state.slot + yield 'pre', state + + block = build_empty_block_for_next_slot(state) block.slot += 3 + yield 'blocks', [block], [spec.BeaconBlock] - state_transition(test_state, block) + state_transition(state, block) + yield 'post', state - assert test_state.slot == block.slot - for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.previous_block_root - - return state, [block], test_state + assert state.slot == block.slot + for slot in range(pre_slot, state.slot): + assert get_block_root_at_slot(state, slot) == block.previous_block_root +@spec_state_test def test_empty_epoch_transition(state): - test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + pre_slot = state.slot + yield 'pre', state + + block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH + yield 'blocks', [block], [spec.BeaconBlock] - state_transition(test_state, block) + state_transition(state, block) + yield 'post', state - assert test_state.slot == block.slot - for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.previous_block_root - - return state, [block], test_state + assert state.slot == block.slot + for slot in range(pre_slot, state.slot): + assert get_block_root_at_slot(state, slot) == block.previous_block_root +@spec_state_test def test_empty_epoch_transition_not_finalizing(state): - test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + # copy for later balance lookups. + pre_state = deepcopy(state) + yield 'pre', state + + block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH * 5 + yield 'blocks', [block], [spec.BeaconBlock] - state_transition(test_state, block) + state_transition(state, block) + yield 'post', state - assert test_state.slot == block.slot - assert test_state.finalized_epoch < get_current_epoch(test_state) - 4 - for index in range(len(test_state.validator_registry)): - assert get_balance(test_state, index) < get_balance(state, index) - - return state, [block], test_state + assert state.slot == block.slot + assert state.finalized_epoch < get_current_epoch(state) - 4 + for index in range(len(state.validator_registry)): + assert get_balance(state, index) < get_balance(pre_state, index) +@spec_state_test def test_proposer_slashing(state): - test_state = deepcopy(state) + # copy for later balance lookups. + pre_state = deepcopy(state) proposer_slashing = get_valid_proposer_slashing(state) validator_index = proposer_slashing.proposer_index + assert not state.validator_registry[validator_index].slashed + + yield 'pre', state + # # Add to state via block transition # - block = build_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(state) block.body.proposer_slashings.append(proposer_slashing) - state_transition(test_state, block) + yield 'blocks', [block], [spec.BeaconBlock] - assert not state.validator_registry[validator_index].slashed + state_transition(state, block) + yield 'post', state - slashed_validator = test_state.validator_registry[validator_index] + # check if slashed + slashed_validator = state.validator_registry[validator_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward - assert get_balance(test_state, validator_index) < get_balance(state, validator_index) - - return state, [block], test_state + assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) +@spec_state_test def test_attester_slashing(state): - test_state = deepcopy(state) + # copy for later balance lookups. + pre_state = deepcopy(state) + attester_slashing = get_valid_attester_slashing(state) validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] + assert not state.validator_registry[validator_index].slashed + + yield 'pre', state + # # Add to state via block transition # - block = build_empty_block_for_next_slot(test_state) + block = build_empty_block_for_next_slot(state) block.body.attester_slashings.append(attester_slashing) - state_transition(test_state, block) + yield 'blocks', [block], [spec.BeaconBlock] - assert not state.validator_registry[validator_index].slashed + state_transition(state, block) + yield 'post', state - slashed_validator = test_state.validator_registry[validator_index] + slashed_validator = state.validator_registry[validator_index] assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward - assert get_balance(test_state, validator_index) < get_balance(state, validator_index) + assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) - proposer_index = get_beacon_proposer_index(test_state) + proposer_index = get_beacon_proposer_index(state) # gained whistleblower reward assert ( - get_balance(test_state, proposer_index) > - get_balance(state, proposer_index) + get_balance(state, proposer_index) > + get_balance(pre_state, proposer_index) ) - return state, [block], test_state +# TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector def test_deposit_in_block(state): pre_state = deepcopy(state) From d2afed847ae0b9a082fa3640472f8213d3bb2745 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 18:05:40 +0200 Subject: [PATCH 014/118] fix deposit testing generalization --- .../block_processing/test_process_deposit.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index db1738acc..eda1c2f8e 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -47,8 +47,13 @@ def run_deposit_processing(state, deposit, validator_index, valid=True): - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - pre_balance = get_balance(state, validator_index) pre_validator_count = len(state.validator_registry) + pre_balance = 0 + if validator_index < pre_validator_count: + pre_balance = get_balance(state, validator_index) + else: + # if it is a new validator, it should be right at the end of the current registry. + assert validator_index == pre_validator_count yield 'pre', state yield 'deposit', deposit @@ -63,10 +68,17 @@ def run_deposit_processing(state, deposit, validator_index, valid=True): yield 'post', state - assert len(state.validator_registry) == pre_validator_count - assert len(state.balances) == pre_validator_count + if validator_index < pre_validator_count: + # top-up + assert len(state.validator_registry) == pre_validator_count + assert len(state.balances) == pre_validator_count + else: + # new validator + assert len(state.validator_registry) == pre_validator_count + 1 + assert len(state.balances) == pre_validator_count + 1 + assert state.deposit_index == state.latest_eth1_data.deposit_count - assert get_balance(state, validator_index) == pre_balance + deposit.amount + assert get_balance(state, validator_index) == pre_balance + deposit.data.amount @spec_state_test From e5a50bc8a0f94565e483de3463bc6e463f762923 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 May 2019 16:12:13 -0600 Subject: [PATCH 015/118] finish port of test_sanity to new format --- .../block_processing/test_process_deposit.py | 24 +- test_libs/pyspec/tests/helpers.py | 23 ++ test_libs/pyspec/tests/test_sanity.py | 260 ++++++++---------- 3 files changed, 134 insertions(+), 173 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index eda1c2f8e..2aed066dc 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -9,6 +9,7 @@ from eth2spec.phase0.spec import ( from tests.helpers import ( get_balance, build_deposit, + prepare_state_and_deposit, privkeys, pubkeys, ) @@ -16,29 +17,6 @@ from tests.helpers import ( from tests.context import spec_state_test -def prepare_state_and_deposit(state, validator_index, amount): - """ - Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. - """ - pre_validator_count = len(state.validator_registry) - # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * pre_validator_count - - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit, root, deposit_data_leaves = build_deposit( - state, - deposit_data_leaves, - pubkey, - privkey, - amount, - ) - - state.latest_eth1_data.deposit_root = root - state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - return deposit - - def run_deposit_processing(state, deposit, validator_index, valid=True): """ Run ``process_deposit``, yielding: diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 3b9b6904d..8eb6d8891 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -420,3 +420,26 @@ def get_state_root(state, slot) -> bytes: """ assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] + + +def prepare_state_and_deposit(state, validator_index, amount): + """ + Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. + """ + pre_validator_count = len(state.validator_registry) + # fill previous deposits with zero-hash + deposit_data_leaves = [ZERO_HASH] * pre_validator_count + + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit, root, deposit_data_leaves = build_deposit( + state, + deposit_data_leaves, + pubkey, + privkey, + amount, + ) + + state.latest_eth1_data.deposit_root = root + state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + return deposit diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1f608f615..95d7492ed 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -38,6 +38,8 @@ from .helpers import ( get_valid_attestation, get_valid_attester_slashing, get_valid_proposer_slashing, + get_valid_transfer, + prepare_state_and_deposit, privkeys, pubkeys, ) @@ -198,211 +200,164 @@ def test_attester_slashing(state): # TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector +@spec_state_test def test_deposit_in_block(state): - pre_state = deepcopy(state) - test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + initial_registry_len = len(state.validator_registry) + initial_balances_len = len(state.balances) - index = len(test_deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_EFFECTIVE_BALANCE) + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount) - item = deposit_data.hash_tree_root() - test_deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) - root = get_merkle_root((tuple(test_deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + yield 'pre', state - deposit = Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) - post_state = deepcopy(pre_state) - block = build_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(state) block.body.deposits.append(deposit) - state_transition(post_state, block) - assert len(post_state.validator_registry) == len(state.validator_registry) + 1 - assert len(post_state.balances) == len(state.balances) + 1 - assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE - assert post_state.validator_registry[index].pubkey == pubkeys[index] + yield 'blocks', [block], [spec.BeaconBlock] - return pre_state, [block], post_state + state_transition(state, block) + yield 'post', state + + assert len(state.validator_registry) == initial_registry_len + 1 + assert len(state.balances) == initial_balances_len + 1 + assert get_balance(state, validator_index) == spec.MAX_EFFECTIVE_BALANCE + assert state.validator_registry[validator_index].pubkey == pubkeys[validator_index] +@spec_state_test def test_deposit_top_up(state): - pre_state = deepcopy(state) - test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount) - merkle_index = len(test_deposit_data_leaves) - item = deposit_data.hash_tree_root() - test_deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) - root = get_merkle_root((tuple(test_deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=merkle_index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) + initial_registry_len = len(state.validator_registry) + initial_balances_len = len(state.balances) + validator_pre_balance = get_balance(state, validator_index) - deposit = Deposit( - proof=list(proof), - index=merkle_index, - data=deposit_data, - ) + yield 'pre', state - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) - block = build_empty_block_for_next_slot(pre_state) + block = build_empty_block_for_next_slot(state) block.body.deposits.append(deposit) - pre_balance = get_balance(pre_state, validator_index) - post_state = deepcopy(pre_state) - state_transition(post_state, block) - assert len(post_state.validator_registry) == len(pre_state.validator_registry) - assert len(post_state.balances) == len(pre_state.balances) - assert get_balance(post_state, validator_index) == pre_balance + amount + yield 'blocks', [block], [spec.BeaconBlock] - return pre_state, [block], post_state + state_transition(state, block) + yield 'post', state + + assert len(state.validator_registry) == initial_registry_len + assert len(state.balances) == initial_balances_len + assert get_balance(state, validator_index) == validator_pre_balance + amount +@spec_state_test def test_attestation(state): state.slot = spec.SLOTS_PER_EPOCH - test_state = deepcopy(state) + + yield 'pre', state + attestation = get_valid_attestation(state) - # # Add to state via block transition - # - attestation_block = build_empty_block_for_next_slot(test_state) + pre_current_attestations_len = len(state.current_epoch_attestations) + attestation_block = build_empty_block_for_next_slot(state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) - state_transition(test_state, attestation_block) + state_transition(state, attestation_block) - assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 + assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - # # Epoch transition should move to previous_epoch_attestations - # - pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) + pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) - epoch_block = build_empty_block_for_next_slot(test_state) + epoch_block = build_empty_block_for_next_slot(state) epoch_block.slot += spec.SLOTS_PER_EPOCH - state_transition(test_state, epoch_block) + state_transition(state, epoch_block) - assert len(test_state.current_epoch_attestations) == 0 - assert test_state.previous_epoch_attestations == pre_current_epoch_attestations + yield 'blocks', [attestation_block, epoch_block], [spec.BeaconBlock] + yield 'post', state - return state, [attestation_block, epoch_block], test_state + assert len(state.current_epoch_attestations) == 0 + assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root +@spec_state_test def test_voluntary_exit(state): - pre_state = deepcopy(state) validator_index = get_active_validator_indices( - pre_state, - get_current_epoch(pre_state) + state, + get_current_epoch(state) )[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - post_state = deepcopy(pre_state) + yield 'pre', state voluntary_exit = VoluntaryExit( - epoch=get_current_epoch(pre_state), + epoch=get_current_epoch(state), validator_index=validator_index, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), privkey=privkeys[validator_index], domain=get_domain( - state=pre_state, + state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, ) ) - # # Add to state via block transition - # - initiate_exit_block = build_empty_block_for_next_slot(post_state) + initiate_exit_block = build_empty_block_for_next_slot(state) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) - state_transition(post_state, initiate_exit_block) + state_transition(state, initiate_exit_block) - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - # # Process within epoch transition - # - exit_block = build_empty_block_for_next_slot(post_state) + exit_block = build_empty_block_for_next_slot(state) exit_block.slot += spec.SLOTS_PER_EPOCH - state_transition(post_state, exit_block) + state_transition(state, exit_block) - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + yield 'blocks', [initiate_exit_block, exit_block], [spec.BeaconBlock] + yield 'post', state - return pre_state, [initiate_exit_block, exit_block], post_state + assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH +@spec_state_test def test_transfer(state): # overwrite default 0 to test spec.MAX_TRANSFERS = 1 - pre_state = deepcopy(state) - current_epoch = get_current_epoch(pre_state) - sender_index = get_active_validator_indices(pre_state, current_epoch)[-1] - recipient_index = get_active_validator_indices(pre_state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - amount = get_balance(pre_state, sender_index) - pre_transfer_recipient_balance = get_balance(pre_state, recipient_index) - transfer = Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=0, - slot=pre_state.slot + 1, - pubkey=transfer_pubkey, - ) - transfer.signature = bls.sign( - message_hash=signing_root(transfer), - privkey=transfer_privkey, - domain=get_domain( - state=pre_state, - domain_type=spec.DOMAIN_TRANSFER, - ) - ) + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + amount = get_balance(state, sender_index) + + transfer = get_valid_transfer(state, state.slot + 1, sender_index, amount) + recipient_index = transfer.recipient + pre_transfer_recipient_balance = get_balance(state, recipient_index) - # ensure withdrawal_credentials reproducable - pre_state.validator_registry[sender_index].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:] - ) # un-activate so validator can transfer - pre_state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield 'pre', state - post_state = deepcopy(pre_state) - # # Add to state via block transition - # - block = build_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(state) block.body.transfers.append(transfer) - state_transition(post_state, block) - sender_balance = get_balance(post_state, sender_index) - recipient_balance = get_balance(post_state, recipient_index) + yield 'blocks', [block], [spec.BeaconBlock] + + state_transition(state, block) + yield 'post', state + + sender_balance = get_balance(state, sender_index) + recipient_balance = get_balance(state, recipient_index) assert sender_balance == 0 assert recipient_balance == pre_transfer_recipient_balance + amount - return pre_state, [block], post_state - +@spec_state_test def test_balance_driven_status_transitions(state): current_epoch = get_current_epoch(state) validator_index = get_active_validator_indices(state, current_epoch)[-1] @@ -412,54 +367,59 @@ def test_balance_driven_status_transitions(state): # set validator balance to below ejection threshold state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE - post_state = deepcopy(state) - # + yield 'pre', state + # trigger epoch transition - # - block = build_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH - state_transition(post_state, block) + state_transition(state, block) - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH + yield 'blocks', [block], [spec.BeaconBlock] + yield 'post', state - return state, [block], post_state + assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH +@spec_state_test def test_historical_batch(state): state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 + pre_historical_roots_len = len(state.historical_roots) - post_state = deepcopy(state) + yield 'pre', state - block = build_empty_block_for_next_slot(post_state) + block = build_empty_block_for_next_slot(state) + state_transition(state, block) - state_transition(post_state, block) + yield 'blocks', [block], [spec.BeaconBlock] + yield 'post', state - assert post_state.slot == block.slot - assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 - assert len(post_state.historical_roots) == len(state.historical_roots) + 1 - - return state, [block], post_state + assert state.slot == block.slot + assert get_current_epoch(state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 + assert len(state.historical_roots) == pre_historical_roots_len + 1 +@spec_state_test def test_eth1_data_votes(state): - post_state = deepcopy(state) + yield 'pre', state expected_votes = 0 assert len(state.eth1_data_votes) == expected_votes blocks = [] for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = build_empty_block_for_next_slot(post_state) - state_transition(post_state, block) + block = build_empty_block_for_next_slot(state) + state_transition(state, block) expected_votes += 1 - assert len(post_state.eth1_data_votes) == expected_votes + assert len(state.eth1_data_votes) == expected_votes blocks.append(block) - block = build_empty_block_for_next_slot(post_state) - state_transition(post_state, block) + block = build_empty_block_for_next_slot(state) blocks.append(block) - assert post_state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 - assert len(post_state.eth1_data_votes) == 1 + state_transition(state, block) - return state, blocks, post_state + yield 'blocks', [block], [spec.BeaconBlock] + yield 'post', state + + assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 + assert len(state.eth1_data_votes) == 1 From 8e619f8ab97732bd13f22891c0071d2c360257b9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 May 2019 16:23:31 -0600 Subject: [PATCH 016/118] port finality tests to new format. still has excessive deepcopies --- test_libs/pyspec/tests/test_finality.py | 99 +++++++++++++------------ test_libs/pyspec/tests/test_sanity.py | 12 --- 2 files changed, 50 insertions(+), 61 deletions(-) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ca048c2b2..e97ce7e40 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -1,7 +1,5 @@ from copy import deepcopy -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.state_transition import ( @@ -16,8 +14,7 @@ from .helpers import ( next_epoch, ) -# mark entire file as 'state' -pytestmark = pytest.mark.state +from .context import spec_state_test def check_finality(state, @@ -73,126 +70,130 @@ def next_epoch_with_attestations(state, return state, blocks, post_state +@spec_state_test def test_finality_rule_4(state): - test_state = deepcopy(state) + yield 'pre', state blocks = [] for epoch in range(4): - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) blocks += new_blocks # justification/finalization skipped at GENESIS_EPOCH if epoch == 0: - check_finality(test_state, prev_state, False, False, False) + check_finality(state, prev_state, False, False, False) # justification/finalization skipped at GENESIS_EPOCH + 1 elif epoch == 1: - check_finality(test_state, prev_state, False, False, False) + check_finality(state, prev_state, False, False, False) elif epoch == 2: - check_finality(test_state, prev_state, True, False, False) + check_finality(state, prev_state, True, False, False) elif epoch >= 3: # rule 4 of finality - check_finality(test_state, prev_state, True, True, True) - assert test_state.finalized_epoch == prev_state.current_justified_epoch - assert test_state.finalized_root == prev_state.current_justified_root + check_finality(state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root - return state, blocks, test_state + yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'post', state +@spec_state_test def test_finality_rule_1(state): # get past first two epochs that finality does not run on next_epoch(state) next_epoch(state) - pre_state = deepcopy(state) - test_state = deepcopy(state) + yield 'pre', state blocks = [] for epoch in range(3): - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) blocks += new_blocks if epoch == 0: - check_finality(test_state, prev_state, True, False, False) + check_finality(state, prev_state, True, False, False) elif epoch == 1: - check_finality(test_state, prev_state, True, True, False) + check_finality(state, prev_state, True, True, False) elif epoch == 2: # finalized by rule 1 - check_finality(test_state, prev_state, True, True, True) - assert test_state.finalized_epoch == prev_state.previous_justified_epoch - assert test_state.finalized_root == prev_state.previous_justified_root + check_finality(state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root - return pre_state, blocks, test_state + yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'post', state +@spec_state_test def test_finality_rule_2(state): # get past first two epochs that finality does not run on next_epoch(state) next_epoch(state) - pre_state = deepcopy(state) - test_state = deepcopy(state) + yield 'pre', state blocks = [] for epoch in range(3): if epoch == 0: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) - check_finality(test_state, prev_state, True, False, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) + check_finality(state, prev_state, True, False, False) elif epoch == 1: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) - check_finality(test_state, prev_state, False, True, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, False, False) + check_finality(state, prev_state, False, True, False) elif epoch == 2: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) # finalized by rule 2 - check_finality(test_state, prev_state, True, False, True) - assert test_state.finalized_epoch == prev_state.previous_justified_epoch - assert test_state.finalized_root == prev_state.previous_justified_root + check_finality(state, prev_state, True, False, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root blocks += new_blocks - return pre_state, blocks, test_state + yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'post', state +@spec_state_test def test_finality_rule_3(state): """ Test scenario described here https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 """ - # get past first two epochs that finality does not run on next_epoch(state) next_epoch(state) - pre_state = deepcopy(state) - test_state = deepcopy(state) + yield 'pre', state blocks = [] - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) blocks += new_blocks - check_finality(test_state, prev_state, True, False, False) + check_finality(state, prev_state, True, False, False) # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) blocks += new_blocks - check_finality(test_state, prev_state, True, True, True) + check_finality(state, prev_state, True, True, True) # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) + prev_state, new_blocks, state = next_epoch_with_attestations(state, False, False) blocks += new_blocks - check_finality(test_state, prev_state, False, True, False) + check_finality(state, prev_state, False, True, False) # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) blocks += new_blocks # rule 2 - check_finality(test_state, prev_state, True, False, True) + check_finality(state, prev_state, True, False, True) # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, True) + prev_state, new_blocks, state = next_epoch_with_attestations(state, True, True) blocks += new_blocks # rule 3 - check_finality(test_state, prev_state, True, True, True) - assert test_state.finalized_epoch == prev_state.current_justified_epoch - assert test_state.finalized_root == prev_state.current_justified_root + check_finality(state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root - return pre_state, blocks, test_state + yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'post', state diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 95d7492ed..1951415ac 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -5,11 +5,7 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( - # constants - ZERO_HASH, # SSZ - Deposit, - Transfer, VoluntaryExit, # functions get_active_validator_indices, @@ -19,20 +15,12 @@ from eth2spec.phase0.spec import ( get_domain, advance_slot, cache_state, - verify_merkle_branch, - hash, ) from eth2spec.phase0.state_transition import ( state_transition, ) -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) from .helpers import ( get_balance, - build_deposit_data, build_empty_block_for_next_slot, get_state_root, get_valid_attestation, From 683fe0062f4f4ec2cdded0cca3f6f68a22d79f66 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 7 May 2019 18:38:58 +0200 Subject: [PATCH 017/118] fix blocks --- test_libs/pyspec/tests/test_finality.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index e97ce7e40..16bf24a4e 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -93,7 +93,7 @@ def test_finality_rule_4(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'blocks', blocks, [spec.BeaconBlock] yield 'post', state @@ -120,7 +120,7 @@ def test_finality_rule_1(state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'blocks', blocks, [spec.BeaconBlock] yield 'post', state @@ -149,7 +149,7 @@ def test_finality_rule_2(state): blocks += new_blocks - yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'blocks', blocks, [spec.BeaconBlock] yield 'post', state @@ -195,5 +195,5 @@ def test_finality_rule_3(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', [blocks], [spec.BeaconBlock] + yield 'blocks', blocks, [spec.BeaconBlock] yield 'post', state From 0d06f6bcc1bb16dc77d85067e480ce3623b94934 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 8 May 2019 15:04:27 +1000 Subject: [PATCH 018/118] Added the first draft of the BN-VC API RFC, as it was listed on the issue #1011. --- .../0_beacon-node-validator-client-api.md | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 specs/validator/0_beacon-node-validator-client-api.md diff --git a/specs/validator/0_beacon-node-validator-client-api.md b/specs/validator/0_beacon-node-validator-client-api.md new file mode 100644 index 000000000..fff4107ca --- /dev/null +++ b/specs/validator/0_beacon-node-validator-client-api.md @@ -0,0 +1,235 @@ +# ETH2.0 Beacon Node & Validator Client RPC Proposal + +## Outline + +This issue proposes a minimal communications specification between a beacon node and validator client, targeting _phase 0_ of the Eth2.0 specification. The protocol is designed to be a simple local communications interface, permitting two separate binaries to communicate. + +This specification is intended to describe communication abstractly, without choosing any particular protocol. The protocol used (e.g. gRPC/JSON-RPC) is discussed in a separate issue: #1012. + +This issue follows on from a discussion during the [Client Architecture](https://notes.ethereum.org/iuPB2YjKQMGua0nwCZJVAQ) session at the [Sydney Implementers meeting](https://notes.ethereum.org/7w7diW1-St2_Yu-YHjg6NA). This also follow from the [Client Architecture Roundtable](https://hackmd.io/s/rJl05Lk6X) in Prague. +There is an editable version of this document, here: https://hackmd.io/r7D61hs4RWKm8nz_O2iinQ + +### Background +The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. + +The validator client is conceptually a separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". This includes the production of beacon blocks and signing of attestations. + +Since it is recommended to separate these concerns in the client implementations, it is necessary for us to clearly define the communication between them. + +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. + +This proposal has been adapted from the [Lighthouse gRPC protocol specification](https://github.com/sigp/lighthouse/blob/master/protos/src/services.proto). + +There is also another [WIP proposal for a Minimum Validator Interface](https://notes.ethereum.org/Ia2kvjy0RX2J-GxrWfoCAQ), which describes additional functions possibly necessary for phase 1 and beyond. + +## Specification + +### Entities +The following are the two entities that participate in this protocol: + - **`BeaconNode`**: + A beacon node instance, run with a `--rpc` flag to enable the RPC interface. Runs stand-alone. + + - **`ValidatorClient`**: +A validator client instance, which must connect to at least one instance of `BeaconNode`. + + + +### Endpoints +This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. + +This proposal is a minimum set of messages necessary to enable effective communication, without any extra features. Anything extra is beyond the scope of this document. + +#### Summary Table +| Name | Type | Parameters | Returns | +| -------- | --- | ----- | ----- | +| [`get_client_version`](#get_client_version) | GET | N/A | `client_version` | +| [`get_genesis_time`](#get_genesis_time) | GET | N/A | `genesis_time` | +| [`get_syncing_status`](#get_syncing_status) | GET | N/A | `syncing_status` | +| [`get_duties`](#get_duties) | GET | `validator_pubkeys` | `syncing_status`, `current_version`, [`ValidatorDuty`]| +| [`produce_block`](#produce_block) | GET | `slot`, `randao_reveal` | `beacon_block` | +| [`publish_block`](#publish_block) | POST | `beacon_block` | N/A | +| [`produce_attestation`](#produce_attestation) | GET | `slot`, `shard` | `indexed_attestation` | +| [`publish_attestation`](#publish_attestation) | POST | `indexed_attestation` | N/A | Publishes the IndexedAttestation after having been signed by the ValidatorClient | + +#### Status Codes +For each of these endpoints the underlying transport protocol should provide status codes. Assuming this will be based on HTTP, one of the following standard status codes will always be included as part of a response: + +| Code | Meaning | +| --- | --- | +| `200` | The API call succeeded. | +| `40X` | The request was malformed. | +| `500` | The `BeaconNode` cannot complete the request due to an internal error. | +| `503` | The `BeaconNode` is currently syncing, try again later. _A call can be made to `get_syncing_status` to in order to find out how much has been synchronised._ | + +#### `get_client_version` +Requests that the `BeaconNode` identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field. + + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `client_version` | bytes32 | An ASCII-encoded hex string which uniquely defines the implementation of the `BeaconNode` and its current software version. | + + **Note**: _Unlike most other endpoints, `get_client_version` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ + + +#### `get_genesis_time` + Requests the `genesis_time` parameter from the `BeaconNode`, which should be consistent across all `BeaconNodes` that follow the same beacon chain. + + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `genesis_time` | uint64 | The [`genesis_time`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#on-genesis), which is a fairly static configuration option for the `BeaconNode`. | + + **Note**: _Unlike most other endpoints, `get_genesis_time` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ + + +#### `get_syncing_status` + Requests the `BeaconNode` to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC [`eth_syncing`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing) call. + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `syncing` | `false` OR `SyncingStatus` | Either `false` if the node is not syncing, or a [`SyncingStatus`](#SyncingStatus) object if it is. | + + **Note**: _Unlike most other endpoints, `get_syncing_status` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200` with the `SyncingStatus` object._ + + +#### `get_duties` + Requests the BeaconNode to provide a set of “duties”, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected `BeaconNode` is properly synchronised. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `validator_pubkeys` | [bytes48] | A list of unique validator public keys, where each item is a `0x` encoded hex string. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `current_version` | bytes4 | The `current_version`, as described by the current [`Fork`](#Fork). | + | `validator_duties` | [`ValidatorDuty`] | A list where each item is a custom [`ValidatorDuty`](#ValidatorDuty) object. | + + + #### `produce_block` + Requests a `BeaconNode` to produce a valid block, which can then be signed by a ValidatorClient. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `slot` | uint64 | The slot for which the block should be proposed. | + | `randao_reveal` | bytes | The ValidatorClient's randao reveal value. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `beacon_block` | `BeaconBlock` | A proposed [`BeaconBlock`](#BeaconBlock) object, but with the `signature` field left blank. + + + #### `publish_block` + Instructs the `BeaconNode` to publish a newly signed beacon block to the beacon network, to be included in the beacon chain. + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `beacon_block` | `BeaconBlock` | The [`BeaconBlock`](#BeaconBlock) object, as sent from the `BeaconNode` originally, but now with the `signature` field completed. + + - **Returns**: N/A + + + #### `produce_attestation` + Requests that the `BeaconNode` produce an `IndexedAttestation`, with a blank `signature` field, which the `ValidatorClient` will then sign. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `slot` | uint64 | The slot for which the attestation should be proposed. | + | `shard` | uint64 | The shard number for which the attestation is to be proposed. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure with the `signature` field left blank. | + + #### `publish_attestation` + Instructs the `BeaconNode` to publish a newly signed `IndexedAttestation` object, to be incorporated into the beacon chain. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure, as originally provided by the `BeaconNode`, but now with the `signature` field completed. | + - **Returns**: N/A + + + + ----- + +### Data Structures +Two new data objects are proposed for the sake of implementation, and several other data objects from the Eth2.0 specs are referenced. + +The `bytes` data types are encoded hex strings, with `0x` preceeding them. `uint64` are decimal encoded integers, and `None` may be `null`, which is distinct from `0`. + +#### `ValidatorDuty` +```asm +{ + + # The validator's public key, uniquely identifying them + 'validator_pubkey': 'bytes48', + # The index of the validator in the committee + 'committee_index': 'uint64', + # The slot at which the validator must attest. + 'attestation_slot': 'uint64', + # The shard in which the validator must attest + 'attestation_shard': 'uint64', + # The slot in which a validator must propose a block. This field can also be None. + 'block_production_slot': 'uint64' or None +} +``` + +#### `SyncingStatus` +As described by the [Eth1.0 JSON-RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing).: +```asm +{ + # The block at which syncing started (will only be reset, after the sync reached his head) + 'startingBlock': 'uint64', + # The current block + 'currentBlock': 'uint64', + # The estimated highest block, or current target block number + 'highestBlock': 'uint64' +} +``` + +#### `Fork` +As described by [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) in the Eth2.0 specs. + +#### `BeaconBlock` +As described by [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#BeaconBlock) in the Eth2.0 specs. + +#### `IndexedAttestation` +As described by [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#IndexedAttestation) in the Eth2.0 specs. + + + + +## Optional Extras + +#### Endpoint: `get_fork` + Requests the `BeaconNode` to provide which fork version it is currently on. + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `fork` | [`Fork`](#Fork) | Provides the current version information for the fork which the `BeaconNode` is currently following. | + | `chain_id` | uint64 | Sometimes called the network id, this number discerns the active chain for the `BeaconNode`. Analagous to Eth1.0 JSON-RPC [`net_version`](https://github.com/ethereum/wiki/wiki/JSON-RPC#net_version). | + From 4d2e752bb90437afb73ce58253cb68122fac16e6 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 8 May 2019 23:51:53 +1000 Subject: [PATCH 019/118] Started updating the markdown description of the BNVC REST API, removing stuff specific to the issue and conforming to standard text layout. --- ...-api.md => 0_beacon-node-validator-api.md} | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) rename specs/validator/{0_beacon-node-validator-client-api.md => 0_beacon-node-validator-api.md} (87%) diff --git a/specs/validator/0_beacon-node-validator-client-api.md b/specs/validator/0_beacon-node-validator-api.md similarity index 87% rename from specs/validator/0_beacon-node-validator-client-api.md rename to specs/validator/0_beacon-node-validator-api.md index fff4107ca..342e7f934 100644 --- a/specs/validator/0_beacon-node-validator-client-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -1,26 +1,30 @@ -# ETH2.0 Beacon Node & Validator Client RPC Proposal +# Ethereum 2.0 Phase 0 -- Beacon Node API for Validator + +__NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. + +## Table of Contents + + + + ## Outline -This issue proposes a minimal communications specification between a beacon node and validator client, targeting _phase 0_ of the Eth2.0 specification. The protocol is designed to be a simple local communications interface, permitting two separate binaries to communicate. +This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. -This specification is intended to describe communication abstractly, without choosing any particular protocol. The protocol used (e.g. gRPC/JSON-RPC) is discussed in a separate issue: #1012. - -This issue follows on from a discussion during the [Client Architecture](https://notes.ethereum.org/iuPB2YjKQMGua0nwCZJVAQ) session at the [Sydney Implementers meeting](https://notes.ethereum.org/7w7diW1-St2_Yu-YHjg6NA). This also follow from the [Client Architecture Roundtable](https://hackmd.io/s/rJl05Lk6X) in Prague. -There is an editable version of this document, here: https://hackmd.io/r7D61hs4RWKm8nz_O2iinQ +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is conceptually a separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". This includes the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties includes the production of beacon blocks and signing of attestations. -Since it is recommended to separate these concerns in the client implementations, it is necessary for us to clearly define the communication between them. +Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. -This proposal has been adapted from the [Lighthouse gRPC protocol specification](https://github.com/sigp/lighthouse/blob/master/protos/src/services.proto). +This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) -There is also another [WIP proposal for a Minimum Validator Interface](https://notes.ethereum.org/Ia2kvjy0RX2J-GxrWfoCAQ), which describes additional functions possibly necessary for phase 1 and beyond. ## Specification @@ -33,7 +37,6 @@ The following are the two entities that participate in this protocol: A validator client instance, which must connect to at least one instance of `BeaconNode`. - ### Endpoints This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. From f9539ed1ab961e03d392252f279f754301ef74c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 8 May 2019 17:33:09 +0200 Subject: [PATCH 020/118] move tests, update operations test vector generation --- test_generators/operations/deposits.py | 180 ------------------ test_generators/operations/genesis.py | 44 ----- test_generators/operations/keys.py | 7 - test_generators/operations/main.py | 28 ++- test_generators/operations/requirements.txt | 3 +- test_generators/operations/suite_creator.py | 39 ++++ test_libs/pyspec/eth2spec/testing/__init__.py | 0 .../testing/block_processing/__init__.py | 0 .../test_process_attestation.py | 8 +- .../test_process_attester_slashing.py | 9 +- .../test_process_block_header.py | 9 +- .../block_processing/test_process_deposit.py | 12 +- .../test_process_proposer_slashing.py | 9 +- .../block_processing/test_process_transfer.py | 9 +- .../test_process_voluntary_exit.py} | 9 +- .../{tests => eth2spec/testing}/context.py | 16 +- .../testing/epoch_processing/__init__.py | 0 .../test_process_crosslinks.py | 5 +- .../test_process_registry_updates.py | 4 +- .../{tests => eth2spec/testing}/helpers.py | 0 .../testing}/test_finality.py | 0 .../testing}/test_sanity.py | 0 .../{tests => eth2spec/testing}/utils.py | 0 23 files changed, 103 insertions(+), 288 deletions(-) delete mode 100644 test_generators/operations/deposits.py delete mode 100644 test_generators/operations/genesis.py delete mode 100644 test_generators/operations/keys.py create mode 100644 test_generators/operations/suite_creator.py create mode 100644 test_libs/pyspec/eth2spec/testing/__init__.py create mode 100644 test_libs/pyspec/eth2spec/testing/block_processing/__init__.py rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_attestation.py (95%) rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_attester_slashing.py (94%) rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_block_header.py (90%) rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_deposit.py (92%) rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_proposer_slashing.py (92%) rename test_libs/pyspec/{tests => eth2spec/testing}/block_processing/test_process_transfer.py (95%) rename test_libs/pyspec/{tests/block_processing/test_voluntary_exit.py => eth2spec/testing/block_processing/test_process_voluntary_exit.py} (96%) rename test_libs/pyspec/{tests => eth2spec/testing}/context.py (56%) create mode 100644 test_libs/pyspec/eth2spec/testing/epoch_processing/__init__.py rename test_libs/pyspec/{tests => eth2spec/testing}/epoch_processing/test_process_crosslinks.py (97%) rename test_libs/pyspec/{tests => eth2spec/testing}/epoch_processing/test_process_registry_updates.py (95%) rename test_libs/pyspec/{tests => eth2spec/testing}/helpers.py (100%) rename test_libs/pyspec/{tests => eth2spec/testing}/test_finality.py (100%) rename test_libs/pyspec/{tests => eth2spec/testing}/test_sanity.py (100%) rename test_libs/pyspec/{tests => eth2spec/testing}/utils.py (100%) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py deleted file mode 100644 index 075ccbd5b..000000000 --- a/test_generators/operations/deposits.py +++ /dev/null @@ -1,180 +0,0 @@ -from eth2spec.phase0 import spec -from eth_utils import ( - to_dict, to_tuple -) -from gen_base import gen_suite, gen_typing -from preset_loader import loader -from eth2spec.debug.encode import encode -from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof - -from typing import List, Tuple - -import genesis -import keys -from py_ecc import bls - - -def build_deposit_data(state, - pubkey: spec.BLSPubkey, - withdrawal_cred: spec.Bytes32, - privkey: int, - amount: int): - deposit_data = spec.DepositData( - pubkey=pubkey, - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:], - amount=amount, - ) - deposit_data.proof_of_possession = bls.sign( - message_hash=signing_root(deposit_data), - privkey=privkey, - domain=spec.get_domain( - state, - spec.get_current_epoch(state), - spec.DOMAIN_DEPOSIT, - ) - ) - return deposit_data - - -def build_deposit(state, - deposit_data_leaves: List[spec.Bytes32], - pubkey: spec.BLSPubkey, - withdrawal_cred: spec.Bytes32, - privkey: int, - amount: int) -> spec.Deposit: - - deposit_data = build_deposit_data(state, pubkey, withdrawal_cred, privkey, amount) - - item = deposit_data.hash_tree_root() - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - proof = list(get_merkle_proof(tree, item_index=index)) - - deposit = spec.Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, get_merkle_root(tuple(deposit_data_leaves))) - - return deposit - - -def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState]: - genesis_deposits = genesis.create_deposits( - keys.pubkeys[:initial_validator_count], - keys.withdrawal_creds[:initial_validator_count] - ) - state = genesis.create_genesis_state(genesis_deposits) - - deposit_data_leaves = [dep.data.hash_tree_root() for dep in genesis_deposits] - - deposit = build_deposit( - state, - deposit_data_leaves, - keys.pubkeys[index], - keys.withdrawal_creds[index], - keys.privkeys[index], - spec.MAX_EFFECTIVE_BALANCE, - ) - - state.latest_eth1_data.deposit_root = get_merkle_root(tuple(deposit_data_leaves)) - state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - return deposit, state - - -@to_dict -def valid_deposit(): - new_dep, state = build_deposit_for_index(10, 10) - yield 'description', 'valid deposit to add new validator' - yield 'pre', encode(state, spec.BeaconState) - yield 'deposit', encode(new_dep, spec.Deposit) - spec.process_deposit(state, new_dep) - yield 'post', encode(state, spec.BeaconState) - - -@to_dict -def valid_topup(): - new_dep, state = build_deposit_for_index(10, 3) - yield 'description', 'valid deposit to top-up existing validator' - yield 'pre', encode(state, spec.BeaconState) - yield 'deposit', encode(new_dep, spec.Deposit) - spec.process_deposit(state, new_dep) - yield 'post', encode(state, spec.BeaconState) - - -@to_dict -def invalid_deposit_index(): - new_dep, state = build_deposit_for_index(10, 10) - # Mess up deposit index, 1 too small - state.deposit_index = 9 - - yield 'description', 'invalid deposit index' - yield 'pre', encode(state, spec.BeaconState) - yield 'deposit', encode(new_dep, spec.Deposit) - try: - spec.process_deposit(state, new_dep) - except AssertionError: - # expected - yield 'post', None - return - raise Exception('invalid_deposit_index has unexpectedly allowed deposit') - - -@to_dict -def invalid_deposit_proof(): - new_dep, state = build_deposit_for_index(10, 10) - # Make deposit proof invalid (at bottom of proof) - new_dep.proof[-1] = spec.ZERO_HASH - - yield 'description', 'invalid deposit proof' - yield 'pre', encode(state, spec.BeaconState) - yield 'deposit', encode(new_dep, spec.Deposit) - try: - spec.process_deposit(state, new_dep) - except AssertionError: - # expected - yield 'post', None - return - raise Exception('invalid_deposit_index has unexpectedly allowed deposit') - - -@to_tuple -def deposit_cases(): - yield valid_deposit() - yield valid_topup() - yield invalid_deposit_index() - yield invalid_deposit_proof() - - -def mini_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - presets = loader.load_presets(configs_path, 'minimal') - spec.apply_constants_preset(presets) - - return ("deposit_minimal", "deposits", gen_suite.render_suite( - title="deposit operation", - summary="Test suite for deposit type operation processing", - forks_timeline="testing", - forks=["phase0"], - config="minimal", - runner="operations", - handler="deposits", - test_cases=deposit_cases())) - - -def full_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - presets = loader.load_presets(configs_path, 'mainnet') - spec.apply_constants_preset(presets) - - return ("deposit_full", "deposits", gen_suite.render_suite( - title="deposit operation", - summary="Test suite for deposit type operation processing", - forks_timeline="mainnet", - forks=["phase0"], - config="mainnet", - runner="operations", - handler="deposits", - test_cases=deposit_cases())) diff --git a/test_generators/operations/genesis.py b/test_generators/operations/genesis.py deleted file mode 100644 index f4d63c10e..000000000 --- a/test_generators/operations/genesis.py +++ /dev/null @@ -1,44 +0,0 @@ -from eth2spec.phase0 import spec -from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof -from typing import List - - -def create_genesis_state(deposits: List[spec.Deposit]) -> spec.BeaconState: - deposit_root = get_merkle_root((tuple([(dep.data.hash_tree_root()) for dep in deposits]))) - - return spec.get_genesis_beacon_state( - deposits, - genesis_time=0, - genesis_eth1_data=spec.Eth1Data( - deposit_root=deposit_root, - deposit_count=len(deposits), - block_hash=spec.ZERO_HASH, - ), - ) - - -def create_deposits(pubkeys: List[spec.BLSPubkey], withdrawal_cred: List[spec.Bytes32]) -> List[spec.Deposit]: - - # Mock proof of possession - proof_of_possession = b'\x33' * 96 - - deposit_data = [ - spec.DepositData( - pubkey=pubkeys[i], - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[i][1:], - amount=spec.MAX_EFFECTIVE_BALANCE, - proof_of_possession=proof_of_possession, - ) for i in range(len(pubkeys)) - ] - - # Fill tree with existing deposits - deposit_data_leaves = [data.hash_tree_root() for data in deposit_data] - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - - return [ - spec.Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - data=deposit_data[i] - ) for i in range(len(deposit_data)) - ] diff --git a/test_generators/operations/keys.py b/test_generators/operations/keys.py deleted file mode 100644 index db4f59e0e..000000000 --- a/test_generators/operations/keys.py +++ /dev/null @@ -1,7 +0,0 @@ -from py_ecc import bls -from eth2spec.phase0.spec import hash - -privkeys = list(range(1, 101)) -pubkeys = [bls.privtopub(k) for k in privkeys] -# Insecure, but easier to follow -withdrawal_creds = [hash(bls.privtopub(k)) for k in privkeys] diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 8b0a2a6d8..ab8f0a98a 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,9 +1,31 @@ +from eth2spec.testing.block_processing import ( + test_process_attestation, + test_process_attester_slashing, + test_process_block_header, + test_process_deposit, + test_process_proposer_slashing, + test_process_transfer, + test_process_voluntary_exit +) + from gen_base import gen_runner -from deposits import mini_deposits_suite, full_deposits_suite +from suite_creator import generate_from_tests, create_suite if __name__ == "__main__": gen_runner.run_generator("operations", [ - mini_deposits_suite, - full_deposits_suite + create_suite('attestation', 'minimal', lambda: generate_from_tests(test_process_attestation)), + create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation)), + create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)), + create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), + create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), + create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), + create_suite('deposits', 'minimal', lambda: generate_from_tests(test_process_deposit)), + create_suite('deposits', 'mainnet', lambda: generate_from_tests(test_process_deposit)), + create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), + create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing)), + create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer)), + create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer)), + create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit)), + create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit)), ]) diff --git a/test_generators/operations/requirements.txt b/test_generators/operations/requirements.txt index dfe853536..8f9bede8f 100644 --- a/test_generators/operations/requirements.txt +++ b/test_generators/operations/requirements.txt @@ -1,5 +1,4 @@ eth-utils==1.4.1 ../../test_libs/gen_helpers ../../test_libs/config_helpers -../../test_libs/pyspec -py_ecc \ No newline at end of file +../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py new file mode 100644 index 000000000..0186d026f --- /dev/null +++ b/test_generators/operations/suite_creator.py @@ -0,0 +1,39 @@ +from typing import Callable, Iterable +from inspect import getmembers, isfunction +from gen_base import gen_suite, gen_typing +from preset_loader import loader +from eth2spec.phase0 import spec + + +def generate_from_tests(pkg): + fn_names = [ + name for (name, _) in getmembers(pkg, isfunction) + if name.startswith('test_') + ] + out = [] + print("generating test vectors from tests package: %s" % pkg.__name__) + for name in fn_names: + tfn = getattr(pkg, name) + try: + out.append(tfn(generator_mode=True)) + except AssertionError: + print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__)) + return out + + +def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]])\ + -> Callable[[str], gen_typing.TestSuiteOutput]: + def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite( + title="%s operation" % operation_name, + summary="Test suite for deposit type operation processing", + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="operations", + handler=config_name, + test_cases=get_cases())) + return suite_definition diff --git a/test_libs/pyspec/eth2spec/testing/__init__.py b/test_libs/pyspec/eth2spec/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/__init__.py b/test_libs/pyspec/eth2spec/testing/block_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py similarity index 95% rename from test_libs/pyspec/tests/block_processing/test_process_attestation.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py index 8b43416a6..c162945f0 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py @@ -1,5 +1,4 @@ from copy import deepcopy -import pytest import eth2spec.phase0.spec as spec @@ -10,14 +9,14 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( build_empty_block_for_next_slot, get_valid_attestation, next_epoch, next_slot, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_attestation_processing(state, attestation, valid=True): @@ -35,8 +34,7 @@ def run_attestation_processing(state, attestation, valid=True): # If the attestation is invalid, processing is aborted, and there is no post-state. if not valid: - with pytest.raises(AssertionError): - process_attestation(state, attestation) + expect_assertion_error(lambda: process_attestation(state, attestation)) yield 'post', None return diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py similarity index 94% rename from test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py index 4a19fccb3..6c87324e7 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py @@ -1,17 +1,15 @@ -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( get_balance, get_valid_attester_slashing, next_epoch, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_attester_slashing_processing(state, attester_slashing, valid=True): @@ -27,8 +25,7 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): yield 'attester_slashing', attester_slashing if not valid: - with pytest.raises(AssertionError): - process_attester_slashing(state, attester_slashing) + expect_assertion_error(lambda: process_attester_slashing(state, attester_slashing)) yield 'post', None return diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py similarity index 90% rename from test_libs/pyspec/tests/block_processing/test_process_block_header.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py index 199f2bda4..278faa2c2 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py @@ -1,6 +1,4 @@ from copy import deepcopy -import pytest - from eth2spec.phase0.spec import ( get_beacon_proposer_index, @@ -8,12 +6,12 @@ from eth2spec.phase0.spec import ( advance_slot, process_block_header, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( build_empty_block_for_next_slot, next_slot, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def prepare_state_for_header_processing(state): @@ -35,8 +33,7 @@ def run_block_header_processing(state, block, valid=True): yield 'block', block if not valid: - with pytest.raises(AssertionError): - process_block_header(state, block) + expect_assertion_error(lambda: process_block_header(state, block)) yield 'post', None return diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py similarity index 92% rename from test_libs/pyspec/tests/block_processing/test_process_deposit.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py index 2aed066dc..9fbe77d40 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py @@ -1,12 +1,10 @@ -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( ZERO_HASH, process_deposit, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( get_balance, build_deposit, prepare_state_and_deposit, @@ -14,7 +12,7 @@ from tests.helpers import ( pubkeys, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_deposit_processing(state, deposit, validator_index, valid=True): @@ -37,8 +35,7 @@ def run_deposit_processing(state, deposit, validator_index, valid=True): yield 'deposit', deposit if not valid: - with pytest.raises(AssertionError): - process_deposit(state, deposit) + expect_assertion_error(lambda: process_deposit(state, deposit)) yield 'post', None return @@ -90,6 +87,9 @@ def test_wrong_index(state): yield from run_deposit_processing(state, deposit, validator_index, valid=False) +# TODO: test invalid signature + + @spec_state_test def test_bad_merkle_proof(state): validator_index = len(state.validator_registry) diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py similarity index 92% rename from test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py index 5a16c7d6c..cf4867776 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py @@ -1,16 +1,14 @@ -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( get_balance, get_valid_proposer_slashing, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_proposer_slashing_processing(state, proposer_slashing, valid=True): @@ -27,8 +25,7 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): yield 'proposer_slashing', proposer_slashing if not valid: - with pytest.raises(AssertionError): - process_proposer_slashing(state, proposer_slashing) + expect_assertion_error(lambda: process_proposer_slashing(state, proposer_slashing)) yield 'post', None return diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py similarity index 95% rename from test_libs/pyspec/tests/block_processing/test_process_transfer.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py index 8fc4d1fc4..e8f96a710 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py @@ -1,5 +1,3 @@ -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( @@ -8,12 +6,12 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_transfer, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( get_valid_transfer, next_epoch, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_transfer_processing(state, transfer, valid=True): @@ -34,8 +32,7 @@ def run_transfer_processing(state, transfer, valid=True): yield 'transfer', transfer if not valid: - with pytest.raises(AssertionError): - process_transfer(state, transfer) + expect_assertion_error(lambda: process_transfer(state, transfer)) yield 'post', None return diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py similarity index 96% rename from test_libs/pyspec/tests/block_processing/test_voluntary_exit.py rename to test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py index 566ebf028..a25981e3b 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py @@ -1,5 +1,3 @@ -import pytest - import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( @@ -8,12 +6,12 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_voluntary_exit, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( build_voluntary_exit, pubkey_to_privkey, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test, expect_assertion_error def run_voluntary_exit_processing(state, voluntary_exit, valid=True): @@ -31,8 +29,7 @@ def run_voluntary_exit_processing(state, voluntary_exit, valid=True): yield 'voluntary_exit', voluntary_exit if not valid: - with pytest.raises(AssertionError): - process_voluntary_exit(state, voluntary_exit) + expect_assertion_error(lambda: process_voluntary_exit(state, voluntary_exit)) yield 'post', None return diff --git a/test_libs/pyspec/tests/context.py b/test_libs/pyspec/eth2spec/testing/context.py similarity index 56% rename from test_libs/pyspec/tests/context.py rename to test_libs/pyspec/eth2spec/testing/context.py index d78acdc0b..d402714a1 100644 --- a/test_libs/pyspec/tests/context.py +++ b/test_libs/pyspec/eth2spec/testing/context.py @@ -1,12 +1,8 @@ - from eth2spec.phase0 import spec -from tests.utils import with_args -from .helpers import ( - create_genesis_state, -) +from .helpers import create_genesis_state -from tests.utils import spectest +from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())]) @@ -15,3 +11,11 @@ with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, l # shorthand for decorating @with_state @spectest() def spec_state_test(fn): return with_state(spectest()(fn)) + + +def expect_assertion_error(fn): + try: + fn() + raise AssertionError('expected an assertion error, but got none.') + except AssertionError: + pass diff --git a/test_libs/pyspec/eth2spec/testing/epoch_processing/__init__.py b/test_libs/pyspec/eth2spec/testing/epoch_processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py similarity index 97% rename from test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py rename to test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py index 10de064ed..2d4044baf 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py @@ -10,7 +10,7 @@ from eth2spec.phase0.spec import ( get_crosslink_deltas, process_crosslinks, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( add_attestation_to_state, build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -19,8 +19,7 @@ from tests.helpers import ( next_epoch, next_slot, ) - -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test def run_process_crosslinks(state, valid=True): diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py similarity index 95% rename from test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py rename to test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py index 85d0d07f4..8ba1436ff 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py @@ -4,11 +4,11 @@ from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, ) -from tests.helpers import ( +from eth2spec.testing.helpers import ( next_epoch, ) -from tests.context import spec_state_test +from eth2spec.testing.context import spec_state_test @spec_state_test diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/eth2spec/testing/helpers.py similarity index 100% rename from test_libs/pyspec/tests/helpers.py rename to test_libs/pyspec/eth2spec/testing/helpers.py diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/eth2spec/testing/test_finality.py similarity index 100% rename from test_libs/pyspec/tests/test_finality.py rename to test_libs/pyspec/eth2spec/testing/test_finality.py diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/eth2spec/testing/test_sanity.py similarity index 100% rename from test_libs/pyspec/tests/test_sanity.py rename to test_libs/pyspec/eth2spec/testing/test_sanity.py diff --git a/test_libs/pyspec/tests/utils.py b/test_libs/pyspec/eth2spec/testing/utils.py similarity index 100% rename from test_libs/pyspec/tests/utils.py rename to test_libs/pyspec/eth2spec/testing/utils.py From d9baee24816d2d1db38d2811af2aaf0963cda921 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 8 May 2019 18:14:47 +0200 Subject: [PATCH 021/118] move tests to standard pkg/test folder, enable conftest options with soft-import, update readme and makefile --- Makefile | 2 +- test_generators/operations/main.py | 2 +- test_libs/pyspec/README.md | 3 +- .../eth2spec/{testing => test}/__init__.py | 0 .../block_processing/__init__.py | 0 .../test_process_attestation.py | 4 +-- .../test_process_attester_slashing.py | 4 +-- .../test_process_block_header.py | 4 +-- .../block_processing/test_process_deposit.py | 4 +-- .../test_process_proposer_slashing.py | 4 +-- .../block_processing/test_process_transfer.py | 4 +-- .../test_process_voluntary_exit.py | 4 +-- test_libs/pyspec/eth2spec/test/conftest.py | 36 +++++++++++++++++++ .../eth2spec/{testing => test}/context.py | 0 .../epoch_processing/__init__.py | 0 .../test_process_crosslinks.py | 4 +-- .../test_process_registry_updates.py | 4 +-- .../eth2spec/{testing => test}/helpers.py | 0 .../{testing => test}/test_finality.py | 0 .../eth2spec/{testing => test}/test_sanity.py | 0 .../eth2spec/{testing => test}/utils.py | 0 test_libs/pyspec/tests/__init__.py | 0 test_libs/pyspec/tests/conftest.py | 17 --------- 23 files changed, 58 insertions(+), 38 deletions(-) rename test_libs/pyspec/eth2spec/{testing => test}/__init__.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/__init__.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_attestation.py (97%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_attester_slashing.py (96%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_block_header.py (94%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_deposit.py (96%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_proposer_slashing.py (96%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_transfer.py (97%) rename test_libs/pyspec/eth2spec/{testing => test}/block_processing/test_process_voluntary_exit.py (97%) create mode 100644 test_libs/pyspec/eth2spec/test/conftest.py rename test_libs/pyspec/eth2spec/{testing => test}/context.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/epoch_processing/__init__.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/epoch_processing/test_process_crosslinks.py (97%) rename test_libs/pyspec/eth2spec/{testing => test}/epoch_processing/test_process_registry_updates.py (95%) rename test_libs/pyspec/eth2spec/{testing => test}/helpers.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/test_finality.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/test_sanity.py (100%) rename test_libs/pyspec/eth2spec/{testing => test}/utils.py (100%) delete mode 100644 test_libs/pyspec/tests/__init__.py delete mode 100644 test_libs/pyspec/tests/conftest.py diff --git a/Makefile b/Makefile index 73d8adea8..86303680d 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ install_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . + cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest eth2spec citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index ab8f0a98a..88c302032 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,4 +1,4 @@ -from eth2spec.testing.block_processing import ( +from eth2spec.test.block_processing import ( test_process_attestation, test_process_attester_slashing, test_process_block_header, diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index df1834210..330972e77 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -46,8 +46,9 @@ The `-B` flag may be helpful to force-overwrite the `pyspec` output after you ma Run the tests: ``` -pytest --config=minimal +pytest --config=minimal eth2spec ``` +Note the package-name, this is to locate the tests. ## Contributing diff --git a/test_libs/pyspec/eth2spec/testing/__init__.py b/test_libs/pyspec/eth2spec/test/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/__init__.py rename to test_libs/pyspec/eth2spec/test/__init__.py diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/__init__.py b/test_libs/pyspec/eth2spec/test/block_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/block_processing/__init__.py rename to test_libs/pyspec/eth2spec/test/block_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py similarity index 97% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index c162945f0..e65943c84 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -9,14 +9,14 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( build_empty_block_for_next_slot, get_valid_attestation, next_epoch, next_slot, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_attestation_processing(state, attestation, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py similarity index 96% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 6c87324e7..957b9a9f0 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -3,13 +3,13 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( get_balance, get_valid_attester_slashing, next_epoch, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_attester_slashing_processing(state, attester_slashing, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py similarity index 94% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 278faa2c2..a176f7958 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -6,12 +6,12 @@ from eth2spec.phase0.spec import ( advance_slot, process_block_header, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( build_empty_block_for_next_slot, next_slot, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def prepare_state_for_header_processing(state): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py similarity index 96% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 9fbe77d40..fe2dae6a8 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -4,7 +4,7 @@ from eth2spec.phase0.spec import ( ZERO_HASH, process_deposit, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( get_balance, build_deposit, prepare_state_and_deposit, @@ -12,7 +12,7 @@ from eth2spec.testing.helpers import ( pubkeys, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_deposit_processing(state, deposit, validator_index, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py similarity index 96% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index cf4867776..3bfb17062 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -3,12 +3,12 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( get_balance, get_valid_proposer_slashing, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_proposer_slashing_processing(state, proposer_slashing, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py similarity index 97% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index e8f96a710..31b136cc2 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -6,12 +6,12 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_transfer, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( get_valid_transfer, next_epoch, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_transfer_processing(state, transfer, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py similarity index 97% rename from test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py rename to test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index a25981e3b..1efc950e0 100644 --- a/test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -6,12 +6,12 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_voluntary_exit, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( build_voluntary_exit, pubkey_to_privkey, ) -from eth2spec.testing.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error def run_voluntary_exit_processing(state, voluntary_exit, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py new file mode 100644 index 000000000..dadb0d5d0 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -0,0 +1,36 @@ +from eth2spec.phase0 import spec + +# We import pytest only when it's present, i.e. when we are running tests. +# The test-cases themselves can be generated without installing pytest. + +def module_exists(module_name): + try: + __import__(module_name) + except ImportError: + return False + else: + return True + + +def fixture(*args, **kwargs): + if module_exists("pytest"): + import pytest + return pytest.fixture(*args, **kwargs) + else: + def ignore(): + pass + return ignore + + +def pytest_addoption(parser): + parser.addoption( + "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + ) + + +@fixture(autouse=True) +def config(request): + config_name = request.config.getoption("--config") + from preset_loader import loader + presets = loader.load_presets('../../configs/', config_name) + spec.apply_constants_preset(presets) diff --git a/test_libs/pyspec/eth2spec/testing/context.py b/test_libs/pyspec/eth2spec/test/context.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/context.py rename to test_libs/pyspec/eth2spec/test/context.py diff --git a/test_libs/pyspec/eth2spec/testing/epoch_processing/__init__.py b/test_libs/pyspec/eth2spec/test/epoch_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/epoch_processing/__init__.py rename to test_libs/pyspec/eth2spec/test/epoch_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py similarity index 97% rename from test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py rename to test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 2d4044baf..203978d29 100644 --- a/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -10,7 +10,7 @@ from eth2spec.phase0.spec import ( get_crosslink_deltas, process_crosslinks, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( add_attestation_to_state, build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -19,7 +19,7 @@ from eth2spec.testing.helpers import ( next_epoch, next_slot, ) -from eth2spec.testing.context import spec_state_test +from eth2spec.test.context import spec_state_test def run_process_crosslinks(state, valid=True): diff --git a/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py similarity index 95% rename from test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py rename to test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 8ba1436ff..970c30942 100644 --- a/test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -4,11 +4,11 @@ from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, ) -from eth2spec.testing.helpers import ( +from eth2spec.test.helpers import ( next_epoch, ) -from eth2spec.testing.context import spec_state_test +from eth2spec.test.context import spec_state_test @spec_state_test diff --git a/test_libs/pyspec/eth2spec/testing/helpers.py b/test_libs/pyspec/eth2spec/test/helpers.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/helpers.py rename to test_libs/pyspec/eth2spec/test/helpers.py diff --git a/test_libs/pyspec/eth2spec/testing/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/test_finality.py rename to test_libs/pyspec/eth2spec/test/test_finality.py diff --git a/test_libs/pyspec/eth2spec/testing/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/test_sanity.py rename to test_libs/pyspec/eth2spec/test/test_sanity.py diff --git a/test_libs/pyspec/eth2spec/testing/utils.py b/test_libs/pyspec/eth2spec/test/utils.py similarity index 100% rename from test_libs/pyspec/eth2spec/testing/utils.py rename to test_libs/pyspec/eth2spec/test/utils.py diff --git a/test_libs/pyspec/tests/__init__.py b/test_libs/pyspec/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py deleted file mode 100644 index 8b90ce3b0..000000000 --- a/test_libs/pyspec/tests/conftest.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest - -from eth2spec.phase0 import spec -from preset_loader import loader - - -def pytest_addoption(parser): - parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" - ) - - -@pytest.fixture(autouse=True) -def config(request): - config_name = request.config.getoption("--config") - presets = loader.load_presets('../../configs/', config_name) - spec.apply_constants_preset(presets) From 39fd625d350a456f4020d07e13cd56dbe3514d9b Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 9 May 2019 23:49:59 +1000 Subject: [PATCH 022/118] Started porting the API proposal into OpenAPI 3 format. --- specs/validator/beacon_node_oapi.yaml | 152 ++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 specs/validator/beacon_node_oapi.yaml diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml new file mode 100644 index 000000000..3eac70a35 --- /dev/null +++ b/specs/validator/beacon_node_oapi.yaml @@ -0,0 +1,152 @@ +openapi: "3.0.2" +info: + title: "Beacon Node API for Validator" + description: "A beacon node API for enabling a validator to perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + version: "0.1" +paths: + /node/version: + get: + summary: "Get version string of the running beacon node." + description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." + responses: + 200: + description: Success + content: + application/json: + schema: + title: Duties + type: object + + /node/genesis_time: + get: + summary: "Get the genesis_time parameter from beacon node configuration." + description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." + responses: + 200: + description: Success + content: + application/json: + schema: + title: genesis_time + type: integer + + /node/syncing: + get: + summary: "Poll to see if the the beacon node is syncing." + description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." + responses: + 200: + description: Success + content: + application/json: + schema: + title: genesis_time + type: integer + + + + /validator/duties: + get: + summary: "Get validator duties for the requested validators." + description: "Requests the BeaconNode to provide a set of _duties_, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected BeaconNode is properly synchronised." + parameters: + - name: validator_pubkeys + in: query + required: true + schema: + type: array + items: + type: string + format: byte + responses: + 200: + description: Success response + content: + application/json: + schema: + title: Duties + type: object + 503: + description: Beacon node syncing + + + /validator/block: + get: + summary: "Produce a new block, without signature." + description: "Requests a BeaconNode to produce a valid block, which can then be signed by a ValidatorClient." + parameters: + - name: slot + in: query + required: true + schema: + type: integer + - name: randao_reveal + in: query + required: true + schema: + type: string + format: byte + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + title: BeaconBlock + post: + summary: "Publish a signed block" + description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + parameters: + - name: beacon_block + in: query + required: true + schema: + type: object + title: BeaconBlock + responses: + 200: + description: Success response + 503: + description: Beacon node syncing + + + /validator/attestation: + get: + summary: "Produce an attestation, without signature." + description: "Requests that the BeaconNode produce an IndexedAttestation, with a blank signature field, which the ValidatorClient will then sign." + parameters: + - name: slot + in: query + required: true + schema: + type: integer + - name: shard + in: query + required: true + schema: + type: integer + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + title: IndexedAttestation + post: + summary: "Published a signed attestation." + description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + parameters: + - name: attestation + in: query + required: true + description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." + schema: + type: object + title: IndexedAttestation + responses: + 200: + description: Success response + 503: + description: Beacon node syncing From b918cc3de35d6474f022c95e39402bac900dfec1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 10 May 2019 14:03:59 +1000 Subject: [PATCH 023/118] Fleshed out a whole lot more of the OpenAPI specification for the API. --- specs/validator/beacon_node_oapi.yaml | 240 +++++++++++++++++++++++--- 1 file changed, 213 insertions(+), 27 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 3eac70a35..0a583bdd6 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -10,12 +10,13 @@ paths: description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: Duties - type: object + $ref: '#/components/schemas/client_version' + 500: + $ref: '#/components/responses/InternalError' /node/genesis_time: get: @@ -23,12 +24,14 @@ paths: description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: genesis_time - type: integer + $ref: '#/components/schemas/genesis_time' + 500: + $ref: '#/components/responses/InternalError' + /node/syncing: get: @@ -36,13 +39,19 @@ paths: description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: genesis_time - type: integer - + type: object + properties: + is_syncing: + type: boolean + description: "A boolean of whether the node is currently syncing or not." + sync_status: + $ref: '#/components/schemas/SyncingStatus' + 500: + $ref: '#/components/responses/InternalError' /validator/duties: @@ -53,21 +62,28 @@ paths: - name: validator_pubkeys in: query required: true + description: "An array of hex-encoded BLS public keys" schema: type: array items: - type: string - format: byte + $ref: '#/components/schemas/pubkey' + minItems: 1 responses: 200: description: Success response content: application/json: schema: - title: Duties - type: object + type: array + items: + $ref: '#/components/schemas/ValidatorDuty' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' + /validator/block: @@ -78,11 +94,14 @@ paths: - name: slot in: query required: true + description: "The slot for which the block should be proposed." schema: type: integer + format: uint64 - name: randao_reveal in: query required: true + description: "The ValidatorClient's randao reveal value." schema: type: string format: byte @@ -92,8 +111,13 @@ paths: content: application/json: schema: - type: object - title: BeaconBlock + $ref: '#/components/schemas/BeaconBlock' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' post: summary: "Publish a signed block" description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." @@ -101,14 +125,18 @@ paths: - name: beacon_block in: query required: true + description: "The BeaconBlock object, as sent from the BeaconNode originally, but now with the signature field completed." schema: - type: object - title: BeaconBlock + $ref: '#/components/schemas/BeaconBlock' responses: 200: - description: Success response + $ref: '#/components/responses/Success' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' /validator/attestation: @@ -119,11 +147,13 @@ paths: - name: slot in: query required: true + description: "The slot for which the attestation should be proposed." schema: type: integer - name: shard in: query required: true + description: "The shard number for which the attestation is to be proposed." schema: type: integer responses: @@ -132,8 +162,14 @@ paths: content: application/json: schema: - type: object - title: IndexedAttestation + $ref: '#/components/schemas/IndexedAttestation' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + post: summary: "Published a signed attestation." description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." @@ -143,10 +179,160 @@ paths: required: true description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." schema: - type: object - title: IndexedAttestation + $ref: '#/components/schemas/IndexedAttestation' responses: 200: - description: Success response + $ref: '#/components/responses/Success' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' + +components: + schemas: + ValidatorDuty: + type: object + properties: + validator_pubkey: + $ref: '#/components/schemas/pubkey' + committee_index: + type: integer + format: uint64 + description: "The index of the validator in the committee." + attestation_slot: + type: integer + format: uint64 + description: "The slot at which the validator must attest." + attestation_shard: + type: integer + format: uint64 + description: "The shard in which the validator must attest." + block_production_slot: + type: integer + format: uint64 + nullable: true + description: "The slot in which a validator must propose a block, or `null` if block production is not required." + SyncingStatus: + type: object + nullable: true + properties: + starting_block: + type: integer + format: uint64 + description: "The block at which syncing started (will only be reset after the sync reached its head)" + current_block: + type: integer + format: uint64 + description: "The current highest block sync'd by the beacon node." + highest_block: + type: integer + format: uint64 + description: "The estimated highest block, or current target block number." + BeaconBlock: + type: object + description: "The [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) from the Eth2.0 spec." + Fork: + type: object + description: "The [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) from the Eth2.0 spec." + properties: + previous_version: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{8}$" + description: "Previous fork version" + current_version: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{8}$" + description: "Current fork version" + epoch: + type: integer + format: uint64 + description: "Fork epoch number" + IndexedAttestation: + type: object + description: "The [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) from the Eth2.0 spec." + properties: + custody_bit_0_indicies: + type: array + description: "Validator indicies for 0 bits." + items: + type: integer + format: uint64 + custody_bit_1_indicies: + type: array + description: "Validator indicies for 1 bits." + items: + type: integer + format: uint64 + data: + $ref: '#/components/schemas/AttestationData' + + AttestationData: + type: object + description: "The [AttestationData](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) from the Eth2.0 spec." + properties: + beacon_block_root: + $ref: '#/components/schemas/merkle_root' + source_epoch: + type: integer + format: uint64 + description: "Source epoch from FFG vote" + source_root: + $ref: '#/components/schemas/merkle_root' + target_epoch: + type: integer + format: uint64 + description: "Target epoch from FFG vote" + target_root: + $ref: '#/components/schemas/merkle_root' + crosslink: + $ref: '#/components/schemas/CrossLink' + CrossLink: + type: object + description: "The [Crosslink](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) from the Eth2.0 spec." + properties: + shard: + type: integer + format: uint64 + description: "The shard number" + epoch: + type: integer + format: uint64 + description: "The epoch number" + parent_root: + $ref: '#/components/schemas/merkle_root' + data_root: + $ref: '#/components/schemas/merkle_root' + + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "The validator's BLS public key, uniquely identifying them." + client_version: + type: string + description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." + example: "Lighthouse / v0.1.5 (Linux x86_64)" + genesis_time: + type: integer + format: uint64 + description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + merkle_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A 32 byte merkle root, used in all kinds of scenarios." + + + responses: + Success: + description: Request successful + InvalidRequest: + description: Invalid request syntax + InternalError: + description: Beacon node internal error + CurrentlySyncing: + description: Beacon node is currently syncing, try again later From 8570584a95f237154f49130ad2d1ad21e3d9e9a7 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 10 May 2019 00:42:27 +0200 Subject: [PATCH 024/118] PR 1052 apply suggestions Co-Authored-By: Danny Ryan --- test_generators/operations/main.py | 4 ++-- test_generators/operations/suite_creator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 88c302032..41a6ff806 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -20,8 +20,8 @@ if __name__ == "__main__": create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), - create_suite('deposits', 'minimal', lambda: generate_from_tests(test_process_deposit)), - create_suite('deposits', 'mainnet', lambda: generate_from_tests(test_process_deposit)), + create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit)), + create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit)), create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing)), create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer)), diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py index 0186d026f..d27e3efc1 100644 --- a/test_generators/operations/suite_creator.py +++ b/test_generators/operations/suite_creator.py @@ -29,7 +29,7 @@ def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite( title="%s operation" % operation_name, - summary="Test suite for deposit type operation processing", + summary="Test suite for %s type operation processing" % operation_name, forks_timeline="testing", forks=["phase0"], config=config_name, From 3189cf00797e5e9b76024428b32996fe5bc237b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 17:01:12 +0200 Subject: [PATCH 025/118] new proposer slashing tests Co-Authored-By: jannikluhn --- .../test_process_proposer_slashing.py | 22 ++++++++++++++++++- test_libs/pyspec/eth2spec/test/context.py | 3 +++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 3bfb17062..609c97ce6 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -19,7 +19,6 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ - pre_proposer_balance = get_balance(state, proposer_slashing.proposer_index) yield 'pre', state yield 'proposer_slashing', proposer_slashing @@ -29,6 +28,8 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): yield 'post', None return + pre_proposer_balance = get_balance(state, proposer_slashing.proposer_index) + process_proposer_slashing(state, proposer_slashing) yield 'post', state @@ -52,6 +53,15 @@ def test_success(state): yield from run_proposer_slashing_processing(state, proposer_slashing) +@spec_state_test +def test_invalid_proposer_index(state): + proposer_slashing = get_valid_proposer_slashing(state) + # Index just too high (by 1) + proposer_slashing.proposer_index = len(state.validator_registry) + + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + @spec_state_test def test_epochs_are_different(state): proposer_slashing = get_valid_proposer_slashing(state) @@ -72,6 +82,16 @@ def test_headers_are_same(state): yield from run_proposer_slashing_processing(state, proposer_slashing, False) +@spec_state_test +def test_proposer_is_not_activated(state): + proposer_slashing = get_valid_proposer_slashing(state) + + # set proposer to be not active yet + state.validator_registry[proposer_slashing.proposer_index].activation_epoch = get_current_epoch(state) + 1 + + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + @spec_state_test def test_proposer_is_slashed(state): proposer_slashing = get_valid_proposer_slashing(state) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index d402714a1..408b7f867 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -19,3 +19,6 @@ def expect_assertion_error(fn): raise AssertionError('expected an assertion error, but got none.') except AssertionError: pass + except IndexError: + # Index errors are special; the spec is not explicit on bound checking, an IndexError is like a failed assert. + pass From a39623a95ad56bfe9590969f89fbeedf8a1df1e5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 17:51:02 +0200 Subject: [PATCH 026/118] fix test tooling bug --- test_libs/pyspec/eth2spec/test/context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 408b7f867..afabd4a57 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -14,11 +14,14 @@ def spec_state_test(fn): def expect_assertion_error(fn): + bad = False try: fn() - raise AssertionError('expected an assertion error, but got none.') + bad = True except AssertionError: pass except IndexError: # Index errors are special; the spec is not explicit on bound checking, an IndexError is like a failed assert. pass + if bad: + raise AssertionError('expected an assertion error, but got none.') From b4be220a48ecc61853ac54aae01f9ed471324ba9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 17:52:44 +0200 Subject: [PATCH 027/118] new attestation tests Co-Authored-By: Dmitry S --- .../test_process_attestation.py | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index e65943c84..248b04ef4 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -63,7 +63,7 @@ def test_success(state): @spec_state_test -def test_success_prevous_epoch(state): +def test_success_previous_epoch(state): attestation = get_valid_attestation(state) block = build_empty_block_for_next_slot(state) block.slot = state.slot + spec.SLOTS_PER_EPOCH @@ -92,11 +92,72 @@ def test_after_epoch_slots(state): @spec_state_test -def test_bad_source_epoch(state): +def test_old_source_epoch(state): + state.slot = spec.SLOTS_PER_EPOCH * 5 + state.finalized_epoch = 2 + state.previous_justified_epoch = 3 + state.current_justified_epoch = 4 + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + + # test logic sanity check: make sure the attestation is pointing to oldest known source epoch + assert attestation.data.source_epoch == state.previous_justified_epoch + + # Now go beyond that, it will be invalid + attestation.data.source_epoch -= 1 + + yield from run_attestation_processing(state, attestation, False) + + +@spec_state_test +def test_wrong_shard(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.source_epoch += 10 + attestation.data.shard += 1 + + yield from run_attestation_processing(state, attestation, False) + + +@spec_state_test +def test_new_source_epoch(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.source_epoch += 1 + + yield from run_attestation_processing(state, attestation, False) + + +@spec_state_test +def test_source_root_is_target_root(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.data.source_root = attestation.data.target_root + + yield from run_attestation_processing(state, attestation, False) + + +@spec_state_test +def test_invalid_current_source_root(state): + state.slot = spec.SLOTS_PER_EPOCH * 5 + state.finalized_epoch = 2 + + state.previous_justified_epoch = 3 + state.previous_justified_root = b'\x01' * 32 + + state.current_justified_epoch = 4 + state.current_justified_root = b'\xff' * 32 + + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + # Test logic sanity checks: + assert state.current_justified_root != state.previous_justified_root + assert attestation.data.source_root == state.previous_justified_root + + # Make attestation source root invalid: should be previous justified, not current one + attestation.data.source_root = state.current_justified_root yield from run_attestation_processing(state, attestation, False) From 17d057e50349c4a876bc24e007c35343021b7ad1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 18:24:30 +0200 Subject: [PATCH 028/118] improve transfer testing --- .../block_processing/test_process_transfer.py | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 31b136cc2..10d2ccede 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -70,9 +70,17 @@ def test_success_withdrawable(state): @spec_state_test def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + + yield from run_transfer_processing(state, transfer) + + +@spec_state_test +def test_success_active_above_max_effective_fee(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) yield from run_transfer_processing(state, transfer) @@ -97,11 +105,10 @@ def test_incorrect_slot(state): @spec_state_test -def test_insufficient_balance(state): +def test_insufficient_balance_for_fee(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -110,7 +117,19 @@ def test_insufficient_balance(state): @spec_state_test -def test_no_dust(state): +def test_insufficient_balance(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(state, transfer, False) + + +@spec_state_test +def test_no_dust_sender(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) @@ -121,6 +140,19 @@ def test_no_dust(state): yield from run_transfer_processing(state, transfer, False) +@spec_state_test +def test_no_dust_recipient(state): + sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] + state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + state.balances[transfer.recipient] = 0 + + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(state, transfer, False) + + @spec_state_test def test_invalid_pubkey(state): transfer = get_valid_transfer(state) From 636a45a415073c650543aa353f0d297726c0be25 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 18:31:05 +0200 Subject: [PATCH 029/118] voluntary exit test case: invalid index --- .../test_process_voluntary_exit.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index 1efc950e0..6b0f79288 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -23,7 +23,6 @@ def run_voluntary_exit_processing(state, voluntary_exit, valid=True): If ``valid == False``, run expecting ``AssertionError`` """ validator_index = voluntary_exit.validator_index - pre_exit_epoch = state.validator_registry[validator_index].exit_epoch yield 'pre', state yield 'voluntary_exit', voluntary_exit @@ -33,6 +32,8 @@ def run_voluntary_exit_processing(state, voluntary_exit, valid=True): yield 'post', None return + pre_exit_epoch = state.validator_registry[validator_index].exit_epoch + process_voluntary_exit(state, voluntary_exit) yield 'post', state @@ -107,6 +108,26 @@ def test_success_exit_queue(state): ) +@spec_state_test +def test_validator_invalid_validator_index(state): + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[0] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + + voluntary_exit = build_voluntary_exit( + state, + current_epoch, + validator_index, + privkey, + ) + voluntary_exit.validator_index = len(state.validator_registry) + + yield from run_voluntary_exit_processing(state, voluntary_exit, False) + + @spec_state_test def test_validator_not_active(state): current_epoch = get_current_epoch(state) From 680017f69ce61e0970cd6e9a72c8ce58725f69d5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 18:33:41 +0200 Subject: [PATCH 030/118] voluntary exit test case: invalid, exit in future --- .../test_process_voluntary_exit.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index 6b0f79288..be0ef1e7a 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -108,6 +108,26 @@ def test_success_exit_queue(state): ) +@spec_state_test +def test_validator_exit_in_future(state): + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[0] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + + voluntary_exit = build_voluntary_exit( + state, + current_epoch, + validator_index, + privkey, + ) + voluntary_exit.epoch += 1 + + yield from run_voluntary_exit_processing(state, voluntary_exit, False) + + @spec_state_test def test_validator_invalid_validator_index(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit From 4f3f68bc12d6a67ea7f50ec2e92157c6a8c8a7b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 11 May 2019 21:31:37 +0200 Subject: [PATCH 031/118] signatures in tests, most block-ops updated, still many to go --- .../test_process_attestation.py | 39 +++- .../test_process_attester_slashing.py | 18 +- .../test_process_block_header.py | 9 +- .../block_processing/test_process_transfer.py | 8 +- test_libs/pyspec/eth2spec/test/context.py | 2 +- .../test_process_crosslinks.py | 14 +- test_libs/pyspec/eth2spec/test/helpers.py | 206 +++++++++++------- .../pyspec/eth2spec/test/test_finality.py | 11 +- test_libs/pyspec/eth2spec/utils/bls_stub.py | 7 +- 9 files changed, 206 insertions(+), 108 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 248b04ef4..98cb62ef7 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -3,17 +3,18 @@ from copy import deepcopy import eth2spec.phase0.spec as spec from eth2spec.phase0.state_transition import ( - state_transition, + state_transition_to, ) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) from eth2spec.test.helpers import ( - build_empty_block_for_next_slot, get_valid_attestation, + apply_empty_block, next_epoch, next_slot, + make_attestation_signature, ) from eth2spec.test.context import spec_state_test, expect_assertion_error @@ -65,9 +66,8 @@ def test_success(state): @spec_state_test def test_success_previous_epoch(state): attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) - block.slot = state.slot + spec.SLOTS_PER_EPOCH - state_transition(state, block) + next_epoch(state) + apply_empty_block(state) yield from run_attestation_processing(state, attestation) @@ -83,10 +83,9 @@ def test_before_inclusion_delay(state): @spec_state_test def test_after_epoch_slots(state): attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) # increment past latest inclusion slot - block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 - state_transition(state, block) + state_transition_to(state, state.slot + spec.SLOTS_PER_EPOCH + 1) + apply_empty_block(state) yield from run_attestation_processing(state, attestation, False) @@ -105,6 +104,9 @@ def test_old_source_epoch(state): # Now go beyond that, it will be invalid attestation.data.source_epoch -= 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -115,6 +117,9 @@ def test_wrong_shard(state): attestation.data.shard += 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -125,6 +130,9 @@ def test_new_source_epoch(state): attestation.data.source_epoch += 1 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -135,6 +143,9 @@ def test_source_root_is_target_root(state): attestation.data.source_root = attestation.data.target_root + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -159,6 +170,9 @@ def test_invalid_current_source_root(state): # Make attestation source root invalid: should be previous justified, not current one attestation.data.source_root = state.current_justified_root + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -169,6 +183,9 @@ def test_bad_source_root(state): attestation.data.source_root = b'\x42' * 32 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @@ -179,15 +196,21 @@ def test_non_zero_crosslink_data_root(state): attestation.data.crosslink_data_root = b'\x42' * 32 + # Re do signature + make_attestation_signature(state, attestation) + yield from run_attestation_processing(state, attestation, False) @spec_state_test def test_bad_previous_crosslink(state): next_epoch(state) + apply_empty_block(state) + attestation = get_valid_attestation(state) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) + apply_empty_block(state) state.current_crosslinks[attestation.data.shard].epoch += 10 diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 957b9a9f0..a0334c892 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -3,14 +3,15 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( get_balance, get_valid_attester_slashing, next_epoch, + apply_empty_block, + make_indexed_attestation_signature ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_attester_slashing_processing(state, attester_slashing, valid=True): """ @@ -47,14 +48,14 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): # lost whistleblower reward assert ( - get_balance(state, slashed_index) < - pre_slashed_balance + get_balance(state, slashed_index) < + pre_slashed_balance ) # gained whistleblower reward assert ( - get_balance(state, proposer_index) > - pre_proposer_balance + get_balance(state, proposer_index) > + pre_proposer_balance ) yield 'post', state @@ -70,6 +71,8 @@ def test_success_double(state): @spec_state_test def test_success_surround(state): next_epoch(state) + apply_empty_block(state) + state.current_justified_epoch += 1 attester_slashing = get_valid_attester_slashing(state) @@ -77,6 +80,9 @@ def test_success_surround(state): attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 + # correct the signature of attestation 1 + make_indexed_attestation_signature(state, attester_slashing.attestation_1) + yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index a176f7958..f2695ad09 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -6,13 +6,13 @@ from eth2spec.phase0.spec import ( advance_slot, process_block_header, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( build_empty_block_for_next_slot, next_slot, + make_block_signature ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def prepare_state_for_header_processing(state): cache_state(state) @@ -43,7 +43,7 @@ def run_block_header_processing(state, block, valid=True): @spec_state_test def test_success(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block) @@ -59,6 +59,7 @@ def test_invalid_slot(state): def test_invalid_previous_block_root(state): block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root + make_block_signature(state, block) yield from run_block_header_processing(state, block, valid=False) @@ -73,6 +74,6 @@ def test_proposer_slashed(state): # set proposer to slashed state.validator_registry[proposer_index].slashed = True - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 10d2ccede..64539d56e 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -1,18 +1,17 @@ import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, process_transfer, ) +from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers import ( get_valid_transfer, next_epoch, + apply_empty_block ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_transfer_processing(state, transfer, valid=True): """ @@ -58,6 +57,7 @@ def test_success_non_activated(state): @spec_state_test def test_success_withdrawable(state): next_epoch(state) + apply_empty_block(state) transfer = get_valid_transfer(state) @@ -97,7 +97,7 @@ def test_active_but_transfer_past_effective_balance(state): @spec_state_test def test_incorrect_slot(state): - transfer = get_valid_transfer(state, slot=state.slot+1) + transfer = get_valid_transfer(state, slot=state.slot + 1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index afabd4a57..99c58ce3d 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -5,7 +5,7 @@ from .helpers import create_genesis_state from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())]) +with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 203978d29..d4e6dc938 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -1,15 +1,15 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, ) +from eth2spec.phase0.state_transition import ( + state_transition, +) +from eth2spec.test.context import spec_state_test from eth2spec.test.helpers import ( add_attestation_to_state, build_empty_block_for_next_slot, @@ -18,8 +18,8 @@ from eth2spec.test.helpers import ( get_valid_attestation, next_epoch, next_slot, + apply_empty_block ) -from eth2spec.test.context import spec_state_test def run_process_crosslinks(state, valid=True): @@ -115,6 +115,8 @@ def test_double_late_crosslink(state): if attestation_2.data.shard == attestation_1.data.shard: break next_slot(state) + apply_empty_block(state) + fill_aggregate_attestation(state, attestation_2) # add attestation_2 in the next epoch after attestation_1 has @@ -126,7 +128,7 @@ def test_double_late_crosslink(state): assert len(state.current_epoch_attestations) == 0 crosslink_deltas = get_crosslink_deltas(state) - + yield from run_process_crosslinks(state) shard = attestation_2.data.shard diff --git a/test_libs/pyspec/eth2spec/test/helpers.py b/test_libs/pyspec/eth2spec/test/helpers.py index 8eb6d8891..f6cedd479 100644 --- a/test_libs/pyspec/eth2spec/test/helpers.py +++ b/test_libs/pyspec/eth2spec/test/helpers.py @@ -2,20 +2,20 @@ from copy import deepcopy from py_ecc import bls -from eth2spec.phase0.state_transition import ( - state_transition, -) +from typing import List + import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, # SSZ Attestation, + IndexedAttestation, AttestationData, AttestationDataAndCustodyBit, AttesterSlashing, BeaconBlock, + BeaconState, BeaconBlockHeader, Deposit, DepositData, @@ -33,7 +33,6 @@ from eth2spec.phase0.spec import ( get_current_epoch, get_domain, get_epoch_start_slot, - get_genesis_beacon_state, get_previous_epoch, get_shard_delta, hash_tree_root, @@ -41,12 +40,15 @@ from eth2spec.phase0.spec import ( verify_merkle_branch, hash, ) +from eth2spec.phase0.state_transition import ( + state_transition, state_transition_to +) from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, ) - +from eth2spec.utils.minimal_ssz import signing_root privkeys = [i + 1 for i in range(1024)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] @@ -64,72 +66,109 @@ def set_bitfield_bit(bitfield, i): byte_index = i // 8 bit_index = i % 8 return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index+1:] + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] ) -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): - if not deposit_data_leaves: - deposit_data_leaves = [] - signature = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys[i] - deposit_data = DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - amount=spec.MAX_EFFECTIVE_BALANCE, - signature=signature, - ) - item = deposit_data.hash_tree_root() - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - data=deposit_data_list[i] - )) - return genesis_validator_deposits, root - - -def create_genesis_state(num_validators, deposit_data_leaves=None): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits( - num_validators, - deposit_data_leaves, +def build_mock_validator(i: int, balance: int): + pubkey = pubkeys[i] + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + return spec.Validator( + pubkey=pubkeys[i], + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, + activation_epoch=spec.FAR_FUTURE_EPOCH, + exit_epoch=spec.FAR_FUTURE_EPOCH, + withdrawable_epoch=spec.FAR_FUTURE_EPOCH, + effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) ) - return get_genesis_beacon_state( - initial_deposits, + + +def create_genesis_state(num_validators): + deposit_root = b'\x42' * 32 + + state = spec.BeaconState( genesis_time=0, - genesis_eth1_data=Eth1Data( + deposit_index=num_validators, + latest_eth1_data=Eth1Data( deposit_root=deposit_root, - deposit_count=len(initial_deposits), + deposit_count=num_validators, block_hash=spec.ZERO_HASH, - ), + )) + + # We "hack" in the initial validators, + # as it is much faster than creating and processing genesis deposits for every single test case. + state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators + state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] + + # Process genesis activations + for validator in state.validator_registry: + if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = spec.GENESIS_EPOCH + validator.activation_epoch = spec.GENESIS_EPOCH + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) + for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + + return state + + +def make_block_signature(state, block): + assert block.slot == state.slot or block.slot == state.slot + 1 + if block.slot == state.slot: + proposer_index = spec.get_beacon_proposer_index(state) + else: + # use stub state to get proposer index of next slot + stub_state = deepcopy(state) + next_slot(stub_state) + proposer_index = spec.get_beacon_proposer_index(stub_state) + + privkey = privkeys[proposer_index] + + block.body.randao_reveal = bls.sign( + privkey=privkey, + message_hash=hash_tree_root(slot_to_epoch(block.slot)), + domain=get_domain( + state, + message_epoch=slot_to_epoch(block.slot), + domain_type=spec.DOMAIN_RANDAO, + ) ) + block.signature = bls.sign( + message_hash=signing_root(block), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_BEACON_PROPOSER, + slot_to_epoch(block.slot))) + return block -def build_empty_block_for_next_slot(state): +def build_empty_block(state, slot=None, signed=False): + if slot is None: + slot = state.slot empty_block = BeaconBlock() - empty_block.slot = state.slot + 1 + empty_block.slot = slot empty_block.body.eth1_data.deposit_count = state.deposit_index previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() empty_block.previous_block_root = signing_root(previous_block_header) + + if signed: + make_block_signature(state, empty_block) + return empty_block +def build_empty_block_for_next_slot(state, signed=False): + return build_empty_block(state, state.slot + 1, signed=signed) + + def build_deposit_data(state, pubkey, privkey, amount): deposit_data = DepositData( pubkey=pubkey, @@ -172,7 +211,8 @@ def build_attestation_data(state, slot, shard): justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root - crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks + crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( + state) else state.previous_crosslinks return AttestationData( shard=shard, beacon_block_root=block_root, @@ -270,6 +310,8 @@ def get_valid_attester_slashing(state): attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 + make_attestation_signature(state, attestation_2) + return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), attestation_2=convert_to_indexed(state, attestation_2), @@ -299,26 +341,38 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - participants = get_attesting_indices( - state, - attestation.data, - attestation.aggregation_bitfield, - ) - assert len(participants) == 2 + make_attestation_signature(state, attestation) + return attestation + +def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): signatures = [] for validator_index in participants: privkey = privkeys[validator_index] signatures.append( get_attestation_signature( state, - attestation.data, + data, privkey ) ) - attestation.aggregation_signature = bls.aggregate_signatures(signatures) - return attestation + return bls.aggregate_signatures(signatures) + + +def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): + participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + + +def make_attestation_signature(state, attestation: Attestation): + participants = get_attesting_indices( + state, + attestation.data, + attestation.aggregation_bitfield, + ) + + attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): @@ -357,7 +411,7 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non # ensure withdrawal_credentials reproducable state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] ) return transfer @@ -390,26 +444,32 @@ def add_attestation_to_state(state, attestation, slot): block = build_empty_block_for_next_slot(state) block.slot = slot block.body.attestations.append(attestation) + state_transition_to(state, block.slot) + make_block_signature(state, block) state_transition(state, block) def next_slot(state): """ - Transition to the next slot via an empty block. - Return the empty block that triggered the transition. + Transition to the next slot. """ - block = build_empty_block_for_next_slot(state) - state_transition(state, block) - return block + state_transition_to(state, state.slot + 1) def next_epoch(state): """ - Transition to the start slot of the next epoch via an empty block. - Return the empty block that triggered the transition. + Transition to the start slot of the next epoch """ - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition_to(state, slot) + + +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state) state_transition(state, block) return block diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 16bf24a4e..75191e59a 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,10 +1,10 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - from eth2spec.phase0.state_transition import ( state_transition, ) +from .context import spec_state_test from .helpers import ( build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -12,10 +12,9 @@ from .helpers import ( get_epoch_start_slot, get_valid_attestation, next_epoch, + apply_empty_block ) -from .context import spec_state_test - def check_finality(state, prev_state, @@ -101,7 +100,9 @@ def test_finality_rule_4(state): def test_finality_rule_1(state): # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state @@ -128,7 +129,9 @@ def test_finality_rule_1(state): def test_finality_rule_2(state): # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state @@ -161,7 +164,9 @@ def test_finality_rule_3(state): """ # get past first two epochs that finality does not run on next_epoch(state) + apply_empty_block(state) next_epoch(state) + apply_empty_block(state) yield 'pre', state diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py index 108c4ef71..0a52d2f65 100644 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ b/test_libs/pyspec/eth2spec/utils/bls_stub.py @@ -1,12 +1,13 @@ +from py_ecc import bls def bls_verify(pubkey, message_hash, signature, domain): - return True + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return True + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 + return bls.aggregate_pubkeys(pubkeys) From 48ed25b2bdb1807b4170fbde4414ffd9baa08609 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:07:15 +1000 Subject: [PATCH 032/118] Fleshed out a lot more of the API, nearly ready. - Added all the fields from BeaconBlock(Body) - Tagged all paths as 'Minimum for validator' - Removed BeaconNode and ValidatorClient conventions - Moved the basic non-object schema components to the top - Broke out common beacon block properties into the BeaconBlockCommon object - Fixed links to Eth2.0 spec --- specs/validator/beacon_node_oapi.yaml | 371 ++++++++++++++++++++++---- 1 file changed, 315 insertions(+), 56 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 0a583bdd6..a2e4559f2 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -1,27 +1,31 @@ openapi: "3.0.2" info: - title: "Beacon Node API for Validator" - description: "A beacon node API for enabling a validator to perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + title: "Minimal Beacon Node API for Validator" + description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" paths: /node/version: get: + tags: + - Minimum for validator summary: "Get version string of the running beacon node." - description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." + description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: 200: description: Request successful content: application/json: schema: - $ref: '#/components/schemas/client_version' + $ref: '#/components/schemas/version' 500: - $ref: '#/components/responses/InternalError' - + $ref: '#/components/responses/InternalError' + /node/genesis_time: get: + tags: + - Minimum for validator summary: "Get the genesis_time parameter from beacon node configuration." - description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." + description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: 200: description: Request successful @@ -31,10 +35,12 @@ paths: $ref: '#/components/schemas/genesis_time' 500: $ref: '#/components/responses/InternalError' - + /node/syncing: get: + tags: + - Minimum for validator summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -52,13 +58,15 @@ paths: $ref: '#/components/schemas/SyncingStatus' 500: $ref: '#/components/responses/InternalError' - - + + /validator/duties: get: + tags: + - Minimum for validator summary: "Get validator duties for the requested validators." - description: "Requests the BeaconNode to provide a set of _duties_, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected BeaconNode is properly synchronised." - parameters: + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." + parameters: - name: validator_pubkeys in: query required: true @@ -83,14 +91,16 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - - - + + + /validator/block: get: + tags: + - Minimum for validator summary: "Produce a new block, without signature." - description: "Requests a BeaconNode to produce a valid block, which can then be signed by a ValidatorClient." - parameters: + description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." + parameters: - name: slot in: query required: true @@ -101,7 +111,7 @@ paths: - name: randao_reveal in: query required: true - description: "The ValidatorClient's randao reveal value." + description: "The validator's randao reveal value." schema: type: string format: byte @@ -119,13 +129,15 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' post: + tags: + - Minimum for validator summary: "Publish a signed block" - description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." - parameters: + description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + parameters: - name: beacon_block in: query required: true - description: "The BeaconBlock object, as sent from the BeaconNode originally, but now with the signature field completed." + description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed." schema: $ref: '#/components/schemas/BeaconBlock' responses: @@ -137,13 +149,15 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - - + + /validator/attestation: get: + tags: + - Minimum for validator summary: "Produce an attestation, without signature." - description: "Requests that the BeaconNode produce an IndexedAttestation, with a blank signature field, which the ValidatorClient will then sign." - parameters: + description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." + parameters: - name: slot in: query required: true @@ -169,15 +183,17 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - + post: + tags: + - Minimum for validator summary: "Published a signed attestation." - description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: - name: attestation in: query required: true - description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." + description: "An `IndexedAttestation` structure, as originally provided by the beacon node, but now with the signature field completed." schema: $ref: '#/components/schemas/IndexedAttestation' responses: @@ -192,6 +208,30 @@ paths: components: schemas: + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc" + + version: + type: string + description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." + example: "Lighthouse / v0.1.5 (Linux x86_64)" + + genesis_time: + type: integer + format: uint64 + description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + example: 1557716289 + + merkle_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A 32 byte merkle root, used in all kinds of scenarios." + ValidatorDuty: type: object properties: @@ -214,6 +254,7 @@ components: format: uint64 nullable: true description: "The slot in which a validator must propose a block, or `null` if block production is not required." + SyncingStatus: type: object nullable: true @@ -230,12 +271,248 @@ components: type: integer format: uint64 description: "The estimated highest block, or current target block number." + + BeaconBlock: + allOf: + - $ref: '#/components/schemas/BeaconBlockCommon' + - type: object + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." + properties: + body: + $ref: '#/components/schemas/BeaconBlockBody' + + + BeaconBlockHeader: + allOf: + - $ref: '#/components/schemas/BeaconBlockCommon' + - type: object + properties: + body_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" + + + + + BeaconBlockBody: type: object - description: "The [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) from the Eth2.0 spec." + description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." + properties: + randao_reveal: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "The RanDAO reveal value provided by the validator." + eth1_data: + title: Eth1Data + type: object + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + properties: + deposit_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the deposit tree." + deposit_count: + type: integer + format: uint64 + description: "Total number of deposits." + block_hash: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Ethereum 1.x block hash" + graffiti: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + proposer_slashings: + type: array + items: + title: ProposerSlashings + type: object + description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." + properties: + proposer_index: + type: integer + format: uint64 + description: "The index of the proposer to be slashed." + header_1: + $ref: '#/components/schemas/BeaconBlockHeader' + header_2: + $ref: '#/components/schemas/BeaconBlockHeader' + attester_slashings: + type: array + items: + title: AttesterSlashings + type: object + description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." + properties: + attestation_1: + $ref: '#/components/schemas/IndexedAttestation' + attestation_2: + $ref: '#/components/schemas/IndexedAttestation' + attestations: + type: array + items: + title: Attestation + type: object + description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." + properties: + aggregation_bitfield: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]+$" + description: "Attester aggregation bitfield." + custody_bitfield: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]+$" + description: "Custody bitfield" + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "BLS aggregate signature." + data: + $ref: '#/components/schemas/AttestationData' + deposits: + type: array + items: + title: Deposit + type: object + description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." + properties: + proof: + type: array + description: "Branch in the deposit tree." + items: + oneOf: + - type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + - type: integer + format: uint64 + enum: [32] + example: 32 + description: "The DEPOSIT_CONTRACT_TREE_DEPTH value." + minItems: 2 + maxItems: 2 + index: + type: integer + format: uint64 + description: "Index in the deposit tree." + data: + title: DepositData + type: object + description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." + properties: + pubkey: + $ref: '#/components/schemas/pubkey' + withdrawal_credentials: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The withdrawal credentials." + amount: + type: integer + format: uint64 + description: "Amount in Gwei." + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Container self-signature." + + voluntary_exits: + type: array + items: + title: VoluntaryExit + type: object + description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." + properties: + epoch: + type: integer + format: uint64 + description: "Minimum epoch for processing exit." + validator_index: + type: integer + format: uint64 + description: "Index of the exiting validator." + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Validator signature." + transfers: + type: array + items: + title: Transfer + type: object + description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." + properties: + sender: + type: integer + format: uint64 + description: "Sender index." + recipient: + type: integer + format: uint64 + description: "Recipient index." + amount: + type: integer + format: uint64 + description: "Amount in Gwei" + fee: + type: integer + format: uint64 + description: "Fee in Gwei for block producer" + slot: + type: integer + format: uint64 + description: "Inclusion slot" + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "Sender withdrawal public key" + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Sender signature" + + BeaconBlockCommon: + # An object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects + type: object + properties: + slot: + type: integer + format: uint64 + description: "The slot to which this block corresponds." + parent_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The signing merkle root of the parent `BeaconBlock`" + state_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`" + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `BeaconBlock` made by the validator of the block" + Fork: type: object - description: "The [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) from the Eth2.0 spec." + description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." properties: previous_version: type: string @@ -251,9 +528,10 @@ components: type: integer format: uint64 description: "Fork epoch number" + IndexedAttestation: type: object - description: "The [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) from the Eth2.0 spec." + description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: custody_bit_0_indicies: type: array @@ -269,10 +547,10 @@ components: format: uint64 data: $ref: '#/components/schemas/AttestationData' - - AttestationData: + + AttestationData: type: object - description: "The [AttestationData](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) from the Eth2.0 spec." + description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: $ref: '#/components/schemas/merkle_root' @@ -290,9 +568,10 @@ components: $ref: '#/components/schemas/merkle_root' crosslink: $ref: '#/components/schemas/CrossLink' + CrossLink: type: object - description: "The [Crosslink](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) from the Eth2.0 spec." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." properties: shard: type: integer @@ -306,27 +585,7 @@ components: $ref: '#/components/schemas/merkle_root' data_root: $ref: '#/components/schemas/merkle_root' - - pubkey: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{96}$" - description: "The validator's BLS public key, uniquely identifying them." - client_version: - type: string - description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." - example: "Lighthouse / v0.1.5 (Linux x86_64)" - genesis_time: - type: integer - format: uint64 - description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." - merkle_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "A 32 byte merkle root, used in all kinds of scenarios." - - + responses: Success: description: Request successful From d10baf1dcea19b29fc9e755dc322bb149a7474d1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:18:56 +1000 Subject: [PATCH 033/118] Added optional path, , renamed tags, and fixed up whitespace issues. --- specs/validator/beacon_node_oapi.yaml | 51 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index a2e4559f2..01eef87a4 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -7,7 +7,7 @@ paths: /node/version: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get version string of the running beacon node." description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: @@ -23,7 +23,7 @@ paths: /node/genesis_time: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get the genesis_time parameter from beacon node configuration." description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: @@ -36,11 +36,10 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/syncing: get: tags: - - Minimum for validator + - Necessary for validator summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -59,11 +58,10 @@ paths: 500: $ref: '#/components/responses/InternalError' - /validator/duties: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get validator duties for the requested validators." description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." parameters: @@ -92,12 +90,10 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - - /validator/block: get: tags: - - Minimum for validator + - Necessary for validator summary: "Produce a new block, without signature." description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." parameters: @@ -130,7 +126,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Minimum for validator + - Necessary for validator summary: "Publish a signed block" description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: @@ -150,11 +146,10 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - /validator/attestation: get: tags: - - Minimum for validator + - Necessary for validator summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: @@ -183,10 +178,9 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - post: tags: - - Minimum for validator + - Necessary for validator summary: "Published a signed attestation." description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: @@ -206,6 +200,29 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' + /node/fork: + get: + tags: + - Optional + summary: "Get fork information from running beacon node." + description: "Requests the beacon node to provide which fork version it is currently on." + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: object + properties: + fork: + $ref: '#/components/schemas/Fork' + chain_id: + type: integer + format: uint64 + description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + 500: + $ref: '#/components/responses/InternalError' + components: schemas: pubkey: @@ -272,7 +289,6 @@ components: format: uint64 description: "The estimated highest block, or current target block number." - BeaconBlock: allOf: - $ref: '#/components/schemas/BeaconBlockCommon' @@ -282,7 +298,6 @@ components: body: $ref: '#/components/schemas/BeaconBlockBody' - BeaconBlockHeader: allOf: - $ref: '#/components/schemas/BeaconBlockCommon' @@ -294,9 +309,6 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" - - - BeaconBlockBody: type: object description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." @@ -426,7 +438,6 @@ components: format: byte pattern: "^0x[a-fA-F0-9]{192}$" description: "Container self-signature." - voluntary_exits: type: array items: From 2035aea0b8e5123536985bf76b176cb33232a6a9 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:52:34 +1000 Subject: [PATCH 034/118] Formatting clean up. - Moved /node/fork up with other node endpoints - Added descriptions and ordering to tags - Removed common merkle_root schema, to be more specific in descriptions. - Moved BeaconBlockCommon next to appropriate schemas. - Lots of small grammar improvements, full stops at end of descriptions. --- specs/validator/beacon_node_oapi.yaml | 206 +++++++++++++------------- 1 file changed, 104 insertions(+), 102 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 01eef87a4..8d7267953 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -3,6 +3,11 @@ info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" +tags: + - name: Necessary for validator + description: The minimal set of endpoints to enable a working validator implementation. + - name: Optional + description: Extra endpoints which are nice-to-haves. paths: /node/version: get: @@ -19,7 +24,6 @@ paths: $ref: '#/components/schemas/version' 500: $ref: '#/components/responses/InternalError' - /node/genesis_time: get: tags: @@ -57,6 +61,28 @@ paths: $ref: '#/components/schemas/SyncingStatus' 500: $ref: '#/components/responses/InternalError' + /node/fork: + get: + tags: + - Optional + summary: "Get fork information from running beacon node." + description: "Requests the beacon node to provide which fork version it is currently on." + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: object + properties: + fork: + $ref: '#/components/schemas/Fork' + chain_id: + type: integer + format: uint64 + description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + 500: + $ref: '#/components/responses/InternalError' /validator/duties: get: @@ -127,7 +153,7 @@ paths: post: tags: - Necessary for validator - summary: "Publish a signed block" + summary: "Publish a signed block." description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: - name: beacon_block @@ -200,29 +226,6 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - /node/fork: - get: - tags: - - Optional - summary: "Get fork information from running beacon node." - description: "Requests the beacon node to provide which fork version it is currently on." - responses: - 200: - description: Request successful - content: - application/json: - schema: - type: object - properties: - fork: - $ref: '#/components/schemas/Fork' - chain_id: - type: integer - format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." - 500: - $ref: '#/components/responses/InternalError' - components: schemas: pubkey: @@ -231,24 +234,15 @@ components: pattern: "^0x[a-fA-F0-9]{96}$" description: "The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._" example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc" - version: type: string description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." example: "Lighthouse / v0.1.5 (Linux x86_64)" - genesis_time: type: integer format: uint64 description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." example: 1557716289 - - merkle_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "A 32 byte merkle root, used in all kinds of scenarios." - ValidatorDuty: type: object properties: @@ -271,7 +265,6 @@ components: format: uint64 nullable: true description: "The slot in which a validator must propose a block, or `null` if block production is not required." - SyncingStatus: type: object nullable: true @@ -290,15 +283,15 @@ components: description: "The estimated highest block, or current target block number." BeaconBlock: + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object - description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." properties: body: $ref: '#/components/schemas/BeaconBlockBody' - BeaconBlockHeader: + description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -308,7 +301,30 @@ components: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" - + BeaconBlockCommon: + # An abstract object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects + type: object + properties: + slot: + type: integer + format: uint64 + description: "The slot to which this block corresponds." + parent_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The signing merkle root of the parent `BeaconBlock`." + state_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`." + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `BeaconBlock` made by the validator of the block." BeaconBlockBody: type: object description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." @@ -336,7 +352,7 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]{64}$" - description: "Ethereum 1.x block hash" + description: "Ethereum 1.x block hash." graffiti: type: string format: byte @@ -383,7 +399,7 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield" + description: "Custody bitfield." signature: type: string format: byte @@ -476,50 +492,25 @@ components: amount: type: integer format: uint64 - description: "Amount in Gwei" + description: "Amount in Gwei." fee: type: integer format: uint64 - description: "Fee in Gwei for block producer" + description: "Fee in Gwei for block producer." slot: type: integer format: uint64 - description: "Inclusion slot" + description: "Inclusion slot." pubkey: type: string format: byte pattern: "^0x[a-fA-F0-9]{96}$" - description: "Sender withdrawal public key" + description: "Sender withdrawal public key." signature: type: string format: byte pattern: "^0x[a-fA-F0-9]{192}$" - description: "Sender signature" - - BeaconBlockCommon: - # An object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects - type: object - properties: - slot: - type: integer - format: uint64 - description: "The slot to which this block corresponds." - parent_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The signing merkle root of the parent `BeaconBlock`" - state_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`" - signature: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{192}$" - example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" - description: "The BLS signature of the `BeaconBlock` made by the validator of the block" + description: "Sender signature." Fork: type: object @@ -529,17 +520,16 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]{8}$" - description: "Previous fork version" + description: "Previous fork version." current_version: type: string format: byte pattern: "^0x[a-fA-F0-9]{8}$" - description: "Current fork version" + description: "Current fork version." epoch: type: integer format: uint64 - description: "Fork epoch number" - + description: "Fork epoch number." IndexedAttestation: type: object description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." @@ -558,51 +548,63 @@ components: format: uint64 data: $ref: '#/components/schemas/AttestationData' - AttestationData: type: object description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "LMD GHOST vote." source_epoch: type: integer format: uint64 - description: "Source epoch from FFG vote" + description: "Source epoch from FFG vote." source_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Source root from FFG vote." target_epoch: type: integer format: uint64 - description: "Target epoch from FFG vote" + description: "Target epoch from FFG vote." target_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Target root from FFG vote." crosslink: - $ref: '#/components/schemas/CrossLink' - - CrossLink: - type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." - properties: - shard: - type: integer - format: uint64 - description: "The shard number" - epoch: - type: integer - format: uint64 - description: "The epoch number" - parent_root: - $ref: '#/components/schemas/merkle_root' - data_root: - $ref: '#/components/schemas/merkle_root' + title: CrossLink + type: object + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." + properties: + shard: + type: integer + format: uint64 + description: "The shard number." + epoch: + type: integer + format: uint64 + description: "The epoch number." + parent_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the previous crosslink." + data_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the crosslinked shard data since the previous crosslink." responses: Success: - description: Request successful + description: "Request successful." InvalidRequest: - description: Invalid request syntax + description: "Invalid request syntax." InternalError: - description: Beacon node internal error + description: "Beacon node internal error." CurrentlySyncing: - description: Beacon node is currently syncing, try again later + description: "Beacon node is currently syncing, try again later." From 0b2c7acdb32a0daf81cf3f54e8ab35679f7083a7 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:25:22 +1000 Subject: [PATCH 035/118] Fixed up markdown. - Removed TOC - Removed all the old spec stuff - Uploaded spec to SwaggerHub and provided a link to it. - Added a 'license' section to the API description. --- .../validator/0_beacon-node-validator-api.md | 216 +----------------- specs/validator/beacon_node_oapi.yaml | 5 +- 2 files changed, 7 insertions(+), 214 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 342e7f934..cf763f778 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -2,12 +2,6 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. -## Table of Contents - - - - - ## Outline This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. @@ -28,211 +22,7 @@ This specification is derived from a proposal and discussion on Issues [#1011](h ## Specification -### Entities -The following are the two entities that participate in this protocol: - - **`BeaconNode`**: - A beacon node instance, run with a `--rpc` flag to enable the RPC interface. Runs stand-alone. +The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. - - **`ValidatorClient`**: -A validator client instance, which must connect to at least one instance of `BeaconNode`. - - -### Endpoints -This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. - -This proposal is a minimum set of messages necessary to enable effective communication, without any extra features. Anything extra is beyond the scope of this document. - -#### Summary Table -| Name | Type | Parameters | Returns | -| -------- | --- | ----- | ----- | -| [`get_client_version`](#get_client_version) | GET | N/A | `client_version` | -| [`get_genesis_time`](#get_genesis_time) | GET | N/A | `genesis_time` | -| [`get_syncing_status`](#get_syncing_status) | GET | N/A | `syncing_status` | -| [`get_duties`](#get_duties) | GET | `validator_pubkeys` | `syncing_status`, `current_version`, [`ValidatorDuty`]| -| [`produce_block`](#produce_block) | GET | `slot`, `randao_reveal` | `beacon_block` | -| [`publish_block`](#publish_block) | POST | `beacon_block` | N/A | -| [`produce_attestation`](#produce_attestation) | GET | `slot`, `shard` | `indexed_attestation` | -| [`publish_attestation`](#publish_attestation) | POST | `indexed_attestation` | N/A | Publishes the IndexedAttestation after having been signed by the ValidatorClient | - -#### Status Codes -For each of these endpoints the underlying transport protocol should provide status codes. Assuming this will be based on HTTP, one of the following standard status codes will always be included as part of a response: - -| Code | Meaning | -| --- | --- | -| `200` | The API call succeeded. | -| `40X` | The request was malformed. | -| `500` | The `BeaconNode` cannot complete the request due to an internal error. | -| `503` | The `BeaconNode` is currently syncing, try again later. _A call can be made to `get_syncing_status` to in order to find out how much has been synchronised._ | - -#### `get_client_version` -Requests that the `BeaconNode` identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field. - - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `client_version` | bytes32 | An ASCII-encoded hex string which uniquely defines the implementation of the `BeaconNode` and its current software version. | - - **Note**: _Unlike most other endpoints, `get_client_version` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ - - -#### `get_genesis_time` - Requests the `genesis_time` parameter from the `BeaconNode`, which should be consistent across all `BeaconNodes` that follow the same beacon chain. - - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `genesis_time` | uint64 | The [`genesis_time`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#on-genesis), which is a fairly static configuration option for the `BeaconNode`. | - - **Note**: _Unlike most other endpoints, `get_genesis_time` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ - - -#### `get_syncing_status` - Requests the `BeaconNode` to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC [`eth_syncing`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing) call. - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `syncing` | `false` OR `SyncingStatus` | Either `false` if the node is not syncing, or a [`SyncingStatus`](#SyncingStatus) object if it is. | - - **Note**: _Unlike most other endpoints, `get_syncing_status` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200` with the `SyncingStatus` object._ - - -#### `get_duties` - Requests the BeaconNode to provide a set of “duties”, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected `BeaconNode` is properly synchronised. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `validator_pubkeys` | [bytes48] | A list of unique validator public keys, where each item is a `0x` encoded hex string. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `current_version` | bytes4 | The `current_version`, as described by the current [`Fork`](#Fork). | - | `validator_duties` | [`ValidatorDuty`] | A list where each item is a custom [`ValidatorDuty`](#ValidatorDuty) object. | - - - #### `produce_block` - Requests a `BeaconNode` to produce a valid block, which can then be signed by a ValidatorClient. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `slot` | uint64 | The slot for which the block should be proposed. | - | `randao_reveal` | bytes | The ValidatorClient's randao reveal value. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `beacon_block` | `BeaconBlock` | A proposed [`BeaconBlock`](#BeaconBlock) object, but with the `signature` field left blank. - - - #### `publish_block` - Instructs the `BeaconNode` to publish a newly signed beacon block to the beacon network, to be included in the beacon chain. - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `beacon_block` | `BeaconBlock` | The [`BeaconBlock`](#BeaconBlock) object, as sent from the `BeaconNode` originally, but now with the `signature` field completed. - - - **Returns**: N/A - - - #### `produce_attestation` - Requests that the `BeaconNode` produce an `IndexedAttestation`, with a blank `signature` field, which the `ValidatorClient` will then sign. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `slot` | uint64 | The slot for which the attestation should be proposed. | - | `shard` | uint64 | The shard number for which the attestation is to be proposed. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure with the `signature` field left blank. | - - #### `publish_attestation` - Instructs the `BeaconNode` to publish a newly signed `IndexedAttestation` object, to be incorporated into the beacon chain. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure, as originally provided by the `BeaconNode`, but now with the `signature` field completed. | - - **Returns**: N/A - - - - ----- - -### Data Structures -Two new data objects are proposed for the sake of implementation, and several other data objects from the Eth2.0 specs are referenced. - -The `bytes` data types are encoded hex strings, with `0x` preceeding them. `uint64` are decimal encoded integers, and `None` may be `null`, which is distinct from `0`. - -#### `ValidatorDuty` -```asm -{ - - # The validator's public key, uniquely identifying them - 'validator_pubkey': 'bytes48', - # The index of the validator in the committee - 'committee_index': 'uint64', - # The slot at which the validator must attest. - 'attestation_slot': 'uint64', - # The shard in which the validator must attest - 'attestation_shard': 'uint64', - # The slot in which a validator must propose a block. This field can also be None. - 'block_production_slot': 'uint64' or None -} -``` - -#### `SyncingStatus` -As described by the [Eth1.0 JSON-RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing).: -```asm -{ - # The block at which syncing started (will only be reset, after the sync reached his head) - 'startingBlock': 'uint64', - # The current block - 'currentBlock': 'uint64', - # The estimated highest block, or current target block number - 'highestBlock': 'uint64' -} -``` - -#### `Fork` -As described by [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) in the Eth2.0 specs. - -#### `BeaconBlock` -As described by [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#BeaconBlock) in the Eth2.0 specs. - -#### `IndexedAttestation` -As described by [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#IndexedAttestation) in the Eth2.0 specs. - - - - -## Optional Extras - -#### Endpoint: `get_fork` - Requests the `BeaconNode` to provide which fork version it is currently on. - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `fork` | [`Fork`](#Fork) | Provides the current version information for the fork which the `BeaconNode` is currently following. | - | `chain_id` | uint64 | Sometimes called the network id, this number discerns the active chain for the `BeaconNode`. Analagous to Eth1.0 JSON-RPC [`net_version`](https://github.com/ethereum/wiki/wiki/JSON-RPC#net_version). | - +For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: +[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 8d7267953..0c2937f3d 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -1,8 +1,11 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" - description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" + license: + name: "Apache 2.0" + url: "https://www.apache.org/licenses/LICENSE-2.0.html" tags: - name: Necessary for validator description: The minimal set of endpoints to enable a working validator implementation. From ac7a40376ba787b55abfc96aa87d0741a6fdb437 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:26:45 +1000 Subject: [PATCH 036/118] Included a link to the API document in the root README.md file. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ad7204f21..ec8056956 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Core specifications for eth2.0 client validation can be found in [specs/core](sp * [General test format](specs/test_formats/README.md) * [Merkle proof formats](specs/light_client/merkle_proofs.md) * [Light client syncing protocol](specs/light_client/sync_protocol.md) +* [Beacon node API for validator](specs/validator/0_beacon-node-validator-api.md) ### Design goals From 67921ab96fdd1ed108fb5bc2f8d7d75871083a8b Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:30:53 +1000 Subject: [PATCH 037/118] Fixed up some small wording in the API readme. --- specs/validator/0_beacon-node-validator-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index cf763f778..5c597aa75 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -4,9 +4,9 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers ## Outline -This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. +This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. -The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. The only supported return data type is currently JSON. ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. From 904e2e9c0c348043a5ad028c3b151df382a14956 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 13 May 2019 23:15:02 +0200 Subject: [PATCH 038/118] BLS on/off deco --- scripts/phase0/build_spec.py | 2 +- test_libs/pyspec/eth2spec/test/context.py | 27 ++++++++++++++++++++- test_libs/pyspec/eth2spec/test/utils.py | 4 +++ test_libs/pyspec/eth2spec/utils/bls.py | 25 +++++++++++++++++++ test_libs/pyspec/eth2spec/utils/bls_stub.py | 12 --------- 5 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/bls.py delete mode 100644 test_libs/pyspec/eth2spec/utils/bls_stub.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index da5845951..26b0e5a8a 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,7 +13,7 @@ from typing import ( Tuple, ) from eth2spec.utils.minimal_ssz import * -from eth2spec.utils.bls_stub import * +from eth2spec.utils.bls import * """) for i in (1, 2, 3, 4, 8, 32, 48, 96): diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index afabd4a57..3ef62eae5 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -1,4 +1,5 @@ from eth2spec.phase0 import spec +from eth2spec.utils import bls from .helpers import create_genesis_state @@ -10,7 +11,7 @@ with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, l # shorthand for decorating @with_state @spectest() def spec_state_test(fn): - return with_state(spectest()(fn)) + return with_state(bls_switch(spectest()(fn))) def expect_assertion_error(fn): @@ -25,3 +26,27 @@ def expect_assertion_error(fn): pass if bad: raise AssertionError('expected an assertion error, but got none.') + + +def always_bls(fn): + """ + Decorator to apply on ``bls_switch`` decorator to force BLS activation. Useful to mark tests as BLS-dependent. + """ + def entry(*args, **kw): + # override bls setting + kw['bls_active'] = True + fn(*args, **kw) + return entry + + +def bls_switch(fn): + """ + Decorator to make a function execute with BLS ON, or BLS off. + Based on an optional bool argument ``bls_active``, passed to the function at runtime. + """ + def entry(*args, **kw): + old_state = bls.bls_active + bls.bls_active = kw.pop('bls_active', False) + fn(*args, **kw) + bls.bls_active = old_state + return entry diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index b19d4df59..1c157bcee 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,4 +1,5 @@ from eth2spec.debug.encode import encode +from eth2spec.utils import bls def spectest(description: str = None): @@ -18,6 +19,9 @@ def spectest(description: str = None): else: # description can be explicit out['description'] = description + # If BLS is not active, mark the test as BLS-ignorant. + if not bls.bls_active: + out['stub_bls'] = True # put all generated data into a dict. for data in fn(*args, **kw): # If there is a type argument, encode it as that type. diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py new file mode 100644 index 000000000..daeb361a9 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -0,0 +1,25 @@ +from py_ecc import bls + +# Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. +bls_active = True + + +def bls_verify(pubkey, message_hash, signature, domain): + if bls_active: + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) + else: + return True + + +def bls_verify_multiple(pubkeys, message_hashes, signature, domain): + if bls_active: + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) + else: + return True + + +def bls_aggregate_pubkeys(pubkeys): + if bls_active: + return bls.aggregate_pubkeys(pubkeys) + else: + return b'\x42' * 96 diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py deleted file mode 100644 index 108c4ef71..000000000 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ /dev/null @@ -1,12 +0,0 @@ - - -def bls_verify(pubkey, message_hash, signature, domain): - return True - - -def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return True - - -def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 From 51c82c5b81cb3c7e72023e3edada1d01c3f59b6e Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:36:32 +0200 Subject: [PATCH 039/118] clean up helpers, make helpers pkg --- test_libs/pyspec/eth2spec/test/helpers.py | 505 ------------------ .../pyspec/eth2spec/test/helpers/__init__.py | 0 .../eth2spec/test/helpers/attestations.py | 145 +++++ .../test/helpers/attester_slashings.py | 17 + .../pyspec/eth2spec/test/helpers/bitfields.py | 11 + .../pyspec/eth2spec/test/helpers/block.py | 62 +++ .../pyspec/eth2spec/test/helpers/deposits.py | 74 +++ .../pyspec/eth2spec/test/helpers/genesis.py | 51 ++ .../pyspec/eth2spec/test/helpers/keys.py | 5 + .../test/helpers/proposer_slashings.py | 48 ++ .../pyspec/eth2spec/test/helpers/state.py | 42 ++ .../pyspec/eth2spec/test/helpers/transfers.py | 50 ++ .../eth2spec/test/helpers/voluntary_exits.py | 24 + 13 files changed, 529 insertions(+), 505 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/__init__.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/attestations.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/block.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/deposits.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/genesis.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/keys.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/state.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/transfers.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py diff --git a/test_libs/pyspec/eth2spec/test/helpers.py b/test_libs/pyspec/eth2spec/test/helpers.py deleted file mode 100644 index f6cedd479..000000000 --- a/test_libs/pyspec/eth2spec/test/helpers.py +++ /dev/null @@ -1,505 +0,0 @@ -from copy import deepcopy - -from py_ecc import bls - -from typing import List - -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( - # constants - ZERO_HASH, - # SSZ - Attestation, - IndexedAttestation, - AttestationData, - AttestationDataAndCustodyBit, - AttesterSlashing, - BeaconBlock, - BeaconState, - BeaconBlockHeader, - Deposit, - DepositData, - Eth1Data, - ProposerSlashing, - Transfer, - VoluntaryExit, - # functions - convert_to_indexed, - get_active_validator_indices, - get_attesting_indices, - get_block_root, - get_block_root_at_slot, - get_crosslink_committee, - get_current_epoch, - get_domain, - get_epoch_start_slot, - get_previous_epoch, - get_shard_delta, - hash_tree_root, - slot_to_epoch, - verify_merkle_branch, - hash, -) -from eth2spec.phase0.state_transition import ( - state_transition, state_transition_to -) -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) -from eth2spec.utils.minimal_ssz import signing_root - -privkeys = [i + 1 for i in range(1024)] -pubkeys = [bls.privtopub(privkey) for privkey in privkeys] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} - - -def get_balance(state, index): - return state.balances[index] - - -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) - - -def build_mock_validator(i: int, balance: int): - pubkey = pubkeys[i] - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] - return spec.Validator( - pubkey=pubkeys[i], - withdrawal_credentials=withdrawal_credentials, - activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, - activation_epoch=spec.FAR_FUTURE_EPOCH, - exit_epoch=spec.FAR_FUTURE_EPOCH, - withdrawable_epoch=spec.FAR_FUTURE_EPOCH, - effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) - ) - - -def create_genesis_state(num_validators): - deposit_root = b'\x42' * 32 - - state = spec.BeaconState( - genesis_time=0, - deposit_index=num_validators, - latest_eth1_data=Eth1Data( - deposit_root=deposit_root, - deposit_count=num_validators, - block_hash=spec.ZERO_HASH, - )) - - # We "hack" in the initial validators, - # as it is much faster than creating and processing genesis deposits for every single test case. - state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators - state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] - - # Process genesis activations - for validator in state.validator_registry: - if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: - validator.activation_eligibility_epoch = spec.GENESIS_EPOCH - validator.activation_epoch = spec.GENESIS_EPOCH - - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) - for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root - - return state - - -def make_block_signature(state, block): - assert block.slot == state.slot or block.slot == state.slot + 1 - if block.slot == state.slot: - proposer_index = spec.get_beacon_proposer_index(state) - else: - # use stub state to get proposer index of next slot - stub_state = deepcopy(state) - next_slot(stub_state) - proposer_index = spec.get_beacon_proposer_index(stub_state) - - privkey = privkeys[proposer_index] - - block.body.randao_reveal = bls.sign( - privkey=privkey, - message_hash=hash_tree_root(slot_to_epoch(block.slot)), - domain=get_domain( - state, - message_epoch=slot_to_epoch(block.slot), - domain_type=spec.DOMAIN_RANDAO, - ) - ) - block.signature = bls.sign( - message_hash=signing_root(block), - privkey=privkey, - domain=get_domain( - state, - spec.DOMAIN_BEACON_PROPOSER, - slot_to_epoch(block.slot))) - return block - - -def build_empty_block(state, slot=None, signed=False): - if slot is None: - slot = state.slot - empty_block = BeaconBlock() - empty_block.slot = slot - empty_block.body.eth1_data.deposit_count = state.deposit_index - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = signing_root(previous_block_header) - - if signed: - make_block_signature(state, empty_block) - - return empty_block - - -def build_empty_block_for_next_slot(state, signed=False): - return build_empty_block(state, state.slot + 1, signed=signed) - - -def build_deposit_data(state, pubkey, privkey, amount): - deposit_data = DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], - amount=amount, - ) - signature = bls.sign( - message_hash=signing_root(deposit_data), - privkey=privkey, - domain=get_domain( - state, - spec.DOMAIN_DEPOSIT, - ) - ) - deposit_data.signature = signature - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - if slot == state.slot: - block_root = build_empty_block_for_next_slot(state).previous_block_root - else: - block_root = get_block_root_at_slot(state, slot) - - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - if slot < current_epoch_start_slot: - epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) - elif slot == current_epoch_start_slot: - epoch_boundary_root = block_root - else: - epoch_boundary_root = get_block_root(state, get_current_epoch(state)) - - if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root - else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root - - crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( - state) else state.previous_crosslinks - return AttestationData( - shard=shard, - beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=slot_to_epoch(slot), - target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink_root=hash_tree_root(crosslinks[shard]), - ) - - -def build_voluntary_exit(state, epoch, validator_index, privkey): - voluntary_exit = VoluntaryExit( - epoch=epoch, - validator_index=validator_index, - ) - voluntary_exit.signature = bls.sign( - message_hash=signing_root(voluntary_exit), - privkey=privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=epoch, - ) - ) - - return voluntary_exit - - -def build_deposit(state, - deposit_data_leaves, - pubkey, - privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) - - item = deposit_data.hash_tree_root() - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - - return deposit, root, deposit_data_leaves - - -def get_valid_proposer_slashing(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - slot = state.slot - - header_1 = BeaconBlockHeader( - slot=slot, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - block_body_root=ZERO_HASH, - ) - header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02' * 32 - header_2.slot = slot + 1 - - domain = get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_PROPOSER, - ) - header_1.signature = bls.sign( - message_hash=signing_root(header_1), - privkey=privkey, - domain=domain, - ) - header_2.signature = bls.sign( - message_hash=signing_root(header_2), - privkey=privkey, - domain=domain, - ) - - return ProposerSlashing( - proposer_index=validator_index, - header_1=header_1, - header_2=header_2, - ) - - -def get_valid_attester_slashing(state): - attestation_1 = get_valid_attestation(state) - attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 - - make_attestation_signature(state, attestation_2) - - return AttesterSlashing( - attestation_1=convert_to_indexed(state, attestation_1), - attestation_2=convert_to_indexed(state, attestation_2), - ) - - -def get_valid_attestation(state, slot=None): - if slot is None: - slot = state.slot - - if slot_to_epoch(slot) == get_current_epoch(state): - shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH - else: - previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) - shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT - - attestation_data = build_attestation_data(state, slot, shard) - - crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) - - committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) - custody_bitfield = b'\x00' * bitfield_length - attestation = Attestation( - aggregation_bitfield=aggregation_bitfield, - data=attestation_data, - custody_bitfield=custody_bitfield, - ) - make_attestation_signature(state, attestation) - return attestation - - -def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): - signatures = [] - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - get_attestation_signature( - state, - data, - privkey - ) - ) - - return bls.aggregate_signatures(signatures) - - -def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): - participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices - indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) - - -def make_attestation_signature(state, attestation: Attestation): - participants = get_attesting_indices( - state, - attestation.data, - attestation.aggregation_bitfield, - ) - - attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) - - -def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): - if slot is None: - slot = state.slot - current_epoch = get_current_epoch(state) - if sender_index is None: - sender_index = get_active_validator_indices(state, current_epoch)[-1] - recipient_index = get_active_validator_indices(state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - - if fee is None: - fee = get_balance(state, sender_index) // 32 - if amount is None: - amount = get_balance(state, sender_index) - fee - - transfer = Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=fee, - slot=slot, - pubkey=transfer_pubkey, - signature=ZERO_HASH, - ) - transfer.signature = bls.sign( - message_hash=signing_root(transfer), - privkey=transfer_privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_TRANSFER, - message_epoch=get_current_epoch(state), - ) - ) - - # ensure withdrawal_credentials reproducable - state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] - ) - - return transfer - - -def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): - message_hash = AttestationDataAndCustodyBit( - data=attestation_data, - custody_bit=custody_bit, - ).hash_tree_root() - - return bls.sign( - message_hash=message_hash, - privkey=privkey, - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, - ) - ) - - -def fill_aggregate_attestation(state, attestation): - crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard) - for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) - - -def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state) - block.slot = slot - block.body.attestations.append(attestation) - state_transition_to(state, block.slot) - make_block_signature(state, block) - state_transition(state, block) - - -def next_slot(state): - """ - Transition to the next slot. - """ - state_transition_to(state, state.slot + 1) - - -def next_epoch(state): - """ - Transition to the start slot of the next epoch - """ - slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) - state_transition_to(state, slot) - - -def apply_empty_block(state): - """ - Transition via an empty block (on current slot, assuming no block has been applied yet). - :return: the empty block that triggered the transition. - """ - block = build_empty_block(state) - state_transition(state, block) - return block - - -def get_state_root(state, slot) -> bytes: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] - - -def prepare_state_and_deposit(state, validator_index, amount): - """ - Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. - """ - pre_validator_count = len(state.validator_registry) - # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * pre_validator_count - - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit, root, deposit_data_leaves = build_deposit( - state, - deposit_data_leaves, - pubkey, - privkey, - amount, - ) - - state.latest_eth1_data.deposit_root = root - state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - return deposit diff --git a/test_libs/pyspec/eth2spec/test/helpers/__init__.py b/test_libs/pyspec/eth2spec/test/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py new file mode 100644 index 000000000..d7fab4c17 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -0,0 +1,145 @@ +from typing import List + +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import ( + Attestation, + AttestationData, + AttestationDataAndCustodyBit, + get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, + get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) + +from eth2spec.phase0.state_transition import ( + state_transition, state_transition_to +) +from eth2spec.test.helpers.bitfields import set_bitfield_bit +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, make_block_signature +from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures + + +def build_attestation_data(state, slot, shard): + assert state.slot >= slot + + if slot == state.slot: + block_root = build_empty_block_for_next_slot(state).previous_block_root + else: + block_root = get_block_root_at_slot(state, slot) + + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + if slot < current_epoch_start_slot: + epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) + elif slot == current_epoch_start_slot: + epoch_boundary_root = block_root + else: + epoch_boundary_root = get_block_root(state, get_current_epoch(state)) + + if slot < current_epoch_start_slot: + justified_epoch = state.previous_justified_epoch + justified_block_root = state.previous_justified_root + else: + justified_epoch = state.current_justified_epoch + justified_block_root = state.current_justified_root + + crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch( + state) else state.previous_crosslinks + return AttestationData( + shard=shard, + beacon_block_root=block_root, + source_epoch=justified_epoch, + source_root=justified_block_root, + target_epoch=slot_to_epoch(slot), + target_root=epoch_boundary_root, + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink_root=hash_tree_root(crosslinks[shard]), + ) + + +def get_valid_attestation(state, slot=None): + if slot is None: + slot = state.slot + + if slot_to_epoch(slot) == get_current_epoch(state): + shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH + else: + previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) + shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT + + attestation_data = build_attestation_data(state, slot, shard) + + crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) + + committee_size = len(crosslink_committee) + bitfield_length = (committee_size + 7) // 8 + aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) + custody_bitfield = b'\x00' * bitfield_length + attestation = Attestation( + aggregation_bitfield=aggregation_bitfield, + data=attestation_data, + custody_bitfield=custody_bitfield, + ) + make_attestation_signature(state, attestation) + return attestation + + +def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): + signatures = [] + for validator_index in participants: + privkey = privkeys[validator_index] + signatures.append( + get_attestation_signature( + state, + data, + privkey + ) + ) + + return bls_aggregate_signatures(signatures) + + +def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): + participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + + +def make_attestation_signature(state, attestation: Attestation): + participants = get_attesting_indices( + state, + attestation.data, + attestation.aggregation_bitfield, + ) + + attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) + + +def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): + message_hash = AttestationDataAndCustodyBit( + data=attestation_data, + custody_bit=custody_bit, + ).hash_tree_root() + + return bls_sign( + message_hash=message_hash, + privkey=privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_ATTESTATION, + message_epoch=attestation_data.target_epoch, + ) + ) + + +def fill_aggregate_attestation(state, attestation): + crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard) + for i in range(len(crosslink_committee)): + attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + + +def add_attestation_to_state(state, attestation, slot): + block = build_empty_block_for_next_slot(state) + block.slot = slot + block.body.attestations.append(attestation) + state_transition_to(state, block.slot) + make_block_signature(state, block) + state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py new file mode 100644 index 000000000..2895c7771 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -0,0 +1,17 @@ +from copy import deepcopy + +from eth2spec.phase0.spec import AttesterSlashing, convert_to_indexed +from eth2spec.test.helpers.attestations import get_valid_attestation, make_attestation_signature + + +def get_valid_attester_slashing(state): + attestation_1 = get_valid_attestation(state) + attestation_2 = deepcopy(attestation_1) + attestation_2.data.target_root = b'\x01' * 32 + + make_attestation_signature(state, attestation_2) + + return AttesterSlashing( + attestation_1=convert_to_indexed(state, attestation_1), + attestation_2=convert_to_indexed(state, attestation_2), + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py new file mode 100644 index 000000000..7c25d073a --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py @@ -0,0 +1,11 @@ +def set_bitfield_bit(bitfield, i): + """ + Set the bit in ``bitfield`` at position ``i`` to ``1``. + """ + byte_index = i // 8 + bit_index = i % 8 + return ( + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py new file mode 100644 index 000000000..906901867 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -0,0 +1,62 @@ +from eth2spec.phase0 import spec +from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock + +from copy import deepcopy + +from eth2spec.test.helpers.keys import privkeys +from eth2spec.test.helpers.state import next_slot +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root + + +def make_block_signature(state, block): + assert block.slot == state.slot or block.slot == state.slot + 1 + if block.slot == state.slot: + proposer_index = get_beacon_proposer_index(state) + else: + # use stub state to get proposer index of next slot + stub_state = deepcopy(state) + next_slot(stub_state) + proposer_index = get_beacon_proposer_index(stub_state) + + privkey = privkeys[proposer_index] + + block.body.randao_reveal = bls_sign( + privkey=privkey, + message_hash=hash_tree_root(slot_to_epoch(block.slot)), + domain=get_domain( + state, + message_epoch=slot_to_epoch(block.slot), + domain_type=spec.DOMAIN_RANDAO, + ) + ) + block.signature = bls_sign( + message_hash=signing_root(block), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_BEACON_PROPOSER, + slot_to_epoch(block.slot))) + return block + + +def build_empty_block(state, slot=None, signed=False): + if slot is None: + slot = state.slot + empty_block = BeaconBlock() + empty_block.slot = slot + empty_block.body.eth1_data.deposit_count = state.deposit_index + previous_block_header = deepcopy(state.latest_block_header) + if previous_block_header.state_root == spec.ZERO_HASH: + previous_block_header.state_root = state.hash_tree_root() + empty_block.previous_block_root = signing_root(previous_block_header) + + if signed: + make_block_signature(state, empty_block) + + return empty_block + + +def build_empty_block_for_next_slot(state, signed=False): + return build_empty_block(state, state.slot + 1, signed=signed) + diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py new file mode 100644 index 000000000..cfc6add5e --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -0,0 +1,74 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_domain, DepositData, verify_merkle_branch, Deposit, ZERO_HASH +from eth2spec.test.helpers.keys import pubkeys, privkeys +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof +from eth2spec.utils.minimal_ssz import signing_root + + +def build_deposit_data(state, pubkey, privkey, amount): + deposit_data = DepositData( + pubkey=pubkey, + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + amount=amount, + ) + signature = bls_sign( + message_hash=signing_root(deposit_data), + privkey=privkey, + domain=get_domain( + state, + spec.DOMAIN_DEPOSIT, + ) + ) + deposit_data.signature = signature + return deposit_data + + +def build_deposit(state, + deposit_data_leaves, + pubkey, + privkey, + amount): + deposit_data = build_deposit_data(state, pubkey, privkey, amount) + + item = deposit_data.hash_tree_root() + index = len(deposit_data_leaves) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + root = get_merkle_root((tuple(deposit_data_leaves))) + proof = list(get_merkle_proof(tree, item_index=index)) + assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + + deposit = Deposit( + proof=list(proof), + index=index, + data=deposit_data, + ) + + return deposit, root, deposit_data_leaves + + +def prepare_state_and_deposit(state, validator_index, amount): + """ + Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. + """ + pre_validator_count = len(state.validator_registry) + # fill previous deposits with zero-hash + deposit_data_leaves = [ZERO_HASH] * pre_validator_count + + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + deposit, root, deposit_data_leaves = build_deposit( + state, + deposit_data_leaves, + pubkey, + privkey, + amount, + ) + + state.latest_eth1_data.deposit_root = root + state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + return deposit diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py new file mode 100644 index 000000000..b991f74e7 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -0,0 +1,51 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import Eth1Data, ZERO_HASH, get_active_validator_indices +from eth2spec.test.helpers.keys import pubkeys +from eth2spec.utils.minimal_ssz import hash_tree_root + + +def build_mock_validator(i: int, balance: int): + pubkey = pubkeys[i] + # insecurely use pubkey as withdrawal key as well + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + return spec.Validator( + pubkey=pubkeys[i], + withdrawal_credentials=withdrawal_credentials, + activation_eligibility_epoch=spec.FAR_FUTURE_EPOCH, + activation_epoch=spec.FAR_FUTURE_EPOCH, + exit_epoch=spec.FAR_FUTURE_EPOCH, + withdrawable_epoch=spec.FAR_FUTURE_EPOCH, + effective_balance=min(balance - balance % spec.EFFECTIVE_BALANCE_INCREMENT, spec.MAX_EFFECTIVE_BALANCE) + ) + + +def create_genesis_state(num_validators): + deposit_root = b'\x42' * 32 + + state = spec.BeaconState( + genesis_time=0, + deposit_index=num_validators, + latest_eth1_data=Eth1Data( + deposit_root=deposit_root, + deposit_count=num_validators, + block_hash=ZERO_HASH, + )) + + # We "hack" in the initial validators, + # as it is much faster than creating and processing genesis deposits for every single test case. + state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators + state.validator_registry = [build_mock_validator(i, state.balances[i]) for i in range(num_validators)] + + # Process genesis activations + for validator in state.validator_registry: + if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = spec.GENESIS_EPOCH + validator.activation_epoch = spec.GENESIS_EPOCH + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, spec.GENESIS_EPOCH)) + for index in range(spec.LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + + return state diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py new file mode 100644 index 000000000..fe27b78a1 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -0,0 +1,5 @@ +from py_ecc import bls + +privkeys = [i + 1 for i in range(1024)] +pubkeys = [bls.privtopub(privkey) for privkey in privkeys] +pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py new file mode 100644 index 000000000..be4f32d1c --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -0,0 +1,48 @@ +from copy import deepcopy + +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ + get_domain, ProposerSlashing +from eth2spec.test.helpers.keys import pubkey_to_privkey +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def get_valid_proposer_slashing(state): + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[-1] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + slot = state.slot + + header_1 = BeaconBlockHeader( + slot=slot, + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + block_body_root=ZERO_HASH, + ) + header_2 = deepcopy(header_1) + header_2.previous_block_root = b'\x02' * 32 + header_2.slot = slot + 1 + + domain = get_domain( + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, + ) + header_1.signature = bls_sign( + message_hash=signing_root(header_1), + privkey=privkey, + domain=domain, + ) + header_2.signature = bls_sign( + message_hash=signing_root(header_2), + privkey=privkey, + domain=domain, + ) + + return ProposerSlashing( + proposer_index=validator_index, + header_1=header_1, + header_2=header_2, + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py new file mode 100644 index 000000000..0261801df --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -0,0 +1,42 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.state_transition import state_transition_to, state_transition +from eth2spec.test.helpers.block import build_empty_block + + +def get_balance(state, index): + return state.balances[index] + + +def next_slot(state): + """ + Transition to the next slot. + """ + state_transition_to(state, state.slot + 1) + + +def next_epoch(state): + """ + Transition to the start slot of the next epoch + """ + slot = state.slot + spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) + state_transition_to(state, slot) + + +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state) + state_transition(state, block) + return block + + +def get_state_root(state, slot) -> bytes: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py new file mode 100644 index 000000000..b78359f4e --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -0,0 +1,50 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, ZERO_HASH, get_domain +from eth2spec.test.helpers.keys import pubkeys, privkeys +from eth2spec.test.helpers.state import get_balance +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): + if slot is None: + slot = state.slot + current_epoch = get_current_epoch(state) + if sender_index is None: + sender_index = get_active_validator_indices(state, current_epoch)[-1] + recipient_index = get_active_validator_indices(state, current_epoch)[0] + transfer_pubkey = pubkeys[-1] + transfer_privkey = privkeys[-1] + + if fee is None: + fee = get_balance(state, sender_index) // 32 + if amount is None: + amount = get_balance(state, sender_index) - fee + + transfer = Transfer( + sender=sender_index, + recipient=recipient_index, + amount=amount, + fee=fee, + slot=slot, + pubkey=transfer_pubkey, + signature=ZERO_HASH, + ) + transfer.signature = bls_sign( + message_hash=signing_root(transfer), + privkey=transfer_privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_TRANSFER, + message_epoch=get_current_epoch(state), + ) + ) + + # ensure withdrawal_credentials reproducable + state.validator_registry[transfer.sender].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + ) + + return transfer diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py new file mode 100644 index 000000000..a42830f83 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -0,0 +1,24 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import VoluntaryExit, get_domain +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def build_voluntary_exit(state, epoch, validator_index, privkey): + voluntary_exit = VoluntaryExit( + epoch=epoch, + validator_index=validator_index, + ) + voluntary_exit.signature = bls_sign( + message_hash=signing_root(voluntary_exit), + privkey=privkey, + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_VOLUNTARY_EXIT, + message_epoch=epoch, + ) + ) + + return voluntary_exit From a4e22639f346e7a0fbc71f0338fc3e9d9709cea7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:37:11 +0200 Subject: [PATCH 040/118] fix/update bls funcs for testing --- test_libs/pyspec/eth2spec/utils/bls.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index daeb361a9..23a9a7529 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -22,4 +22,18 @@ def bls_aggregate_pubkeys(pubkeys): if bls_active: return bls.aggregate_pubkeys(pubkeys) else: - return b'\x42' * 96 + return b'\xaa' * 48 + + +def bls_aggregate_signatures(signatures): + if bls_active: + return bls.aggregate_signatures(signatures) + else: + return b'\x22' * 96 + + +def bls_sign(message_hash, privkey, domain): + if bls_active: + return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) + else: + return b'\x11' * 96 From e07f1bc98ffe259175efd489b200606dc44fdca1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:45:25 +0200 Subject: [PATCH 041/118] update imports to new helpers + fix up imports --- .../block_processing/test_process_attestation.py | 16 ++++++++-------- .../test_process_attester_slashing.py | 6 +++--- .../test_process_block_header.py | 4 ++-- .../block_processing/test_process_deposit.py | 16 +++------------- .../test_process_proposer_slashing.py | 7 ++----- .../block_processing/test_process_transfer.py | 7 ++----- .../test_process_voluntary_exit.py | 8 ++------ .../pyspec/eth2spec/test/helpers/attestations.py | 2 -- test_libs/pyspec/eth2spec/test/helpers/block.py | 5 ++--- .../eth2spec/test/helpers/proposer_slashings.py | 1 - 10 files changed, 24 insertions(+), 48 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 98cb62ef7..7f8ac6e41 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -1,24 +1,24 @@ from copy import deepcopy import eth2spec.phase0.spec as spec - -from eth2spec.phase0.state_transition import ( - state_transition_to, -) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation ) -from eth2spec.test.helpers import ( +from eth2spec.phase0.state_transition import ( + state_transition_to, +) +from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.attestations import ( get_valid_attestation, + make_attestation_signature, +) +from eth2spec.test.helpers.state import ( apply_empty_block, next_epoch, next_slot, - make_attestation_signature, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error - def run_attestation_processing(state, attestation, valid=True): """ diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index a0334c892..0bbdb7d4a 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -4,12 +4,12 @@ from eth2spec.phase0.spec import ( process_attester_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.attestations import make_indexed_attestation_signature +from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.state import ( get_balance, - get_valid_attester_slashing, next_epoch, apply_empty_block, - make_indexed_attestation_signature ) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index f2695ad09..674f659a0 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -7,11 +7,11 @@ from eth2spec.phase0.spec import ( process_block_header, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - next_slot, make_block_signature ) +from eth2spec.test.helpers.state import next_slot def prepare_state_for_header_processing(state): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index fe2dae6a8..8f663c5c2 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,18 +1,8 @@ import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - ZERO_HASH, - process_deposit, -) -from eth2spec.test.helpers import ( - get_balance, - build_deposit, - prepare_state_and_deposit, - privkeys, - pubkeys, -) - +from eth2spec.phase0.spec import process_deposit from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.state import get_balance def run_deposit_processing(state, deposit, validator_index, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 609c97ce6..6efed8685 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -3,12 +3,9 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from eth2spec.test.helpers import ( - get_balance, - get_valid_proposer_slashing, -) - from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing +from eth2spec.test.helpers.state import get_balance def run_proposer_slashing_processing(state, proposer_slashing, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 64539d56e..fed948e63 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -6,11 +6,8 @@ from eth2spec.phase0.spec import ( process_transfer, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers import ( - get_valid_transfer, - next_epoch, - apply_empty_block -) +from eth2spec.test.helpers.state import next_epoch, apply_empty_block +from eth2spec.test.helpers.transfers import get_valid_transfer def run_transfer_processing(state, transfer, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index be0ef1e7a..f842d8b07 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -1,17 +1,13 @@ import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( get_active_validator_indices, get_churn_limit, get_current_epoch, process_voluntary_exit, ) -from eth2spec.test.helpers import ( - build_voluntary_exit, - pubkey_to_privkey, -) - from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.keys import pubkey_to_privkey +from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit def run_voluntary_exit_processing(state, voluntary_exit, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d7fab4c17..ebc76c27f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -2,14 +2,12 @@ from typing import List # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import ( Attestation, AttestationData, AttestationDataAndCustodyBit, get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) - from eth2spec.phase0.state_transition import ( state_transition, state_transition_to ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 906901867..25874e75b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -1,8 +1,7 @@ -from eth2spec.phase0 import spec -from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock - from copy import deepcopy +from eth2spec.phase0 import spec +from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index be4f32d1c..d14621b1c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -2,7 +2,6 @@ from copy import deepcopy # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec - from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ get_domain, ProposerSlashing from eth2spec.test.helpers.keys import pubkey_to_privkey From 5c3e760c2810e5ef567fd6f34940c17b20db3e4b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 18:58:12 +0200 Subject: [PATCH 042/118] update remaining imports --- .../test_process_crosslinks.py | 10 ++++++---- .../test_process_registry_updates.py | 5 +---- .../pyspec/eth2spec/test/test_finality.py | 10 ++++++---- test_libs/pyspec/eth2spec/test/test_sanity.py | 18 ++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index d4e6dc938..f342c5e6d 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -10,15 +10,17 @@ from eth2spec.phase0.state_transition import ( state_transition, ) from eth2spec.test.context import spec_state_test -from eth2spec.test.helpers import ( +from eth2spec.test.helpers.state import ( + next_epoch, + next_slot, + apply_empty_block +) +from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, fill_aggregate_attestation, get_crosslink_committee, get_valid_attestation, - next_epoch, - next_slot, - apply_empty_block ) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 970c30942..2086f4ef2 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -4,10 +4,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, ) -from eth2spec.test.helpers import ( - next_epoch, -) - +from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 75191e59a..732dd8f0c 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -5,14 +5,16 @@ from eth2spec.phase0.state_transition import ( state_transition, ) from .context import spec_state_test -from .helpers import ( - build_empty_block_for_next_slot, +from .helpers.state import ( + next_epoch, + apply_empty_block +) +from .helpers.block import build_empty_block_for_next_slot +from .helpers.attestations import ( fill_aggregate_attestation, get_current_epoch, get_epoch_start_slot, get_valid_attestation, - next_epoch, - apply_empty_block ) diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 1951415ac..1509a9843 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -19,18 +19,20 @@ from eth2spec.phase0.spec import ( from eth2spec.phase0.state_transition import ( state_transition, ) -from .helpers import ( +from .helpers.state import ( get_balance, - build_empty_block_for_next_slot, - get_state_root, - get_valid_attestation, - get_valid_attester_slashing, - get_valid_proposer_slashing, - get_valid_transfer, - prepare_state_and_deposit, + get_state_root +) +from .helpers.transfers import get_valid_transfer +from .helpers.block import build_empty_block_for_next_slot +from .helpers.keys import ( privkeys, pubkeys, ) +from .helpers.attester_slashings import get_valid_attester_slashing +from .helpers.proposer_slashings import get_valid_proposer_slashing +from .helpers.attestations import get_valid_attestation +from .helpers.deposits import prepare_state_and_deposit from .context import spec_state_test From 8a43ec01326cc5d8b0cdfad6b4d4e7a452c2d1ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:31:02 +0200 Subject: [PATCH 043/118] add signing methods --- .../test_process_attestation.py | 16 ++++++------- .../test_process_attester_slashing.py | 4 ++-- .../test_process_block_header.py | 4 ++-- .../eth2spec/test/helpers/attestations.py | 17 ++++++------- .../test/helpers/attester_slashings.py | 4 ++-- .../pyspec/eth2spec/test/helpers/block.py | 4 ++-- .../eth2spec/test/helpers/block_header.py | 18 ++++++++++++++ .../pyspec/eth2spec/test/helpers/deposits.py | 17 +++++++++---- .../test/helpers/proposer_slashings.py | 24 ++++--------------- .../pyspec/eth2spec/test/helpers/transfers.py | 22 ++++++++++------- .../eth2spec/test/helpers/voluntary_exits.py | 12 ++++++---- 11 files changed, 81 insertions(+), 61 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/helpers/block_header.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 7f8ac6e41..bbfab9a13 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -11,7 +11,7 @@ from eth2spec.phase0.state_transition import ( from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.attestations import ( get_valid_attestation, - make_attestation_signature, + sign_attestation, ) from eth2spec.test.helpers.state import ( apply_empty_block, @@ -105,7 +105,7 @@ def test_old_source_epoch(state): attestation.data.source_epoch -= 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -118,7 +118,7 @@ def test_wrong_shard(state): attestation.data.shard += 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -131,7 +131,7 @@ def test_new_source_epoch(state): attestation.data.source_epoch += 1 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -144,7 +144,7 @@ def test_source_root_is_target_root(state): attestation.data.source_root = attestation.data.target_root # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -171,7 +171,7 @@ def test_invalid_current_source_root(state): attestation.data.source_root = state.current_justified_root # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -184,7 +184,7 @@ def test_bad_source_root(state): attestation.data.source_root = b'\x42' * 32 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -197,7 +197,7 @@ def test_non_zero_crosslink_data_root(state): attestation.data.crosslink_data_root = b'\x42' * 32 # Re do signature - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 0bbdb7d4a..56b3711a4 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -4,7 +4,7 @@ from eth2spec.phase0.spec import ( process_attester_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.attestations import make_indexed_attestation_signature +from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.state import ( get_balance, @@ -81,7 +81,7 @@ def test_success_surround(state): attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 # correct the signature of attestation 1 - make_indexed_attestation_signature(state, attester_slashing.attestation_1) + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 674f659a0..09c0c5388 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -9,7 +9,7 @@ from eth2spec.phase0.spec import ( from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - make_block_signature + sign_block ) from eth2spec.test.helpers.state import next_slot @@ -59,7 +59,7 @@ def test_invalid_slot(state): def test_invalid_previous_block_root(state): block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root - make_block_signature(state, block) + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index ebc76c27f..de5f2d521 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -12,9 +12,10 @@ from eth2spec.phase0.state_transition import ( state_transition, state_transition_to ) from eth2spec.test.helpers.bitfields import set_bitfield_bit -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, make_block_signature +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.minimal_ssz import hash_tree_root def build_attestation_data(state, slot, shard): @@ -77,11 +78,11 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - make_attestation_signature(state, attestation) + sign_attestation(state, attestation) return attestation -def make_aggregate_attestation_signature(state: BeaconState, data: AttestationData, participants: List[int]): +def sign_aggregate_attestation(state: BeaconState, data: AttestationData, participants: List[int]): signatures = [] for validator_index in participants: privkey = privkeys[validator_index] @@ -96,19 +97,19 @@ def make_aggregate_attestation_signature(state: BeaconState, data: AttestationDa return bls_aggregate_signatures(signatures) -def make_indexed_attestation_signature(state, indexed_attestation: IndexedAttestation): +def sign_indexed_attestation(state, indexed_attestation: IndexedAttestation): participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices - indexed_attestation.signature = make_aggregate_attestation_signature(state, indexed_attestation.data, participants) + indexed_attestation.signature = sign_aggregate_attestation(state, indexed_attestation.data, participants) -def make_attestation_signature(state, attestation: Attestation): +def sign_attestation(state, attestation: Attestation): participants = get_attesting_indices( state, attestation.data, attestation.aggregation_bitfield, ) - attestation.signature = make_aggregate_attestation_signature(state, attestation.data, participants) + attestation.signature = sign_aggregate_attestation(state, attestation.data, participants) def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): @@ -139,5 +140,5 @@ def add_attestation_to_state(state, attestation, slot): block.slot = slot block.body.attestations.append(attestation) state_transition_to(state, block.slot) - make_block_signature(state, block) + sign_block(state, block) state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index 2895c7771..b34f6ff45 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.phase0.spec import AttesterSlashing, convert_to_indexed -from eth2spec.test.helpers.attestations import get_valid_attestation, make_attestation_signature +from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation def get_valid_attester_slashing(state): @@ -9,7 +9,7 @@ def get_valid_attester_slashing(state): attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 - make_attestation_signature(state, attestation_2) + sign_attestation(state, attestation_2) return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 25874e75b..36659f8fa 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -8,7 +8,7 @@ from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root -def make_block_signature(state, block): +def sign_block(state, block): assert block.slot == state.slot or block.slot == state.slot + 1 if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -51,7 +51,7 @@ def build_empty_block(state, slot=None, signed=False): empty_block.previous_block_root = signing_root(previous_block_header) if signed: - make_block_signature(state, empty_block) + sign_block(state, empty_block) return empty_block diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py new file mode 100644 index 000000000..9aba62d37 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -0,0 +1,18 @@ +# Access constants from spec pkg reference. +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.spec import get_domain +from eth2spec.utils.bls import bls_sign +from eth2spec.utils.minimal_ssz import signing_root + + +def sign_block_header(state, header, privkey): + domain = get_domain( + state=state, + domain_type=spec.DOMAIN_BEACON_PROPOSER, + ) + header.signature = bls_sign( + message_hash=signing_root(header), + privkey=privkey, + domain=domain, + ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index cfc6add5e..1bea2f923 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -8,13 +8,19 @@ from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merk from eth2spec.utils.minimal_ssz import signing_root -def build_deposit_data(state, pubkey, privkey, amount): +def build_deposit_data(state, pubkey, privkey, amount, signed=False): deposit_data = DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], amount=amount, ) + if signed: + sign_deposit_data(state, deposit_data, privkey) + return deposit_data + + +def sign_deposit_data(state, deposit_data, privkey): signature = bls_sign( message_hash=signing_root(deposit_data), privkey=privkey, @@ -24,15 +30,15 @@ def build_deposit_data(state, pubkey, privkey, amount): ) ) deposit_data.signature = signature - return deposit_data def build_deposit(state, deposit_data_leaves, pubkey, privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) + amount, + signed): + deposit_data = build_deposit_data(state, pubkey, privkey, amount, signed) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -51,7 +57,7 @@ def build_deposit(state, return deposit, root, deposit_data_leaves -def prepare_state_and_deposit(state, validator_index, amount): +def prepare_state_and_deposit(state, validator_index, amount, signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ @@ -67,6 +73,7 @@ def prepare_state_and_deposit(state, validator_index, amount): pubkey, privkey, amount, + signed ) state.latest_eth1_data.deposit_root = root diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d14621b1c..0dd1ce5c1 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,12 +1,8 @@ from copy import deepcopy -# Access constants from spec pkg reference. -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, \ - get_domain, ProposerSlashing +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, ProposerSlashing +from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey -from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root def get_valid_proposer_slashing(state): @@ -25,20 +21,8 @@ def get_valid_proposer_slashing(state): header_2.previous_block_root = b'\x02' * 32 header_2.slot = slot + 1 - domain = get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_PROPOSER, - ) - header_1.signature = bls_sign( - message_hash=signing_root(header_1), - privkey=privkey, - domain=domain, - ) - header_2.signature = bls_sign( - message_hash=signing_root(header_2), - privkey=privkey, - domain=domain, - ) + sign_block_header(state, header_1, privkey) + sign_block_header(state, header_2, privkey) return ProposerSlashing( proposer_index=validator_index, diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index b78359f4e..37547bf97 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -8,7 +8,7 @@ from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root -def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): +def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None, signed=False): if slot is None: slot = state.slot current_epoch = get_current_epoch(state) @@ -32,19 +32,25 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non pubkey=transfer_pubkey, signature=ZERO_HASH, ) + if signed: + sign_transfer(state, transfer, transfer_privkey) + + # ensure withdrawal_credentials reproducible + state.validator_registry[transfer.sender].withdrawal_credentials = ( + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + ) + + return transfer + + +def sign_transfer(state, transfer, privkey): transfer.signature = bls_sign( message_hash=signing_root(transfer), - privkey=transfer_privkey, + privkey=privkey, domain=get_domain( state=state, domain_type=spec.DOMAIN_TRANSFER, message_epoch=get_current_epoch(state), ) ) - - # ensure withdrawal_credentials reproducable - state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] - ) - return transfer diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index a42830f83..54376d694 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -6,19 +6,23 @@ from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root -def build_voluntary_exit(state, epoch, validator_index, privkey): +def build_voluntary_exit(state, epoch, validator_index, privkey, signed=False): voluntary_exit = VoluntaryExit( epoch=epoch, validator_index=validator_index, ) + if signed: + sign_voluntary_exit(state, voluntary_exit, privkey) + return voluntary_exit + + +def sign_voluntary_exit(state, voluntary_exit, privkey): voluntary_exit.signature = bls_sign( message_hash=signing_root(voluntary_exit), privkey=privkey, domain=get_domain( state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=epoch, + message_epoch=voluntary_exit.epoch, ) ) - - return voluntary_exit From 4dad74eec3f68b4e682bfba3c102a3196c1ecbe5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:47:50 +0200 Subject: [PATCH 044/118] attestation signing --- .../test_process_attestation.py | 39 +++++++++---------- .../eth2spec/test/helpers/attestations.py | 5 ++- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index bbfab9a13..c44e8e7b1 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -57,7 +57,7 @@ def run_attestation_processing(state, attestation, valid=True): @spec_state_test def test_success(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY yield from run_attestation_processing(state, attestation) @@ -65,7 +65,7 @@ def test_success(state): @spec_state_test def test_success_previous_epoch(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) next_epoch(state) apply_empty_block(state) @@ -74,7 +74,7 @@ def test_success_previous_epoch(state): @spec_state_test def test_before_inclusion_delay(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # do not increment slot to allow for inclusion delay yield from run_attestation_processing(state, attestation, False) @@ -82,7 +82,7 @@ def test_before_inclusion_delay(state): @spec_state_test def test_after_epoch_slots(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # increment past latest inclusion slot state_transition_to(state, state.slot + spec.SLOTS_PER_EPOCH + 1) apply_empty_block(state) @@ -96,7 +96,7 @@ def test_old_source_epoch(state): state.finalized_epoch = 2 state.previous_justified_epoch = 3 state.current_justified_epoch = 4 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch assert attestation.data.source_epoch == state.previous_justified_epoch @@ -104,7 +104,6 @@ def test_old_source_epoch(state): # Now go beyond that, it will be invalid attestation.data.source_epoch -= 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -112,12 +111,11 @@ def test_old_source_epoch(state): @spec_state_test def test_wrong_shard(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.shard += 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -125,12 +123,11 @@ def test_wrong_shard(state): @spec_state_test def test_new_source_epoch(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 1 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -138,12 +135,11 @@ def test_new_source_epoch(state): @spec_state_test def test_source_root_is_target_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = attestation.data.target_root - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -160,7 +156,7 @@ def test_invalid_current_source_root(state): state.current_justified_epoch = 4 state.current_justified_root = b'\xff' * 32 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: @@ -170,7 +166,6 @@ def test_invalid_current_source_root(state): # Make attestation source root invalid: should be previous justified, not current one attestation.data.source_root = state.current_justified_root - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -178,12 +173,11 @@ def test_invalid_current_source_root(state): @spec_state_test def test_bad_source_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -191,12 +185,11 @@ def test_bad_source_root(state): @spec_state_test def test_non_zero_crosslink_data_root(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink_data_root = b'\x42' * 32 - # Re do signature sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @@ -207,7 +200,7 @@ def test_bad_previous_crosslink(state): next_epoch(state) apply_empty_block(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) apply_empty_block(state) @@ -219,19 +212,23 @@ def test_bad_previous_crosslink(state): @spec_state_test def test_non_empty_custody_bitfield(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + + sign_attestation(state, attestation) yield from run_attestation_processing(state, attestation, False) @spec_state_test def test_empty_aggregation_bitfield(state): - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=False) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + sign_attestation(state, attestation) + yield from run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index de5f2d521..8b667f27d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -55,7 +55,7 @@ def build_attestation_data(state, slot, shard): ) -def get_valid_attestation(state, slot=None): +def get_valid_attestation(state, slot=None, signed=False): if slot is None: slot = state.slot @@ -78,7 +78,8 @@ def get_valid_attestation(state, slot=None): data=attestation_data, custody_bitfield=custody_bitfield, ) - sign_attestation(state, attestation) + if signed: + sign_attestation(state, attestation) return attestation From a9069cbf9e12b6154c90e5bfe524496bb84aacd3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 19:53:35 +0200 Subject: [PATCH 045/118] attester slashing signing --- .../test_process_attester_slashing.py | 14 ++++++++------ .../eth2spec/test/helpers/attester_slashings.py | 8 +++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 56b3711a4..35e6b0f71 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -74,13 +74,12 @@ def test_success_surround(state): apply_empty_block(state) state.current_justified_epoch += 1 - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) # set attestion1 to surround attestation 2 attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - # correct the signature of attestation 1 sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing) @@ -88,25 +87,27 @@ def test_success_surround(state): @spec_state_test def test_same_data(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.data = attester_slashing.attestation_2.data + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) @spec_state_test def test_no_double_or_surround(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.data.target_epoch += 1 + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) @spec_state_test def test_participants_already_slashed(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) # set all indices to slashed attestation_1 = attester_slashing.attestation_1 @@ -119,10 +120,11 @@ def test_participants_already_slashed(state): @spec_state_test def test_custody_bit_0_and_1(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) attester_slashing.attestation_1.custody_bit_1_indices = ( attester_slashing.attestation_1.custody_bit_0_indices ) + sign_indexed_attestation(state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(state, attester_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py index b34f6ff45..d19b41dfe 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -4,12 +4,14 @@ from eth2spec.phase0.spec import AttesterSlashing, convert_to_indexed from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -def get_valid_attester_slashing(state): - attestation_1 = get_valid_attestation(state) +def get_valid_attester_slashing(state, signed_1=False, signed_2=False): + attestation_1 = get_valid_attestation(state, signed=signed_1) + attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 - sign_attestation(state, attestation_2) + if signed_2: + sign_attestation(state, attestation_2) return AttesterSlashing( attestation_1=convert_to_indexed(state, attestation_1), From b92a9d88573652d0a275651bb89802eb509acab9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 20:50:11 +0200 Subject: [PATCH 046/118] sign block headers --- .../test/block_processing/test_process_block_header.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 09c0c5388..9d4d6192e 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,15 +49,16 @@ def test_success(state): @spec_state_test def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = state.slot + 2 # invalid slot + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) @spec_state_test def test_invalid_previous_block_root(state): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.previous_block_root = b'\12' * 32 # invalid prev root sign_block(state, block) From 242bb8c912a5acc836cf9f008c762aeb9879ac0d Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 20:50:20 +0200 Subject: [PATCH 047/118] sign deposits --- .../test/block_processing/test_process_deposit.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 8f663c5c2..a796be817 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,8 +1,9 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import process_deposit from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.deposits import prepare_state_and_deposit, sign_deposit_data from eth2spec.test.helpers.state import get_balance +from eth2spec.test.helpers.keys import privkeys def run_deposit_processing(state, deposit, validator_index, valid=True): @@ -51,7 +52,7 @@ def test_success(state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield from run_deposit_processing(state, deposit, validator_index) @@ -60,7 +61,7 @@ def test_success(state): def test_success_top_up(state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield from run_deposit_processing(state, deposit, validator_index) @@ -69,11 +70,13 @@ def test_success_top_up(state): def test_wrong_index(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # mess up deposit_index deposit.index = state.deposit_index + 1 + sign_deposit_data(state, deposit.data, privkeys[validator_index]) + yield from run_deposit_processing(state, deposit, validator_index, valid=False) @@ -84,9 +87,11 @@ def test_wrong_index(state): def test_bad_merkle_proof(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH + sign_deposit_data(state, deposit.data, privkeys[validator_index]) + yield from run_deposit_processing(state, deposit, validator_index, valid=False) From 32efe49b3358ecfbf234c71764e80ee1b069cb8c Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 21:07:24 +0200 Subject: [PATCH 048/118] proposer slashing signing --- .../test_process_proposer_slashing.py | 17 +++++++++------- .../test/helpers/proposer_slashings.py | 20 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 6efed8685..65d748ab5 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -4,6 +4,8 @@ from eth2spec.phase0.spec import ( process_proposer_slashing, ) from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.helpers.block_header import sign_block_header +from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.state import get_balance @@ -45,14 +47,14 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): @spec_state_test def test_success(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) yield from run_proposer_slashing_processing(state, proposer_slashing) @spec_state_test def test_invalid_proposer_index(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # Index just too high (by 1) proposer_slashing.proposer_index = len(state.validator_registry) @@ -61,17 +63,18 @@ def test_invalid_proposer_index(state): @spec_state_test def test_epochs_are_different(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH + sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.validator_index]) yield from run_proposer_slashing_processing(state, proposer_slashing, False) @spec_state_test def test_headers_are_same(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) # set headers to be the same proposer_slashing.header_2 = proposer_slashing.header_1 @@ -81,7 +84,7 @@ def test_headers_are_same(state): @spec_state_test def test_proposer_is_not_activated(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer to be not active yet state.validator_registry[proposer_slashing.proposer_index].activation_epoch = get_current_epoch(state) + 1 @@ -91,7 +94,7 @@ def test_proposer_is_not_activated(state): @spec_state_test def test_proposer_is_slashed(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer to slashed state.validator_registry[proposer_slashing.proposer_index].slashed = True @@ -101,7 +104,7 @@ def test_proposer_is_slashed(state): @spec_state_test def test_proposer_is_withdrawn(state): - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) # set proposer withdrawable_epoch in past current_epoch = get_current_epoch(state) diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 0dd1ce5c1..dfb8895dc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -1,11 +1,13 @@ from copy import deepcopy -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ZERO_HASH, ProposerSlashing +from eth2spec.phase0.spec import ( + get_current_epoch, get_active_validator_indices, BeaconBlockHeader, ProposerSlashing +) from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import pubkey_to_privkey -def get_valid_proposer_slashing(state): +def get_valid_proposer_slashing(state, signed_1=False, signed_2=False): current_epoch = get_current_epoch(state) validator_index = get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] @@ -13,16 +15,18 @@ def get_valid_proposer_slashing(state): header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - block_body_root=ZERO_HASH, + previous_block_root=b'\x33' * 32, + state_root=b'\x44' * 32, + block_body_root=b'\x55' * 32, ) header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02' * 32 + header_2.previous_block_root = b'\x99' * 32 header_2.slot = slot + 1 - sign_block_header(state, header_1, privkey) - sign_block_header(state, header_2, privkey) + if signed_1: + sign_block_header(state, header_1, privkey) + if signed_2: + sign_block_header(state, header_2, privkey) return ProposerSlashing( proposer_index=validator_index, From 9f00e4f0e464eddd09b06aabb2ade470e0e57e39 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 21:07:36 +0200 Subject: [PATCH 049/118] sign transfers --- .../block_processing/test_process_transfer.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index fed948e63..dc63bf491 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -44,7 +44,7 @@ def run_transfer_processing(state, transfer, valid=True): @spec_state_test def test_success_non_activated(state): - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH @@ -56,7 +56,7 @@ def test_success_withdrawable(state): next_epoch(state) apply_empty_block(state) - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) # withdrawable_epoch in past so can transfer state.validator_registry[transfer.sender].withdrawable_epoch = get_current_epoch(state) - 1 @@ -68,7 +68,7 @@ def test_success_withdrawable(state): def test_success_active_above_max_effective(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) yield from run_transfer_processing(state, transfer) @@ -77,7 +77,7 @@ def test_success_active_above_max_effective(state): def test_success_active_above_max_effective_fee(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1, signed=True) yield from run_transfer_processing(state, transfer) @@ -87,14 +87,14 @@ def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0, signed=True) yield from run_transfer_processing(state, transfer, False) @spec_state_test def test_incorrect_slot(state): - transfer = get_valid_transfer(state, slot=state.slot + 1) + transfer = get_valid_transfer(state, slot=state.slot + 1, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -105,7 +105,7 @@ def test_incorrect_slot(state): def test_insufficient_balance_for_fee(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=0, fee=1, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -117,7 +117,7 @@ def test_insufficient_balance_for_fee(state): def test_insufficient_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -129,7 +129,7 @@ def test_insufficient_balance(state): def test_no_dust_sender(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0, signed=True) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -141,7 +141,7 @@ def test_no_dust_sender(state): def test_no_dust_recipient(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 - transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0) + transfer = get_valid_transfer(state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 # un-activate so validator can transfer @@ -152,7 +152,7 @@ def test_no_dust_recipient(state): @spec_state_test def test_invalid_pubkey(state): - transfer = get_valid_transfer(state) + transfer = get_valid_transfer(state, signed=True) state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer From eea6c8f6456c9db7079ad4d90041a14ced5fd60b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 15 May 2019 23:26:05 +0200 Subject: [PATCH 050/118] voluntary exit testing sigs --- .../block_processing/test_process_voluntary_exit.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index f842d8b07..e5be38722 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -7,7 +7,7 @@ from eth2spec.phase0.spec import ( ) from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.keys import pubkey_to_privkey -from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit +from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit def run_voluntary_exit_processing(state, voluntary_exit, valid=True): @@ -52,6 +52,7 @@ def test_success(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit) @@ -76,6 +77,7 @@ def test_success_exit_queue(state): current_epoch, index, privkey, + signed=True, )) # Now run all the exits @@ -92,6 +94,7 @@ def test_success_exit_queue(state): current_epoch, validator_index, privkey, + signed=True, ) # This is the interesting part of the test: on a pre-state with a full exit queue, @@ -118,8 +121,10 @@ def test_validator_exit_in_future(state): current_epoch, validator_index, privkey, + signed=False, ) voluntary_exit.epoch += 1 + sign_voluntary_exit(state, voluntary_exit, privkey) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -138,8 +143,10 @@ def test_validator_invalid_validator_index(state): current_epoch, validator_index, privkey, + signed=False, ) voluntary_exit.validator_index = len(state.validator_registry) + sign_voluntary_exit(state, voluntary_exit, privkey) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -158,6 +165,7 @@ def test_validator_not_active(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -180,6 +188,7 @@ def test_validator_already_exited(state): current_epoch, validator_index, privkey, + signed=True, ) yield from run_voluntary_exit_processing(state, voluntary_exit, False) @@ -196,6 +205,7 @@ def test_validator_not_active_long_enough(state): current_epoch, validator_index, privkey, + signed=True, ) assert ( From 62999d8ded3e52bf7c3f60a0e30cec3218faa73f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 20:31:21 +0200 Subject: [PATCH 051/118] import fixes + fix import loop + fix minor signing things --- .../block_processing/test_process_attestation.py | 2 +- .../test_process_attester_slashing.py | 4 ++-- .../block_processing/test_process_block_header.py | 3 +-- .../test_process_proposer_slashing.py | 2 +- .../test/block_processing/test_process_transfer.py | 3 ++- test_libs/pyspec/eth2spec/test/context.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 11 +++++++++++ test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/state.py | 13 +------------ 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index c44e8e7b1..2c88f12b5 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -14,10 +14,10 @@ from eth2spec.test.helpers.attestations import ( sign_attestation, ) from eth2spec.test.helpers.state import ( - apply_empty_block, next_epoch, next_slot, ) +from eth2spec.test.helpers.block import apply_empty_block def run_attestation_processing(state, attestation, valid=True): diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 35e6b0f71..ea3a31aea 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -6,10 +6,10 @@ from eth2spec.phase0.spec import ( from eth2spec.test.context import spec_state_test, expect_assertion_error from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import ( get_balance, next_epoch, - apply_empty_block, ) @@ -63,7 +63,7 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): @spec_state_test def test_success_double(state): - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) yield from run_attester_slashing_processing(state, attester_slashing) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 9d4d6192e..69bacc84f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,9 +49,8 @@ def test_success(state): @spec_state_test def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state, signed=True) block.slot = state.slot + 2 # invalid slot - sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 65d748ab5..f3d1593a8 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -67,7 +67,7 @@ def test_epochs_are_different(state): # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.validator_index]) + sign_block_header(state, proposer_slashing.header_2, privkeys[proposer_slashing.proposer_index]) yield from run_proposer_slashing_processing(state, proposer_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index dc63bf491..2bbc022d0 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -6,7 +6,8 @@ from eth2spec.phase0.spec import ( process_transfer, ) from eth2spec.test.context import spec_state_test, expect_assertion_error -from eth2spec.test.helpers.state import next_epoch, apply_empty_block +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index ce088f81c..b2e67d761 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -1,7 +1,7 @@ from eth2spec.phase0 import spec from eth2spec.utils import bls -from .helpers import create_genesis_state +from .helpers.genesis import create_genesis_state from .utils import spectest, with_args diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 36659f8fa..b9c4ca712 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -2,6 +2,7 @@ from copy import deepcopy from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock +from eth2spec.phase0.state_transition import state_transition from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign @@ -39,6 +40,16 @@ def sign_block(state, block): return block +def apply_empty_block(state): + """ + Transition via an empty block (on current slot, assuming no block has been applied yet). + :return: the empty block that triggered the transition. + """ + block = build_empty_block(state, signed=True) + state_transition(state, block) + return block + + def build_empty_block(state, slot=None, signed=False): if slot is None: slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 1bea2f923..c5deb124e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -12,7 +12,7 @@ def build_deposit_data(state, pubkey, privkey, amount, signed=False): deposit_data = DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], amount=amount, ) if signed: diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index b991f74e7..01011cacd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -9,7 +9,7 @@ from eth2spec.utils.minimal_ssz import hash_tree_root def build_mock_validator(i: int, balance: int): pubkey = pubkeys[i] # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] return spec.Validator( pubkey=pubkeys[i], withdrawal_credentials=withdrawal_credentials, diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/test_libs/pyspec/eth2spec/test/helpers/state.py index 0261801df..e720a9709 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/state.py +++ b/test_libs/pyspec/eth2spec/test/helpers/state.py @@ -1,8 +1,7 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import state_transition_to, state_transition -from eth2spec.test.helpers.block import build_empty_block +from eth2spec.phase0.state_transition import state_transition_to def get_balance(state, index): @@ -24,16 +23,6 @@ def next_epoch(state): state_transition_to(state, slot) -def apply_empty_block(state): - """ - Transition via an empty block (on current slot, assuming no block has been applied yet). - :return: the empty block that triggered the transition. - """ - block = build_empty_block(state) - state_transition(state, block) - return block - - def get_state_root(state, slot) -> bytes: """ Return the state root at a recent ``slot``. From f937dec2a11551c4558b8dd170fb3c786b10a1b1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 20:54:34 +0200 Subject: [PATCH 052/118] more keys, more validators, fix import --- test_libs/pyspec/eth2spec/test/context.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/keys.py | 3 ++- test_libs/pyspec/eth2spec/test/test_finality.py | 7 ++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index b2e67d761..d3d9be588 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,7 +6,7 @@ from .helpers.genesis import create_genesis_state from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) +with_state = with_args(lambda: [create_genesis_state(spec.SHARD_COUNT * 2)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py index fe27b78a1..f59fbb159 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/keys.py +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -1,5 +1,6 @@ from py_ecc import bls +from eth2spec.phase0 import spec -privkeys = [i + 1 for i in range(1024)] +privkeys = [i + 1 for i in range(spec.SHARD_COUNT * 8)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 732dd8f0c..f47ee5f77 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -5,11 +5,8 @@ from eth2spec.phase0.state_transition import ( state_transition, ) from .context import spec_state_test -from .helpers.state import ( - next_epoch, - apply_empty_block -) -from .helpers.block import build_empty_block_for_next_slot +from .helpers.state import next_epoch +from .helpers.block import build_empty_block_for_next_slot, apply_empty_block from .helpers.attestations import ( fill_aggregate_attestation, get_current_epoch, From 183e3a515330fe7deb52c87993655c08f0bb19cb Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:02:00 +0200 Subject: [PATCH 053/118] minor refactor import fix --- .../eth2spec/test/epoch_processing/test_process_crosslinks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index f342c5e6d..5cada0aa5 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -12,9 +12,9 @@ from eth2spec.phase0.state_transition import ( from eth2spec.test.context import spec_state_test from eth2spec.test.helpers.state import ( next_epoch, - next_slot, - apply_empty_block + next_slot ) +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, From 08d0ff9336df627ec67bafc3bee538079b15be51 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:20:09 +0200 Subject: [PATCH 054/118] fix attestation aggregate bitfields --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 8b667f27d..9e00489a7 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -71,7 +71,7 @@ def get_valid_attestation(state, slot=None, signed=False): committee_size = len(crosslink_committee) bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) + aggregation_bitfield = b'\x00' * bitfield_length custody_bitfield = b'\x00' * bitfield_length attestation = Attestation( aggregation_bitfield=aggregation_bitfield, From 0f00b436986dccbd17b19270ab62be58bf47974f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:26:18 +0200 Subject: [PATCH 055/118] reduce validator key count again, fix valid attestation creation - snippet from Danny --- test_libs/pyspec/eth2spec/test/context.py | 2 +- .../pyspec/eth2spec/test/helpers/attestations.py | 14 +++++++------- test_libs/pyspec/eth2spec/test/helpers/keys.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index d3d9be588..b2e67d761 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -6,7 +6,7 @@ from .helpers.genesis import create_genesis_state from .utils import spectest, with_args # Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SHARD_COUNT * 2)]) +with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) # shorthand for decorating @with_state @spectest() diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 9e00489a7..7164cf160 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -6,8 +6,9 @@ from eth2spec.phase0.spec import ( Attestation, AttestationData, AttestationDataAndCustodyBit, - get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, get_shard_delta, - get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot) + get_epoch_start_slot, get_block_root, get_current_epoch, get_previous_epoch, slot_to_epoch, + get_crosslink_committee, get_domain, IndexedAttestation, get_attesting_indices, BeaconState, get_block_root_at_slot, + get_epoch_start_shard, get_epoch_committee_count) from eth2spec.phase0.state_transition import ( state_transition, state_transition_to ) @@ -59,11 +60,10 @@ def get_valid_attestation(state, slot=None, signed=False): if slot is None: slot = state.slot - if slot_to_epoch(slot) == get_current_epoch(state): - shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH - else: - previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) - shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT + epoch = slot_to_epoch(slot) + epoch_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot = get_epoch_committee_count(state, epoch) // spec.SLOTS_PER_EPOCH + shard = (epoch_start_shard + committees_per_slot * (slot % spec.SLOTS_PER_EPOCH)) % spec.SHARD_COUNT attestation_data = build_attestation_data(state, slot, shard) diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py index f59fbb159..f47cd7c10 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/keys.py +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -1,6 +1,6 @@ from py_ecc import bls from eth2spec.phase0 import spec -privkeys = [i + 1 for i in range(spec.SHARD_COUNT * 8)] +privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 16)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} From a10aba4bb9009e6c82f59491e0eb057f48c40d25 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:36:17 +0200 Subject: [PATCH 056/118] make valid attest creation fill aggregate bitfield --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 1 + test_libs/pyspec/eth2spec/test/test_finality.py | 2 -- test_libs/pyspec/eth2spec/test/test_sanity.py | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 7164cf160..b541e610f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -78,6 +78,7 @@ def get_valid_attestation(state, slot=None, signed=False): data=attestation_data, custody_bitfield=custody_bitfield, ) + fill_aggregate_attestation(state, attestation) if signed: sign_attestation(state, attestation) return attestation diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index f47ee5f77..26900340d 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -53,13 +53,11 @@ def next_epoch_with_attestations(state, slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): cur_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, cur_attestation) block.body.attestations.append(cur_attestation) if fill_prev_epoch: slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 prev_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, prev_attestation) block.body.attestations.append(prev_attestation) state_transition(post_state, block) diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 1509a9843..5e3b32416 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -157,7 +157,8 @@ def test_attester_slashing(state): pre_state = deepcopy(state) attester_slashing = get_valid_attester_slashing(state) - validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] + validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + + attester_slashing.attestation_1.custody_bit_1_indices)[0] assert not state.validator_registry[validator_index].slashed From 90bcbd6ff49cf1b836105e2e1e5345d054699dbc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 17 May 2019 21:56:41 +0200 Subject: [PATCH 057/118] fix attester slashing test --- .../test_process_attester_slashing.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index ea3a31aea..01861f04c 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -46,17 +46,22 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert ( - get_balance(state, slashed_index) < - pre_slashed_balance - ) + if slashed_index != proposer_index: + # lost whistleblower reward + assert ( + get_balance(state, slashed_index) < + pre_slashed_balance + ) - # gained whistleblower reward - assert ( - get_balance(state, proposer_index) > - pre_proposer_balance - ) + # gained whistleblower reward + assert ( + get_balance(state, proposer_index) > + pre_proposer_balance + ) + else: + # gained rewards for all slashings, which may include others. And only lost that of themselves. + # Netto at least 0, if more people where slashed, a balance increase. + assert get_balance(state, slashed_index) >= pre_slashed_balance yield 'post', state From 73c9d126dee0a4dafab4bbf00197ea56a2a3da94 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 13:49:06 +1000 Subject: [PATCH 058/118] Updated API spec with suggestions by @hwwhww. - Corrected to use American English instead of Australian - Fixed spelling mistake with indices - Changed tag to 'MinimalSet' and 'OptionalSet' - Added a response to the list of components - Renamed 'block_production' to 'block_proposal' --- .../validator/0_beacon-node-validator-api.md | 2 +- specs/validator/beacon_node_oapi.yaml | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 5c597aa75..ec0300e9a 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -11,7 +11,7 @@ The API is a REST interface, accessed via HTTP, designed for use as a local comm ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is a conceptually separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties includes the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties include the production of beacon blocks and signing of attestations. Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 0c2937f3d..e0e3da7d2 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -7,15 +7,15 @@ info: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" tags: - - name: Necessary for validator + - name: MinimalSet description: The minimal set of endpoints to enable a working validator implementation. - - name: Optional + - name: OptionalSet description: Extra endpoints which are nice-to-haves. paths: /node/version: get: tags: - - Necessary for validator + - MinimalSet summary: "Get version string of the running beacon node." description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: @@ -30,7 +30,7 @@ paths: /node/genesis_time: get: tags: - - Necessary for validator + - MinimalSet summary: "Get the genesis_time parameter from beacon node configuration." description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: @@ -46,7 +46,7 @@ paths: /node/syncing: get: tags: - - Necessary for validator + - MinimalSet summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -67,7 +67,7 @@ paths: /node/fork: get: tags: - - Optional + - OptionalSet summary: "Get fork information from running beacon node." description: "Requests the beacon node to provide which fork version it is currently on." responses: @@ -83,16 +83,16 @@ paths: chain_id: type: integer format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analagous to Eth1.0 JSON-RPC net_version." 500: $ref: '#/components/responses/InternalError' /validator/duties: get: tags: - - Necessary for validator + - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -122,7 +122,7 @@ paths: /validator/block: get: tags: - - Necessary for validator + - MinimalSet summary: "Produce a new block, without signature." description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." parameters: @@ -155,7 +155,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Necessary for validator + - MinimalSet summary: "Publish a signed block." description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: @@ -178,7 +178,7 @@ paths: /validator/attestation: get: tags: - - Necessary for validator + - MinimalSet summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: @@ -209,7 +209,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Necessary for validator + - MinimalSet summary: "Published a signed attestation." description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: @@ -263,7 +263,7 @@ components: type: integer format: uint64 description: "The shard in which the validator must attest." - block_production_slot: + block_proposal_slot: type: integer format: uint64 nullable: true @@ -537,15 +537,15 @@ components: type: object description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: - custody_bit_0_indicies: + custody_bit_0_indices: type: array - description: "Validator indicies for 0 bits." + description: "Validator indices for 0 bits." items: type: integer format: uint64 - custody_bit_1_indicies: + custody_bit_1_indices: type: array - description: "Validator indicies for 1 bits." + description: "Validator indices for 1 bits." items: type: integer format: uint64 @@ -611,3 +611,5 @@ components: description: "Beacon node internal error." CurrentlySyncing: description: "Beacon node is currently syncing, try again later." + NotFound: + description: "The requested API endpoint does not exist." From 490cf9e347c5229127af94e0bfa431d3a0f38b16 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 13:56:25 +1000 Subject: [PATCH 059/118] Updated version number to reflect small changes. --- specs/validator/beacon_node_oapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index e0e3da7d2..e995e9c84 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1" + version: "0.1.1" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" From 8e67dec7e40a241ace49efd1e02114c2830ddb7a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 16:43:21 +1000 Subject: [PATCH 060/118] Fixed misinterpretation of the proof array in the Deposit object, bumped version. --- specs/validator/beacon_node_oapi.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index e995e9c84..900b8eefa 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1.1" + version: "0.1.2" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" @@ -421,17 +421,11 @@ components: type: array description: "Branch in the deposit tree." items: - oneOf: - - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - - type: integer - format: uint64 - enum: [32] - example: 32 - description: "The DEPOSIT_CONTRACT_TREE_DEPTH value." - minItems: 2 - maxItems: 2 + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + minItems: 32 + maxItems: 32 index: type: integer format: uint64 From b0aea2a111d8be31b992863f0ca0261e5a1ce56f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 20 May 2019 19:38:18 +0200 Subject: [PATCH 061/118] bugfix block proc transfer test --- test_libs/pyspec/eth2spec/test/context.py | 11 +++++++++++ .../pyspec/eth2spec/test/helpers/transfers.py | 3 +-- test_libs/pyspec/eth2spec/test/test_finality.py | 7 +++++-- test_libs/pyspec/eth2spec/test/test_sanity.py | 14 ++++++++------ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index b2e67d761..a9eaea349 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -28,6 +28,17 @@ def expect_assertion_error(fn): raise AssertionError('expected an assertion error, but got none.') +def never_bls(fn): + """ + Decorator to apply on ``bls_switch`` decorator to force BLS de-activation. Useful to mark tests as BLS-ignorant. + """ + def entry(*args, **kw): + # override bls setting + kw['bls_active'] = False + fn(*args, **kw) + return entry + + def always_bls(fn): """ Decorator to apply on ``bls_switch`` decorator to force BLS activation. Useful to mark tests as BLS-dependent. diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 37547bf97..2045f48ad 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -1,7 +1,7 @@ # Access constants from spec pkg reference. import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, ZERO_HASH, get_domain +from eth2spec.phase0.spec import get_current_epoch, get_active_validator_indices, Transfer, get_domain from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.test.helpers.state import get_balance from eth2spec.utils.bls import bls_sign @@ -30,7 +30,6 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non fee=fee, slot=slot, pubkey=transfer_pubkey, - signature=ZERO_HASH, ) if signed: sign_transfer(state, transfer, transfer_privkey) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 26900340d..56f65eca9 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -4,11 +4,10 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.state_transition import ( state_transition, ) -from .context import spec_state_test +from .context import spec_state_test, never_bls from .helpers.state import next_epoch from .helpers.block import build_empty_block_for_next_slot, apply_empty_block from .helpers.attestations import ( - fill_aggregate_attestation, get_current_epoch, get_epoch_start_slot, get_valid_attestation, @@ -66,6 +65,7 @@ def next_epoch_with_attestations(state, return state, blocks, post_state +@never_bls @spec_state_test def test_finality_rule_4(state): yield 'pre', state @@ -93,6 +93,7 @@ def test_finality_rule_4(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_1(state): # get past first two epochs that finality does not run on @@ -122,6 +123,7 @@ def test_finality_rule_1(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_2(state): # get past first two epochs that finality does not run on @@ -153,6 +155,7 @@ def test_finality_rule_2(state): yield 'post', state +@never_bls @spec_state_test def test_finality_rule_3(state): """ diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 5e3b32416..27b40f93b 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -1,7 +1,7 @@ from copy import deepcopy -from py_ecc import bls import eth2spec.phase0.spec as spec +from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( @@ -24,7 +24,7 @@ from .helpers.state import ( get_state_root ) from .helpers.transfers import get_valid_transfer -from .helpers.block import build_empty_block_for_next_slot +from .helpers.block import build_empty_block_for_next_slot, sign_block from .helpers.keys import ( privkeys, pubkeys, @@ -58,7 +58,7 @@ def test_empty_block_transition(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -73,8 +73,9 @@ def test_skipped_slots(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += 3 + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -90,8 +91,9 @@ def test_empty_epoch_transition(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -289,7 +291,7 @@ def test_voluntary_exit(state): epoch=get_current_epoch(state), validator_index=validator_index, ) - voluntary_exit.signature = bls.sign( + voluntary_exit.signature = bls_sign( message_hash=signing_root(voluntary_exit), privkey=privkeys[validator_index], domain=get_domain( From ab251d47970ce7fed5ae3d6fe3551708c244ed86 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 20 May 2019 20:44:40 +0200 Subject: [PATCH 062/118] make tests work fast + bls signed optionally --- .../test_process_block_header.py | 4 +- .../test_process_crosslinks.py | 13 +- .../eth2spec/test/helpers/attestations.py | 2 +- .../pyspec/eth2spec/test/helpers/block.py | 26 ++-- test_libs/pyspec/eth2spec/test/test_sanity.py | 132 ++++++++++-------- 5 files changed, 97 insertions(+), 80 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 69bacc84f..3faf51e1b 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -42,13 +42,13 @@ def run_block_header_processing(state, block, valid=True): @spec_state_test -def test_success(state): +def test_success_block_header(state): block = build_empty_block_for_next_slot(state, signed=True) yield from run_block_header_processing(state, block) @spec_state_test -def test_invalid_slot(state): +def test_invalid_slot_block_header(state): block = build_empty_block_for_next_slot(state, signed=True) block.slot = state.slot + 2 # invalid slot diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 5cada0aa5..688bb54ac 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -14,7 +14,7 @@ from eth2spec.test.helpers.state import ( next_epoch, next_slot ) -from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.helpers.block import apply_empty_block, sign_block from eth2spec.test.helpers.attestations import ( add_attestation_to_state, build_empty_block_for_next_slot, @@ -33,8 +33,9 @@ def run_process_crosslinks(state, valid=True): """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = slot + sign_block(state, block) state_transition(state, block) # cache state before epoch transition @@ -57,7 +58,7 @@ def test_no_attestations(state): def test_single_crosslink_update_from_current_epoch(state): next_epoch(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation) add_attestation_to_state(state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) @@ -77,7 +78,7 @@ def test_single_crosslink_update_from_current_epoch(state): def test_single_crosslink_update_from_previous_epoch(state): next_epoch(state) - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation) add_attestation_to_state(state, attestation, state.slot + spec.SLOTS_PER_EPOCH) @@ -105,7 +106,7 @@ def test_double_late_crosslink(state): next_epoch(state) state.slot += 4 - attestation_1 = get_valid_attestation(state) + attestation_1 = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation_1) # add attestation_1 in the next epoch @@ -113,7 +114,7 @@ def test_double_late_crosslink(state): add_attestation_to_state(state, attestation_1, state.slot + 1) for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = get_valid_attestation(state) + attestation_2 = get_valid_attestation(state, signed=True) if attestation_2.data.shard == attestation_1.data.shard: break next_slot(state) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index b541e610f..e9b863463 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -138,7 +138,7 @@ def fill_aggregate_attestation(state, attestation): def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = slot block.body.attestations.append(attestation) state_transition_to(state, block.slot) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index b9c4ca712..6f1a0fe60 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -2,22 +2,26 @@ from copy import deepcopy from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock -from eth2spec.phase0.state_transition import state_transition +from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.test.helpers.state import next_slot from eth2spec.utils.bls import bls_sign from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root -def sign_block(state, block): - assert block.slot == state.slot or block.slot == state.slot + 1 - if block.slot == state.slot: - proposer_index = get_beacon_proposer_index(state) - else: - # use stub state to get proposer index of next slot - stub_state = deepcopy(state) - next_slot(stub_state) - proposer_index = get_beacon_proposer_index(stub_state) +def sign_block(state, block, proposer_index=None): + assert state.slot <= block.slot + + if proposer_index is None: + if block.slot == state.slot: + proposer_index = get_beacon_proposer_index(state) + else: + if slot_to_epoch(state.slot) + 1 > slot_to_epoch(block.slot): + print("warning: block slot far away, and no proposer index manually given." + " Signing block is slow due to transition for proposer index calculation.") + # use stub state to get proposer index of future slot + stub_state = deepcopy(state) + state_transition_to(stub_state, block.slot) + proposer_index = get_beacon_proposer_index(stub_state) privkey = privkeys[proposer_index] diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/test_sanity.py index 27b40f93b..6d65cc7f4 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/test_sanity.py @@ -34,7 +34,7 @@ from .helpers.proposer_slashings import get_valid_proposer_slashing from .helpers.attestations import get_valid_attestation from .helpers.deposits import prepare_state_and_deposit -from .context import spec_state_test +from .context import spec_state_test, never_bls @spec_state_test @@ -51,6 +51,7 @@ def test_slot_transition(state): assert get_state_root(state, pre_slot) == pre_root +@never_bls @spec_state_test def test_empty_block_transition(state): pre_slot = state.slot @@ -68,6 +69,7 @@ def test_empty_block_transition(state): assert get_block_root_at_slot(state, pre_slot) == block.previous_block_root +@never_bls @spec_state_test def test_skipped_slots(state): pre_slot = state.slot @@ -104,30 +106,31 @@ def test_empty_epoch_transition(state): assert get_block_root_at_slot(state, slot) == block.previous_block_root -@spec_state_test -def test_empty_epoch_transition_not_finalizing(state): - # copy for later balance lookups. - pre_state = deepcopy(state) - yield 'pre', state - - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH * 5 - yield 'blocks', [block], [spec.BeaconBlock] - - state_transition(state, block) - yield 'post', state - - assert state.slot == block.slot - assert state.finalized_epoch < get_current_epoch(state) - 4 - for index in range(len(state.validator_registry)): - assert get_balance(state, index) < get_balance(pre_state, index) +# @spec_state_test +# def test_empty_epoch_transition_not_finalizing(state): +# # copy for later balance lookups. +# pre_state = deepcopy(state) +# yield 'pre', state +# +# block = build_empty_block_for_next_slot(state, signed=False) +# block.slot += spec.SLOTS_PER_EPOCH * 5 +# sign_block(state, block, proposer_index=0) +# yield 'blocks', [block], [spec.BeaconBlock] +# +# state_transition(state, block) +# yield 'post', state +# +# assert state.slot == block.slot +# assert state.finalized_epoch < get_current_epoch(state) - 4 +# for index in range(len(state.validator_registry)): +# assert get_balance(state, index) < get_balance(pre_state, index) @spec_state_test def test_proposer_slashing(state): # copy for later balance lookups. pre_state = deepcopy(state) - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) validator_index = proposer_slashing.proposer_index assert not state.validator_registry[validator_index].slashed @@ -137,8 +140,9 @@ def test_proposer_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.proposer_slashings.append(proposer_slashing) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -158,7 +162,7 @@ def test_attester_slashing(state): # copy for later balance lookups. pre_state = deepcopy(state) - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=True) validator_index = (attester_slashing.attestation_1.custody_bit_0_indices + attester_slashing.attestation_1.custody_bit_1_indices)[0] @@ -169,8 +173,9 @@ def test_attester_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.attester_slashings.append(attester_slashing) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] state_transition(state, block) @@ -200,12 +205,13 @@ def test_deposit_in_block(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount) + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=True) yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.deposits.append(deposit) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -230,8 +236,9 @@ def test_deposit_top_up(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.deposits.append(deposit) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -249,23 +256,24 @@ def test_attestation(state): yield 'pre', state - attestation = get_valid_attestation(state) + attestation = get_valid_attestation(state, signed=True) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block_for_next_slot(state) + attestation_block = build_empty_block_for_next_slot(state, signed=False) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) + sign_block(state, attestation_block) state_transition(state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 - # Epoch transition should move to previous_epoch_attestations pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) - epoch_block = build_empty_block_for_next_slot(state) + epoch_block = build_empty_block_for_next_slot(state, signed=False) epoch_block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, epoch_block) state_transition(state, epoch_block) yield 'blocks', [attestation_block, epoch_block], [spec.BeaconBlock] @@ -301,15 +309,17 @@ def test_voluntary_exit(state): ) # Add to state via block transition - initiate_exit_block = build_empty_block_for_next_slot(state) + initiate_exit_block = build_empty_block_for_next_slot(state, signed=False) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) + sign_block(state, initiate_exit_block) state_transition(state, initiate_exit_block) assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH # Process within epoch transition - exit_block = build_empty_block_for_next_slot(state) + exit_block = build_empty_block_for_next_slot(state, signed=False) exit_block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, exit_block) state_transition(state, exit_block) yield 'blocks', [initiate_exit_block, exit_block], [spec.BeaconBlock] @@ -326,7 +336,7 @@ def test_transfer(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] amount = get_balance(state, sender_index) - transfer = get_valid_transfer(state, state.slot + 1, sender_index, amount) + transfer = get_valid_transfer(state, state.slot + 1, sender_index, amount, signed=True) recipient_index = transfer.recipient pre_transfer_recipient_balance = get_balance(state, recipient_index) @@ -336,8 +346,9 @@ def test_transfer(state): yield 'pre', state # Add to state via block transition - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.body.transfers.append(transfer) + sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -363,8 +374,9 @@ def test_balance_driven_status_transitions(state): yield 'pre', state # trigger epoch transition - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=False) block.slot += spec.SLOTS_PER_EPOCH + sign_block(state, block) state_transition(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -380,7 +392,7 @@ def test_historical_batch(state): yield 'pre', state - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(state, signed=True) state_transition(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -391,28 +403,28 @@ def test_historical_batch(state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -@spec_state_test -def test_eth1_data_votes(state): - yield 'pre', state - - expected_votes = 0 - assert len(state.eth1_data_votes) == expected_votes - - blocks = [] - for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = build_empty_block_for_next_slot(state) - state_transition(state, block) - expected_votes += 1 - assert len(state.eth1_data_votes) == expected_votes - blocks.append(block) - - block = build_empty_block_for_next_slot(state) - blocks.append(block) - - state_transition(state, block) - - yield 'blocks', [block], [spec.BeaconBlock] - yield 'post', state - - assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 - assert len(state.eth1_data_votes) == 1 +# @spec_state_test +# def test_eth1_data_votes(state): +# yield 'pre', state +# +# expected_votes = 0 +# assert len(state.eth1_data_votes) == expected_votes +# +# blocks = [] +# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): +# block = build_empty_block_for_next_slot(state, signed=False) +# state_transition(state, block) +# expected_votes += 1 +# assert len(state.eth1_data_votes) == expected_votes +# blocks.append(block) +# +# block = build_empty_block_for_next_slot(state, signed=False) +# blocks.append(block) +# +# state_transition(state, block) +# +# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'post', state +# +# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 +# assert len(state.eth1_data_votes) == 1 From 4ce441c4a45ee8273aaa94e5b6c65b1772ee6942 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 12:54:38 -0600 Subject: [PATCH 063/118] minor fix to how eth1 data is counted in validator guide --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d05f25ef2..d82bf6e56 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -233,7 +233,7 @@ epoch_signature = bls_sign( * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. * If `D` is nonempty: - * Let `best_vote_data` be the `eth1_data` of the member of `D` that has the highest `vote.vote_count`, breaking ties by favoring block hashes with higher associated block height. + * Let `best_vote_data` be the `eth1_data` member of `D` that has the highest vote count (`D.count(eth1_data)`), breaking ties by favoring block hashes with higher associated block height. * Set `block.eth1_data = best_vote_data`. ##### Signature From 4d08e9d7d6117b876f09cdc6f49784819c265cc1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 21:22:32 +0200 Subject: [PATCH 064/118] signed block header --- .../test/block_processing/test_process_block_header.py | 3 ++- test_libs/pyspec/eth2spec/test/context.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 3faf51e1b..9918e6283 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -49,8 +49,9 @@ def test_success_block_header(state): @spec_state_test def test_invalid_slot_block_header(state): - block = build_empty_block_for_next_slot(state, signed=True) + block = build_empty_block_for_next_slot(state, signed=False) block.slot = state.slot + 2 # invalid slot + sign_block(state, block) yield from run_block_header_processing(state, block, valid=False) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index a9eaea349..4c8ff255b 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -57,7 +57,7 @@ def bls_switch(fn): """ def entry(*args, **kw): old_state = bls.bls_active - bls.bls_active = kw.pop('bls_active', False) + bls.bls_active = kw.pop('bls_active', True) fn(*args, **kw) bls.bls_active = old_state return entry From ee9c1d911fd8cf1516a24136175115ef8857aa5e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 21:51:28 +0200 Subject: [PATCH 065/118] test tagging pattern --- test_libs/pyspec/eth2spec/test/context.py | 26 ++++++++++++++++--- .../pyspec/eth2spec/test/helpers/block.py | 9 ++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 4c8ff255b..7a8e959d0 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -3,12 +3,22 @@ from eth2spec.utils import bls from .helpers.genesis import create_genesis_state -from .utils import spectest, with_args +from .utils import spectest, with_args, with_tags # Provides a genesis state as first argument to the function decorated with this with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) +# BLS is turned off by default *for performance purposes during TESTING*. +# The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). +# - Some tests are marked as BLS-requiring, and ignore this setting. +# (tests that express differences caused by BLS, e.g. invalid signatures being rejected) +# - Some other tests are marked as BLS-ignoring, and ignore this setting. +# (tests that are heavily performance impacted / require unsigned state transitions) +# - Most tests respect the BLS setting. +DEFAULT_BLS_ACTIVE = False + + # shorthand for decorating @with_state @spectest() def spec_state_test(fn): return with_state(bls_switch(spectest()(fn))) @@ -28,6 +38,10 @@ def expect_assertion_error(fn): raise AssertionError('expected an assertion error, but got none.') +# Tags a test to be ignoring BLS for it to pass. +bls_ignored = with_tags({'bls_ignored': True}) + + def never_bls(fn): """ Decorator to apply on ``bls_switch`` decorator to force BLS de-activation. Useful to mark tests as BLS-ignorant. @@ -36,7 +50,11 @@ def never_bls(fn): # override bls setting kw['bls_active'] = False fn(*args, **kw) - return entry + return bls_ignored(entry) + + +# Tags a test to be requiring BLS for it to pass. +bls_required = with_tags({'bls_required': True}) def always_bls(fn): @@ -47,7 +65,7 @@ def always_bls(fn): # override bls setting kw['bls_active'] = True fn(*args, **kw) - return entry + return bls_required(entry) def bls_switch(fn): @@ -57,7 +75,7 @@ def bls_switch(fn): """ def entry(*args, **kw): old_state = bls.bls_active - bls.bls_active = kw.pop('bls_active', True) + bls.bls_active = kw.pop('bls_active', DEFAULT_BLS_ACTIVE) fn(*args, **kw) bls.bls_active = old_state return entry diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 6f1a0fe60..094423ab3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -4,13 +4,16 @@ from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils import bls from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root def sign_block(state, block, proposer_index=None): assert state.slot <= block.slot + if not bls.bls_active: + return + if proposer_index is None: if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -25,7 +28,7 @@ def sign_block(state, block, proposer_index=None): privkey = privkeys[proposer_index] - block.body.randao_reveal = bls_sign( + block.body.randao_reveal = bls.bls_sign( privkey=privkey, message_hash=hash_tree_root(slot_to_epoch(block.slot)), domain=get_domain( @@ -34,7 +37,7 @@ def sign_block(state, block, proposer_index=None): domain_type=spec.DOMAIN_RANDAO, ) ) - block.signature = bls_sign( + block.signature = bls.bls_sign( message_hash=signing_root(block), privkey=privkey, domain=get_domain( From cfc037fe75eff3d16a963ae0973de55688082544 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:28:47 +0200 Subject: [PATCH 066/118] tags applied after generation of vector (and only if), make BLS decorator --- .../pyspec/eth2spec/test/helpers/block.py | 12 ++--- test_libs/pyspec/eth2spec/test/utils.py | 33 ++++++++++--- test_libs/pyspec/eth2spec/utils/bls.py | 47 +++++++++++-------- 3 files changed, 59 insertions(+), 33 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 094423ab3..81c5e9ef5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -4,16 +4,15 @@ from eth2spec.phase0 import spec from eth2spec.phase0.spec import get_beacon_proposer_index, slot_to_epoch, get_domain, BeaconBlock from eth2spec.phase0.state_transition import state_transition, state_transition_to from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils import bls +from eth2spec.utils.bls import bls_sign, only_with_bls from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root +# Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. +@only_with_bls() def sign_block(state, block, proposer_index=None): assert state.slot <= block.slot - if not bls.bls_active: - return - if proposer_index is None: if block.slot == state.slot: proposer_index = get_beacon_proposer_index(state) @@ -28,7 +27,7 @@ def sign_block(state, block, proposer_index=None): privkey = privkeys[proposer_index] - block.body.randao_reveal = bls.bls_sign( + block.body.randao_reveal = bls_sign( privkey=privkey, message_hash=hash_tree_root(slot_to_epoch(block.slot)), domain=get_domain( @@ -37,14 +36,13 @@ def sign_block(state, block, proposer_index=None): domain_type=spec.DOMAIN_RANDAO, ) ) - block.signature = bls.bls_sign( + block.signature = bls_sign( message_hash=signing_root(block), privkey=privkey, domain=get_domain( state, spec.DOMAIN_BEACON_PROPOSER, slot_to_epoch(block.slot))) - return block def apply_empty_block(state): diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index 1c157bcee..c1d424109 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,5 +1,5 @@ +from typing import Dict, Any, Callable, Iterable from eth2spec.debug.encode import encode -from eth2spec.utils import bls def spectest(description: str = None): @@ -19,9 +19,6 @@ def spectest(description: str = None): else: # description can be explicit out['description'] = description - # If BLS is not active, mark the test as BLS-ignorant. - if not bls.bls_active: - out['stub_bls'] = True # put all generated data into a dict. for data in fn(*args, **kw): # If there is a type argument, encode it as that type. @@ -44,10 +41,34 @@ def spectest(description: str = None): return runner -def with_args(create_args): +def with_tags(tags: Dict[str, Any]): + """ + Decorator factory, adds tags (key, value) pairs to the output of the function. + Useful to build test-vector annotations with. + This decorator is applied after the ``spectest`` decorator is applied. + :param tags: dict of tags + :return: Decorator. + """ + def runner(fn): + def entry(*args, **kw): + fn_out = fn(*args, **kw) + # do not add tags if the function is not returning a dict at all (i.e. not in generator mode) + if fn_out is None: + return fn_out + return {**tags, **fn_out} + return entry + return runner + + +def with_args(create_args: Callable[[], Iterable[Any]]): + """ + Decorator factory, adds given extra arguments to the decorated function. + :param create_args: function to create arguments with. + :return: Decorator. + """ def runner(fn): # this wraps the function, to hide that the function actually yielding data. def entry(*args, **kw): - return fn(*(create_args() + list(args)), **kw) + return fn(*(list(create_args()) + list(args)), **kw) return entry return runner diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 23a9a7529..52f1fed63 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -3,37 +3,44 @@ from py_ecc import bls # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True +STUB_SIGNATURE = b'\x11' * 96 +STUB_PUBKEY = b'\x22' * 48 + +def only_with_bls(alt_return=None): + """ + Decorator factory to make a function only run when BLS is active. Otherwise return the default. + """ + def runner(fn): + def entry(*args, **kw): + if bls_active: + return fn(*args, **kw) + else: + return alt_return + return entry + return runner + + +@only_with_bls(alt_return=True) def bls_verify(pubkey, message_hash, signature, domain): - if bls_active: - return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) - else: - return True + return bls.verify(message_hash=message_hash, pubkey=pubkey, signature=signature, domain=domain) +@only_with_bls(alt_return=True) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - if bls_active: - return bls.verify_multiple(pubkeys, message_hashes, signature, domain) - else: - return True + return bls.verify_multiple(pubkeys, message_hashes, signature, domain) +@only_with_bls(alt_return=STUB_PUBKEY) def bls_aggregate_pubkeys(pubkeys): - if bls_active: - return bls.aggregate_pubkeys(pubkeys) - else: - return b'\xaa' * 48 + return bls.aggregate_pubkeys(pubkeys) +@only_with_bls(alt_return=STUB_SIGNATURE) def bls_aggregate_signatures(signatures): - if bls_active: - return bls.aggregate_signatures(signatures) - else: - return b'\x22' * 96 + return bls.aggregate_signatures(signatures) +@only_with_bls(alt_return=STUB_SIGNATURE) def bls_sign(message_hash, privkey, domain): - if bls_active: - return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) - else: - return b'\x11' * 96 + return bls.sign(message_hash=message_hash, privkey=privkey, domain=domain) From 8303c58aa41248ffef99175e2707c6e63fe63591 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:53:46 +0200 Subject: [PATCH 067/118] bugfix, make BLS wrapper propagate test output properly --- test_libs/pyspec/eth2spec/test/context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 7a8e959d0..2136a6053 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -76,6 +76,7 @@ def bls_switch(fn): def entry(*args, **kw): old_state = bls.bls_active bls.bls_active = kw.pop('bls_active', DEFAULT_BLS_ACTIVE) - fn(*args, **kw) + out = fn(*args, **kw) bls.bls_active = old_state + return out return entry From f1ba5aadceb755623ee11e5ada52b2b83a1f2f3e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:55:02 +0200 Subject: [PATCH 068/118] make test generator use BLS --- test_generators/operations/suite_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py index d27e3efc1..c0bc1aa44 100644 --- a/test_generators/operations/suite_creator.py +++ b/test_generators/operations/suite_creator.py @@ -15,7 +15,7 @@ def generate_from_tests(pkg): for name in fn_names: tfn = getattr(pkg, name) try: - out.append(tfn(generator_mode=True)) + out.append(tfn(generator_mode=True, bls_active=True)) except AssertionError: print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__)) return out From 0e136b1104d6a5eed90dba8d2779768bbc3a8c83 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:55:22 +0200 Subject: [PATCH 069/118] update yaml and eth-utils dependencies --- test_generators/README.md | 2 +- test_generators/bls/requirements.txt | 2 +- test_generators/operations/requirements.txt | 2 +- test_generators/shuffling/requirements.txt | 2 +- test_generators/ssz_generic/requirements.txt | 2 +- test_generators/ssz_static/requirements.txt | 2 +- test_libs/config_helpers/requirements.txt | 2 +- test_libs/config_helpers/setup.py | 2 +- test_libs/gen_helpers/requirements.txt | 4 ++-- test_libs/gen_helpers/setup.py | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test_generators/README.md b/test_generators/README.md index 43bf7af03..309a64bd9 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -58,7 +58,7 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 8a933d41c..329a4ce15 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ py-ecc==1.6.0 -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers diff --git a/test_generators/operations/requirements.txt b/test_generators/operations/requirements.txt index 8f9bede8f..595cee69c 100644 --- a/test_generators/operations/requirements.txt +++ b/test_generators/operations/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/shuffling/requirements.txt b/test_generators/shuffling/requirements.txt index 8f9bede8f..595cee69c 100644 --- a/test_generators/shuffling/requirements.txt +++ b/test_generators/shuffling/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/ssz_generic/requirements.txt b/test_generators/ssz_generic/requirements.txt index 94afc9d91..dcdb0824f 100644 --- a/test_generators/ssz_generic/requirements.txt +++ b/test_generators/ssz_generic/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ssz==0.1.0a2 diff --git a/test_generators/ssz_static/requirements.txt b/test_generators/ssz_static/requirements.txt index 8f9bede8f..595cee69c 100644 --- a/test_generators/ssz_static/requirements.txt +++ b/test_generators/ssz_static/requirements.txt @@ -1,4 +1,4 @@ -eth-utils==1.4.1 +eth-utils==1.6.0 ../../test_libs/gen_helpers ../../test_libs/config_helpers ../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt index e441a474b..f2f208c3f 100644 --- a/test_libs/config_helpers/requirements.txt +++ b/test_libs/config_helpers/requirements.txt @@ -1 +1 @@ -ruamel.yaml==0.15.87 +ruamel.yaml==0.15.96 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 90ad94ee4..9f0ea0641 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -4,6 +4,6 @@ setup( name='config_helpers', packages=['preset_loader'], install_requires=[ - "ruamel.yaml==0.15.87" + "ruamel.yaml==0.15.96" ] ) diff --git a/test_libs/gen_helpers/requirements.txt b/test_libs/gen_helpers/requirements.txt index 3d6a39458..557cae631 100644 --- a/test_libs/gen_helpers/requirements.txt +++ b/test_libs/gen_helpers/requirements.txt @@ -1,2 +1,2 @@ -ruamel.yaml==0.15.87 -eth-utils==1.4.1 +ruamel.yaml==0.15.96 +eth-utils==1.6.0 diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 5de27a6db..29cf04fd1 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -4,7 +4,7 @@ setup( name='gen_helpers', packages=['gen_base'], install_requires=[ - "ruamel.yaml==0.15.87", - "eth-utils==1.4.1" + "ruamel.yaml==0.15.96", + "eth-utils==1.6.0" ] ) From 6b3b5121f29fcb7b0ff2d316e7b50e1f17054629 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 22:59:47 +0200 Subject: [PATCH 070/118] fix header format of operations tests --- test_generators/operations/suite_creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py index c0bc1aa44..caff0c7db 100644 --- a/test_generators/operations/suite_creator.py +++ b/test_generators/operations/suite_creator.py @@ -34,6 +34,6 @@ def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], forks=["phase0"], config=config_name, runner="operations", - handler=config_name, + handler=operation_name, test_cases=get_cases())) return suite_definition From 12af078a6c4765f8486262f7f3a36fc11eafaf0a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 21 May 2019 23:03:59 +0200 Subject: [PATCH 071/118] make decorators return wrapped fn results properly --- test_libs/pyspec/eth2spec/test/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 2136a6053..a484cc995 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -49,7 +49,7 @@ def never_bls(fn): def entry(*args, **kw): # override bls setting kw['bls_active'] = False - fn(*args, **kw) + return fn(*args, **kw) return bls_ignored(entry) @@ -64,7 +64,7 @@ def always_bls(fn): def entry(*args, **kw): # override bls setting kw['bls_active'] = True - fn(*args, **kw) + return fn(*args, **kw) return bls_required(entry) From b919d08ab2362915b88ef4eae3b214236eea2570 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 00:07:52 +0200 Subject: [PATCH 072/118] comment on the deposit signature being soft-rejected --- specs/core/0_beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e56fd976c..46c811fed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1756,7 +1756,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) + # Verify the deposit signature (proof of possession). + # Invalid signatures are allowed by the deposit contract, and hence included on-chain, but must not be processed. if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): return From a4363ba0969f5ec6347665d5da087e286c730eec Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 00:52:57 +0200 Subject: [PATCH 073/118] tests for invalid signatures --- .../test_process_attestation.py | 11 ++++- .../test_process_attester_slashing.py | 23 +++++++++- .../test_process_block_header.py | 9 +++- .../block_processing/test_process_deposit.py | 45 +++++++++++++++---- .../test_process_proposer_slashing.py | 23 +++++++++- .../block_processing/test_process_transfer.py | 12 ++++- .../test_process_voluntary_exit.py | 25 +++++++---- 7 files changed, 126 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 2c88f12b5..9491ee001 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -8,7 +8,7 @@ from eth2spec.phase0.spec import ( from eth2spec.phase0.state_transition import ( state_transition_to, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, @@ -72,6 +72,15 @@ def test_success_previous_epoch(state): yield from run_attestation_processing(state, attestation) +@always_bls +@spec_state_test +def test_invalid_attestation_signature(state): + attestation = get_valid_attestation(state, signed=False) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + yield from run_attestation_processing(state, attestation, False) + + @spec_state_test def test_before_inclusion_delay(state): attestation = get_valid_attestation(state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 01861f04c..1481c2b87 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -3,7 +3,7 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.block import apply_empty_block @@ -90,6 +90,27 @@ def test_success_surround(state): yield from run_attester_slashing_processing(state, attester_slashing) +@always_bls +@spec_state_test +def test_invalid_sig_1(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_2(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=True, signed_2=False) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_1_and_2(state): + attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=False) + yield from run_attester_slashing_processing(state, attester_slashing, False) + + @spec_state_test def test_same_data(state): attester_slashing = get_valid_attester_slashing(state, signed_1=False, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 9918e6283..28e215a3a 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -6,7 +6,7 @@ from eth2spec.phase0.spec import ( advance_slot, process_block_header, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, sign_block @@ -47,6 +47,13 @@ def test_success_block_header(state): yield from run_block_header_processing(state, block) +@always_bls +@spec_state_test +def test_invalid_sig_block_header(state): + block = build_empty_block_for_next_slot(state, signed=False) + yield from run_block_header_processing(state, block, valid=False) + + @spec_state_test def test_invalid_slot_block_header(state): block = build_empty_block_for_next_slot(state, signed=False) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index a796be817..f734508aa 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -1,12 +1,12 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import process_deposit -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.deposits import prepare_state_and_deposit, sign_deposit_data from eth2spec.test.helpers.state import get_balance from eth2spec.test.helpers.keys import privkeys -def run_deposit_processing(state, deposit, validator_index, valid=True): +def run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False): """ Run ``process_deposit``, yielding: - pre-state ('pre') @@ -34,21 +34,27 @@ def run_deposit_processing(state, deposit, validator_index, valid=True): yield 'post', state - if validator_index < pre_validator_count: - # top-up + if non_effective: assert len(state.validator_registry) == pre_validator_count assert len(state.balances) == pre_validator_count + if validator_index < pre_validator_count: + assert get_balance(state, validator_index) == pre_balance else: - # new validator - assert len(state.validator_registry) == pre_validator_count + 1 - assert len(state.balances) == pre_validator_count + 1 + if validator_index < pre_validator_count: + # top-up + assert len(state.validator_registry) == pre_validator_count + assert len(state.balances) == pre_validator_count + else: + # new validator + assert len(state.validator_registry) == pre_validator_count + 1 + assert len(state.balances) == pre_validator_count + 1 + assert get_balance(state, validator_index) == pre_balance + deposit.data.amount assert state.deposit_index == state.latest_eth1_data.deposit_count - assert get_balance(state, validator_index) == pre_balance + deposit.data.amount @spec_state_test -def test_success(state): +def test_new_deposit(state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE @@ -57,6 +63,16 @@ def test_success(state): yield from run_deposit_processing(state, deposit, validator_index) +@always_bls +@spec_state_test +def test_invalid_sig_new_deposit(state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validator_registry) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=True) + + @spec_state_test def test_success_top_up(state): validator_index = 0 @@ -66,6 +82,17 @@ def test_success_top_up(state): yield from run_deposit_processing(state, deposit, validator_index) +@always_bls +@spec_state_test +def test_invalid_sig_top_up(state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + + # invalid signatures, in top-ups, are allowed! + yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False) + + @spec_state_test def test_wrong_index(state): validator_index = len(state.validator_registry) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index f3d1593a8..7ec10406d 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -3,7 +3,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -52,6 +52,27 @@ def test_success(state): yield from run_proposer_slashing_processing(state, proposer_slashing) +@always_bls +@spec_state_test +def test_invalid_sig_1(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=False, signed_2=True) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_2(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=False) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + +@always_bls +@spec_state_test +def test_invalid_sig_1_and_2(state): + proposer_slashing = get_valid_proposer_slashing(state, signed_1=False, signed_2=False) + yield from run_proposer_slashing_processing(state, proposer_slashing, False) + + @spec_state_test def test_invalid_proposer_index(state): proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 2bbc022d0..e5f52e209 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -5,7 +5,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_transfer, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer @@ -83,6 +83,16 @@ def test_success_active_above_max_effective_fee(state): yield from run_transfer_processing(state, transfer) +@always_bls +@spec_state_test +def test_invalid_signature(state): + transfer = get_valid_transfer(state, signed=False) + # un-activate so validator can transfer + state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(state, transfer, False) + + @spec_state_test def test_active_but_transfer_past_effective_balance(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index e5be38722..fe33fb631 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -5,7 +5,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, process_voluntary_exit, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit @@ -47,17 +47,26 @@ def test_success(state): validator_index = get_active_validator_indices(state, current_epoch)[0] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - signed=True, - ) + voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey, signed=True) yield from run_voluntary_exit_processing(state, voluntary_exit) +@always_bls +@spec_state_test +def test_invalid_signature(state): + # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + current_epoch = get_current_epoch(state) + validator_index = get_active_validator_indices(state, current_epoch)[0] + privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + + voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey, signed=False) + + yield from run_voluntary_exit_processing(state, voluntary_exit, False) + + @spec_state_test def test_success_exit_queue(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit From 7d3147d418e320c0b433d9de872f8b4dad0d5186 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 04:19:54 +0200 Subject: [PATCH 074/118] add inconsistent bitfields test --- .../block_processing/test_process_attestation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 9491ee001..0dae852f0 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -219,6 +219,18 @@ def test_bad_previous_crosslink(state): yield from run_attestation_processing(state, attestation, False) +@spec_state_test +def test_inconsistent_bitfields(state): + attestation = get_valid_attestation(state, signed=False) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + + sign_attestation(state, attestation) + + yield from run_attestation_processing(state, attestation, False) + + @spec_state_test def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state, signed=False) From dad89ae926988fa4a484c301e24f57646d8980a2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 23:20:12 -0600 Subject: [PATCH 075/118] minor lint --- .../test_process_attester_slashing.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py index 1481c2b87..28e232277 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py @@ -48,16 +48,9 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): if slashed_index != proposer_index: # lost whistleblower reward - assert ( - get_balance(state, slashed_index) < - pre_slashed_balance - ) - + assert get_balance(state, slashed_index) < pre_slashed_balance # gained whistleblower reward - assert ( - get_balance(state, proposer_index) > - pre_proposer_balance - ) + assert get_balance(state, proposer_index) > pre_proposer_balance else: # gained rewards for all slashings, which may include others. And only lost that of themselves. # Netto at least 0, if more people where slashed, a balance increase. From 58fd712607ba3fd635feb29e5364dff72add5fdf Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 20:50:18 +0200 Subject: [PATCH 076/118] fix style issue with deposit processing helper --- .../test/block_processing/test_process_deposit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index f734508aa..b520c809f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.state import get_balance from eth2spec.test.helpers.keys import privkeys -def run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False): +def run_deposit_processing(state, deposit, validator_index, valid=True, effective=True): """ Run ``process_deposit``, yielding: - pre-state ('pre') @@ -34,7 +34,7 @@ def run_deposit_processing(state, deposit, validator_index, valid=True, non_effe yield 'post', state - if non_effective: + if not effective: assert len(state.validator_registry) == pre_validator_count assert len(state.balances) == pre_validator_count if validator_index < pre_validator_count: @@ -70,7 +70,7 @@ def test_invalid_sig_new_deposit(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) - yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=True) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=False) @spec_state_test @@ -90,7 +90,7 @@ def test_invalid_sig_top_up(state): deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) # invalid signatures, in top-ups, are allowed! - yield from run_deposit_processing(state, deposit, validator_index, valid=True, non_effective=False) + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) @spec_state_test From 958f71bb68e6dede30a1e875cadb85edb880a9a4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 21:03:46 +0200 Subject: [PATCH 077/118] minor underflow fix for proposer slashing test --- .../test/block_processing/test_process_proposer_slashing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 7ec10406d..07ccc25f1 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -127,6 +127,8 @@ def test_proposer_is_slashed(state): def test_proposer_is_withdrawn(state): proposer_slashing = get_valid_proposer_slashing(state, signed_1=True, signed_2=True) + # move 1 epoch into future, to allow for past withdrawable epoch + state.slot += spec.SLOTS_PER_EPOCH # set proposer withdrawable_epoch in past current_epoch = get_current_epoch(state) proposer_index = proposer_slashing.proposer_index From 6cd681e952aac4641f1e6653b339848208d62259 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 11:22:10 +0200 Subject: [PATCH 078/118] update docs on operation testing --- specs/test_formats/operations/README.md | 36 +++++++++++++++++++++-- specs/test_formats/operations/deposits.md | 18 ------------ 2 files changed, 33 insertions(+), 21 deletions(-) delete mode 100644 specs/test_formats/operations/deposits.md diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 842dc3615..a99a70788 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -2,9 +2,39 @@ The different kinds of operations ("transactions") are tested individually with test handlers. -The tested operation kinds are: -- [`deposits`](./deposits.md) -- More tests are work-in-progress. +## Test case format +```yaml +description: string -- description of test case, purely for debugging purposes +bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. +bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +pre: BeaconState -- state before applying the deposit +: -- the YAML encoded operation, e.g. a "ProposerSlashing", or "Deposit". +post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. +``` +Note: if both `bls_required` and `bls_ignored` are false (or simply not included), + then the test consumer can freely choose to run with BLS ON or OFF. +One may choose for OFF for performance reasons during repeated testing. Otherwise it is recommended to run with BLS ON. +## Condition + +A handler of the `operations` should process these cases, + calling the corresponding processing implementation. + +Operations: + +| *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | +|-------------------------|----------------------|----------------------|--------------------------------------------------------| +| `attestation` | `Attestation` | `attestation` | `process_deposit(state, attestation)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_deposit(state, attester_slashing)` | +| `block_header` | `Block` | `block` | `process_block_header(state, block)` | +| `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | +| `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | +| `transfer` | `Transfer` | `transfer` | `process_transfer(state, transfer)` | +| `voluntary_exit` | `VoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | + +Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. + +The resulting state should match the expected `post` state, or if the `post` state is left blank, + the handler should reject the input operation as invalid. diff --git a/specs/test_formats/operations/deposits.md b/specs/test_formats/operations/deposits.md deleted file mode 100644 index 8f44ebb22..000000000 --- a/specs/test_formats/operations/deposits.md +++ /dev/null @@ -1,18 +0,0 @@ -# Test format: Deposit operations - -A deposit is a form of an operation (or "transaction"), modifying the state. - -## Test case format - -```yaml -description: string -- description of test case, purely for debugging purposes -pre: BeaconState -- state before applying the deposit -deposit: Deposit -- the deposit -post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. -``` - -## Condition - -A `deposits` handler of the `operations` should process these cases, - calling the implementation of the `process_deposit(state, deposit)` functionality described in the spec. -The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the inputs as invalid. From e218c4f61cb4cb188d965768cc95c35c023cd8f9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 11:23:03 +0200 Subject: [PATCH 079/118] update operations readme, fix wording --- test_generators/operations/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/test_generators/operations/README.md b/test_generators/operations/README.md index e0b9d0e18..5cb3afc98 100644 --- a/test_generators/operations/README.md +++ b/test_generators/operations/README.md @@ -3,7 +3,6 @@ Operations (or "transactions" in previous spec iterations), are atomic changes to the state, introduced by embedding in blocks. -This generator provides a series of test suites, divided into handler, for each operation type. An operation test-runner can consume these operation test-suites, and handle different kinds of operations by processing the cases using the specified test handler. From 754d97210888543e7f705b4bce62a6705a8cdb9d Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 15:11:46 +0200 Subject: [PATCH 080/118] implement epoch processing test-gen, bugfix tests --- test_generators/epoch_processing/README.md | 11 ++++++ test_generators/epoch_processing/main.py | 38 ++++++++++++++++++ .../epoch_processing/requirements.txt | 4 ++ test_generators/operations/suite_creator.py | 39 ------------------- .../gen_helpers/gen_from_tests/__init__.py | 0 test_libs/gen_helpers/gen_from_tests/gen.py | 22 +++++++++++ test_libs/gen_helpers/setup.py | 2 +- .../test_process_registry_updates.py | 25 ++++++------ 8 files changed, 87 insertions(+), 54 deletions(-) create mode 100644 test_generators/epoch_processing/README.md create mode 100644 test_generators/epoch_processing/main.py create mode 100644 test_generators/epoch_processing/requirements.txt delete mode 100644 test_generators/operations/suite_creator.py create mode 100644 test_libs/gen_helpers/gen_from_tests/__init__.py create mode 100644 test_libs/gen_helpers/gen_from_tests/gen.py diff --git a/test_generators/epoch_processing/README.md b/test_generators/epoch_processing/README.md new file mode 100644 index 000000000..9b57875e2 --- /dev/null +++ b/test_generators/epoch_processing/README.md @@ -0,0 +1,11 @@ +# Epoch processing + +Epoch processing covers the sub-transitions during an epoch change. + +An epoch-processing test-runner can consume these sub-transition test-suites, + and handle different kinds of epoch sub-transitions by processing the cases using the specified test handler. + +Information on the format of the tests can be found in the [epoch-processing test formats documentation](../../specs/test_formats/epoch_processing/README.md). + + + diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py new file mode 100644 index 000000000..ea69efd64 --- /dev/null +++ b/test_generators/epoch_processing/main.py @@ -0,0 +1,38 @@ +from eth2spec.test.epoch_processing import ( + test_process_crosslinks, + test_process_registry_updates +) + +from gen_base import gen_runner, gen_suite, gen_typing +from gen_from_tests.gen import generate_from_tests +from typing import Callable, Iterable +from preset_loader import loader +from eth2spec.phase0 import spec + + +def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ + -> Callable[[str], gen_typing.TestSuiteOutput]: + def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + return ("%s_%s" % (transition_name, config_name), transition_name, gen_suite.render_suite( + title="%s epoch processing" % transition_name, + summary="Test suite for %s type epoch processing" % transition_name, + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="epoch_processing", + handler=transition_name, + test_cases=get_cases())) + return suite_definition + + +if __name__ == "__main__": + gen_runner.run_generator("epoch_processing", [ + create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), + # To be updated to support mainnet config. + # create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), + create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)), + create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)), + ]) diff --git a/test_generators/epoch_processing/requirements.txt b/test_generators/epoch_processing/requirements.txt new file mode 100644 index 000000000..595cee69c --- /dev/null +++ b/test_generators/epoch_processing/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/operations/suite_creator.py b/test_generators/operations/suite_creator.py deleted file mode 100644 index caff0c7db..000000000 --- a/test_generators/operations/suite_creator.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Callable, Iterable -from inspect import getmembers, isfunction -from gen_base import gen_suite, gen_typing -from preset_loader import loader -from eth2spec.phase0 import spec - - -def generate_from_tests(pkg): - fn_names = [ - name for (name, _) in getmembers(pkg, isfunction) - if name.startswith('test_') - ] - out = [] - print("generating test vectors from tests package: %s" % pkg.__name__) - for name in fn_names: - tfn = getattr(pkg, name) - try: - out.append(tfn(generator_mode=True, bls_active=True)) - except AssertionError: - print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__)) - return out - - -def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]])\ - -> Callable[[str], gen_typing.TestSuiteOutput]: - def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: - presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) - - return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite( - title="%s operation" % operation_name, - summary="Test suite for %s type operation processing" % operation_name, - forks_timeline="testing", - forks=["phase0"], - config=config_name, - runner="operations", - handler=operation_name, - test_cases=get_cases())) - return suite_definition diff --git a/test_libs/gen_helpers/gen_from_tests/__init__.py b/test_libs/gen_helpers/gen_from_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py new file mode 100644 index 000000000..1b9d60f7e --- /dev/null +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -0,0 +1,22 @@ +from inspect import getmembers, isfunction + +def generate_from_tests(src, bls_active=True): + """ + Generate a list of test cases by running tests from the given src in generator-mode. + :param src: to retrieve tests from (discovered using inspect.getmembers) + :param bls_active: optional, to override BLS switch preference. Defaults to True. + :return: the list of test cases. + """ + fn_names = [ + name for (name, _) in getmembers(src, isfunction) + if name.startswith('test_') + ] + out = [] + print("generating test vectors from tests source: %s" % src.__name__) + for name in fn_names: + tfn = getattr(src, name) + try: + out.append(tfn(generator_mode=True, bls_active=bls_active)) + except AssertionError: + print("ERROR: failed to generate vector from test: %s (src: %s)" % (name, src.__name__)) + return out diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 29cf04fd1..ee2c815c7 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup setup( name='gen_helpers', - packages=['gen_base'], + packages=['gen_base', 'gen_from_tests'], install_requires=[ "ruamel.yaml==0.15.96", "eth-utils==1.6.0" diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 2086f4ef2..e11a5be2d 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -4,6 +4,7 @@ from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, ) +from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test @@ -19,15 +20,13 @@ def test_activation(state): state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE assert not is_active_validator(state.validator_registry[index], get_current_epoch(state)) + for _ in range(spec.ACTIVATION_EXIT_DELAY): + next_epoch(state) + yield 'pre', state - blocks = [] - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): - block = next_epoch(state) - blocks.append(block) - - # provide extra type hinting here, since it is wrapped in a list. - yield 'blocks', blocks, [spec.BeaconBlock] + next_epoch(state) + yield 'trigger_block', apply_empty_block(state) yield 'post', state @@ -48,15 +47,13 @@ def test_ejection(state): # Mock an ejection state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE + for _ in range(spec.ACTIVATION_EXIT_DELAY): + next_epoch(state) + yield 'pre', state - blocks = [] - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): - block = next_epoch(state) - blocks.append(block) - - # provide extra type hinting here, since it is wrapped in a list. - yield 'blocks', blocks, [spec.BeaconBlock] + next_epoch(state) + yield 'trigger_block', apply_empty_block(state) yield 'post', state From c11f963bc99535d7a826545891b833502ea7c50e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 15:16:59 +0200 Subject: [PATCH 081/118] cleanup generator code, use helper pkg to load and generate test cases with --- test_generators/epoch_processing/main.py | 3 ++- test_generators/operations/main.py | 26 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index ea69efd64..530f9485d 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -1,3 +1,5 @@ +from typing import Callable, Iterable + from eth2spec.test.epoch_processing import ( test_process_crosslinks, test_process_registry_updates @@ -5,7 +7,6 @@ from eth2spec.test.epoch_processing import ( from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests -from typing import Callable, Iterable from preset_loader import loader from eth2spec.phase0 import spec diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 41a6ff806..96c639d12 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,3 +1,5 @@ +from typing import Callable, Iterable + from eth2spec.test.block_processing import ( test_process_attestation, test_process_attester_slashing, @@ -8,9 +10,29 @@ from eth2spec.test.block_processing import ( test_process_voluntary_exit ) -from gen_base import gen_runner +from gen_base import gen_runner, gen_suite, gen_typing +from gen_from_tests.gen import generate_from_tests +from preset_loader import loader +from eth2spec.phase0 import spec + + +def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ + -> Callable[[str], gen_typing.TestSuiteOutput]: + def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite( + title="%s operation" % operation_name, + summary="Test suite for %s type operation processing" % operation_name, + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="operations", + handler=operation_name, + test_cases=get_cases())) + return suite_definition -from suite_creator import generate_from_tests, create_suite if __name__ == "__main__": gen_runner.run_generator("operations", [ From e1b04f49261b8052866fecb0dc0a27f9c80b49a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 15:53:11 +0200 Subject: [PATCH 082/118] sanity tests generator --- test_generators/sanity/README.md | 8 +++++++ test_generators/sanity/main.py | 32 +++++++++++++++++++++++++ test_generators/sanity/requirements.txt | 4 ++++ 3 files changed, 44 insertions(+) create mode 100644 test_generators/sanity/README.md create mode 100644 test_generators/sanity/main.py create mode 100644 test_generators/sanity/requirements.txt diff --git a/test_generators/sanity/README.md b/test_generators/sanity/README.md new file mode 100644 index 000000000..6d2e2f30d --- /dev/null +++ b/test_generators/sanity/README.md @@ -0,0 +1,8 @@ +# Sanity tests + +Sanity tests cover regular state-transitions in a common block-list format, to ensure the basics work. + +Information on the format of the tests can be found in the [sanity test formats documentation](../../specs/test_formats/sanity/README.md). + + + diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py new file mode 100644 index 000000000..887d0eb06 --- /dev/null +++ b/test_generators/sanity/main.py @@ -0,0 +1,32 @@ +from typing import Callable, Iterable + +from eth2spec.test import test_sanity + +from gen_base import gen_runner, gen_suite, gen_typing +from gen_from_tests.gen import generate_from_tests +from preset_loader import loader +from eth2spec.phase0 import spec + + +def create_suite(config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ + -> Callable[[str], gen_typing.TestSuiteOutput]: + def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + return ("%s_sanity" % config_name, "core", gen_suite.render_suite( + title="sanity testing", + summary="Sanity test suite, generated from pytests", + forks_timeline="testing", + forks=["phase0"], + config=config_name, + runner="sanity", + handler="core", + test_cases=get_cases())) + return suite_definition + + +if __name__ == "__main__": + gen_runner.run_generator("sanity", [ + create_suite('minimal', lambda: generate_from_tests(test_sanity)), + ]) diff --git a/test_generators/sanity/requirements.txt b/test_generators/sanity/requirements.txt new file mode 100644 index 000000000..595cee69c --- /dev/null +++ b/test_generators/sanity/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file From 3500bde5947a6f12d9d0dd1018bfe92af5a6636f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 23 May 2019 11:01:07 -0600 Subject: [PATCH 083/118] only sign in test_double_late_crosslink when necessary --- .../test/epoch_processing/test_process_crosslinks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 688bb54ac..4783c11d1 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -21,6 +21,7 @@ from eth2spec.test.helpers.attestations import ( fill_aggregate_attestation, get_crosslink_committee, get_valid_attestation, + sign_attestation, ) @@ -109,13 +110,14 @@ def test_double_late_crosslink(state): attestation_1 = get_valid_attestation(state, signed=True) fill_aggregate_attestation(state, attestation_1) - # add attestation_1 in the next epoch + # add attestation_1 to next epoch next_epoch(state) add_attestation_to_state(state, attestation_1, state.slot + 1) for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = get_valid_attestation(state, signed=True) + attestation_2 = get_valid_attestation(state, signed=False) if attestation_2.data.shard == attestation_1.data.shard: + sign_attestation(state, attestation_2) break next_slot(state) apply_empty_block(state) From 7b30c55cd4a774af76d53013c1c5e0b8f63a1604 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 23 May 2019 13:10:34 -0400 Subject: [PATCH 084/118] minor copy edits to vc api --- specs/validator/0_beacon-node-validator-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index ec0300e9a..afecc395c 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -15,7 +15,7 @@ The validator client is a conceptually separate entity which utilizes private ke Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator clients independet of beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client plugging into the same API. This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) From f0c9e7a3956c81cd8a9823887cd071609ddddf97 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 22:26:36 +0200 Subject: [PATCH 085/118] ignore just the one crosslinks case that is incompatible with mainnet --- test_generators/epoch_processing/main.py | 3 +-- test_libs/gen_helpers/gen_from_tests/gen.py | 5 ++++- .../test/epoch_processing/test_process_crosslinks.py | 4 ++++ test_libs/pyspec/eth2spec/test/utils.py | 8 +++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 530f9485d..14df53aba 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -32,8 +32,7 @@ def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), - # To be updated to support mainnet config. - # create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), + create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)), create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)), ]) diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py index 1b9d60f7e..d954e0485 100644 --- a/test_libs/gen_helpers/gen_from_tests/gen.py +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -16,7 +16,10 @@ def generate_from_tests(src, bls_active=True): for name in fn_names: tfn = getattr(src, name) try: - out.append(tfn(generator_mode=True, bls_active=bls_active)) + test_case = tfn(generator_mode=True, bls_active=bls_active)) + # If no test case data is returned, the test is ignored. + if test_case is not None: + out.append(test_case) except AssertionError: print("ERROR: failed to generate vector from test: %s (src: %s)" % (name, src.__name__)) return out diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 4783c11d1..58c7b669f 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -104,6 +104,10 @@ def test_single_crosslink_update_from_previous_epoch(state): @spec_state_test def test_double_late_crosslink(state): + if spec.SLOTS_PER_EPOCH < spec.SHARD_COUNT: + print("warning: ignoring test, test-assumptions are incompatible with configuration") + return + next_epoch(state) state.slot += 4 diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index c1d424109..02bc9a1e6 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -19,8 +19,10 @@ def spectest(description: str = None): else: # description can be explicit out['description'] = description + has_contents = False # put all generated data into a dict. for data in fn(*args, **kw): + has_contents = True # If there is a type argument, encode it as that type. if len(data) == 3: (key, value, typ) = data @@ -32,11 +34,15 @@ def spectest(description: str = None): out[key] = encode(value, value.__class__) else: out[key] = value - return out + if has_contents: + return out + else: + return None else: # just complete the function, ignore all yielded data, we are not using it for _ in fn(*args, **kw): continue + return None return entry return runner From 1bbab9aa005f527c80ac793ae7639aed6e13672e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 22:28:11 +0200 Subject: [PATCH 086/118] more direct in what is happening in test utils --- test_libs/pyspec/eth2spec/test/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index 02bc9a1e6..b61801c3d 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -60,7 +60,7 @@ def with_tags(tags: Dict[str, Any]): fn_out = fn(*args, **kw) # do not add tags if the function is not returning a dict at all (i.e. not in generator mode) if fn_out is None: - return fn_out + return None return {**tags, **fn_out} return entry return runner From 21c48b574fe61791cd1ff29d38e2610292e9d039 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 23:32:21 +0200 Subject: [PATCH 087/118] move sanity tests, separate slot tests --- test_generators/sanity/main.py | 15 +++-- .../pyspec/eth2spec/test/sanity/__init__.py | 0 .../{test_sanity.py => sanity/test_blocks.py} | 40 +++---------- .../pyspec/eth2spec/test/sanity/test_slots.py | 58 +++++++++++++++++++ 4 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/sanity/__init__.py rename test_libs/pyspec/eth2spec/test/{test_sanity.py => sanity/test_blocks.py} (93%) create mode 100644 test_libs/pyspec/eth2spec/test/sanity/test_slots.py diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index 887d0eb06..bba6ed03d 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -1,6 +1,6 @@ from typing import Callable, Iterable -from eth2spec.test import test_sanity +from eth2spec.test.sanity import test_blocks, test_slots from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests @@ -8,25 +8,28 @@ from preset_loader import loader from eth2spec.phase0 import spec -def create_suite(config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ +def create_suite(handler_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ -> Callable[[str], gen_typing.TestSuiteOutput]: def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, config_name) spec.apply_constants_preset(presets) - return ("%s_sanity" % config_name, "core", gen_suite.render_suite( + return ("%sanity_s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( title="sanity testing", - summary="Sanity test suite, generated from pytests", + summary="Sanity test suite, %s type, generated from pytests" % handler_name, forks_timeline="testing", forks=["phase0"], config=config_name, runner="sanity", - handler="core", + handler=handler_name, test_cases=get_cases())) return suite_definition if __name__ == "__main__": gen_runner.run_generator("sanity", [ - create_suite('minimal', lambda: generate_from_tests(test_sanity)), + create_suite('blocks', 'minimal', lambda: generate_from_tests(test_blocks)), + create_suite('blocks', 'mainnet', lambda: generate_from_tests(test_blocks)), + create_suite('slots', 'minimal', lambda: generate_from_tests(test_slots)), + create_suite('slots', 'mainnet', lambda: generate_from_tests(test_slots)), ]) diff --git a/test_libs/pyspec/eth2spec/test/sanity/__init__.py b/test_libs/pyspec/eth2spec/test/sanity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_libs/pyspec/eth2spec/test/test_sanity.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py similarity index 93% rename from test_libs/pyspec/eth2spec/test/test_sanity.py rename to test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 6d65cc7f4..01ffa304e 100644 --- a/test_libs/pyspec/eth2spec/test/test_sanity.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -13,42 +13,20 @@ from eth2spec.phase0.spec import ( get_block_root_at_slot, get_current_epoch, get_domain, - advance_slot, - cache_state, ) from eth2spec.phase0.state_transition import ( state_transition, ) -from .helpers.state import ( - get_balance, - get_state_root -) -from .helpers.transfers import get_valid_transfer -from .helpers.block import build_empty_block_for_next_slot, sign_block -from .helpers.keys import ( - privkeys, - pubkeys, -) -from .helpers.attester_slashings import get_valid_attester_slashing -from .helpers.proposer_slashings import get_valid_proposer_slashing -from .helpers.attestations import get_valid_attestation -from .helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.state import get_balance +from eth2spec.test.helpers.transfers import get_valid_transfer +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing +from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from .context import spec_state_test, never_bls - - -@spec_state_test -def test_slot_transition(state): - pre_slot = state.slot - pre_root = state.hash_tree_root() - yield 'pre', state - - cache_state(state) - advance_slot(state) - yield 'post', state - - assert state.slot == pre_slot + 1 - assert get_state_root(state, pre_slot) == pre_root +from eth2spec.test.context import spec_state_test, never_bls @never_bls diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py new file mode 100644 index 000000000..2e5f3a5df --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py @@ -0,0 +1,58 @@ +import eth2spec.phase0.spec as spec + +from eth2spec.phase0.state_transition import state_transition_to +from eth2spec.test.helpers.state import get_state_root +from eth2spec.test.context import spec_state_test + + +@spec_state_test +def test_slots_1(state): + pre_slot = state.slot + pre_root = state.hash_tree_root() + yield 'pre', state + + slots = 1 + yield 'slots', slots + state_transition_to(state, state.slot + slots) + + yield 'post', state + assert state.slot == pre_slot + 1 + assert get_state_root(state, pre_slot) == pre_root + + +@spec_state_test +def test_slots_2(state): + yield 'pre', state + slots = 2 + yield 'slots', slots + state_transition_to(state, state.slot + slots) + yield 'post', state + + +@spec_state_test +def test_empty_epoch(state): + yield 'pre', state + slots = spec.SLOTS_PER_EPOCH + yield 'slots', slots + state_transition_to(state, state.slot + slots) + yield 'post', state + + +@spec_state_test +def test_double_empty_epoch(state): + yield 'pre', state + slots = spec.SLOTS_PER_EPOCH * 2 + yield 'slots', slots + state_transition_to(state, state.slot + slots) + yield 'post', state + + +@spec_state_test +def test_over_epoch_boundary(state): + state_transition_to(state, state.slot + (spec.SLOTS_PER_EPOCH // 2)) + yield 'pre', state + slots = spec.SLOTS_PER_EPOCH + yield 'slots', slots + state_transition_to(state, state.slot + slots) + yield 'post', state + From f98a8d534e02e20ea5cff5d57441f60575f1ef87 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 23 May 2019 23:50:58 +0200 Subject: [PATCH 088/118] update epoch processing tests to conform to processing pattern, add docs for epoch sub-transition testing --- specs/test_formats/epoch_processing/README.md | 34 +++++++++++++++++++ .../test_process_registry_updates.py | 12 +++---- 2 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 specs/test_formats/epoch_processing/README.md diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md new file mode 100644 index 000000000..041cb2fed --- /dev/null +++ b/specs/test_formats/epoch_processing/README.md @@ -0,0 +1,34 @@ +# Epoch processing tests + +The different epoch sub-transitions are tested individually with test handlers. +The format is similar to block-processing state-transition tests. +There is no "change" factor however, the transitions are a pure functions with just the pre-state as input. +Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) + +## Test case format + +```yaml +description: string -- description of test case, purely for debugging purposes +bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. +bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +pre: BeaconState -- state before running the sub-transition +post: BeaconState -- state after applying the epoch sub-transition. +``` + +Note: if both `bls_required` and `bls_ignored` are false (or simply not included), + then the test consumer can freely choose to run with BLS ON or OFF. +One may choose for OFF for performance reasons during repeated testing. Otherwise it is recommended to run with BLS ON. + +## Condition + +A handler of the `epoch_processing` test-runner should process these cases, + calling the corresponding processing implementation. + +Sub-transitions: + +| *`sub-transition-name`* | *`processing call`* | +|-------------------------|-----------------------------------| +| `crosslinks` | `process_crosslinks(state)` | +| `registry_updates` | `process_registry_updates(state)` | + +The resulting state should match the expected `post` state. diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index e11a5be2d..05f40218e 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -3,8 +3,8 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, + process_registry_updates ) -from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test @@ -20,13 +20,12 @@ def test_activation(state): state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE assert not is_active_validator(state.validator_registry[index], get_current_epoch(state)) - for _ in range(spec.ACTIVATION_EXIT_DELAY): + for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(state) yield 'pre', state - next_epoch(state) - yield 'trigger_block', apply_empty_block(state) + process_registry_updates(state) yield 'post', state @@ -47,13 +46,12 @@ def test_ejection(state): # Mock an ejection state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE - for _ in range(spec.ACTIVATION_EXIT_DELAY): + for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(state) yield 'pre', state - next_epoch(state) - yield 'trigger_block', apply_empty_block(state) + process_registry_updates(state) yield 'post', state From 902059d6d816a9ced4e60a87402427c4e84d6a36 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 00:13:49 +0200 Subject: [PATCH 089/118] fix operations readme --- specs/test_formats/operations/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index a99a70788..149d54501 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -8,9 +8,9 @@ The different kinds of operations ("transactions") are tested individually with description: string -- description of test case, purely for debugging purposes bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. -pre: BeaconState -- state before applying the deposit +pre: BeaconState -- state before applying the operation : -- the YAML encoded operation, e.g. a "ProposerSlashing", or "Deposit". -post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. +post: BeaconState -- state after applying the operation. No value if operation processing is aborted. ``` Note: if both `bls_required` and `bls_ignored` are false (or simply not included), @@ -19,7 +19,7 @@ One may choose for OFF for performance reasons during repeated testing. Otherwis ## Condition -A handler of the `operations` should process these cases, +A handler of the `operations` test-runner should process these cases, calling the corresponding processing implementation. Operations: From 4ccd304603c351a6ca2c8fd2e3ef33d795f4bc05 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 01:03:21 +0200 Subject: [PATCH 090/118] docs for sanity tests --- specs/test_formats/sanity/README.md | 7 +++++++ specs/test_formats/sanity/blocks.md | 19 +++++++++++++++++++ specs/test_formats/sanity/slots.md | 24 ++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 specs/test_formats/sanity/README.md create mode 100644 specs/test_formats/sanity/blocks.md create mode 100644 specs/test_formats/sanity/slots.md diff --git a/specs/test_formats/sanity/README.md b/specs/test_formats/sanity/README.md new file mode 100644 index 000000000..20b36208a --- /dev/null +++ b/specs/test_formats/sanity/README.md @@ -0,0 +1,7 @@ +# Sanity tests + +The aim of the sanity tests is to set a base-line on what really needs to pass, i.e. the essentials. + +There are two handlers, documented individually: +- [`slots`](./slots.md): transitions of one or more slots (and epoch transitions within) +- [`blocks`](./blocks.md): transitions triggered by one or more blocks diff --git a/specs/test_formats/sanity/blocks.md b/specs/test_formats/sanity/blocks.md new file mode 100644 index 000000000..696813740 --- /dev/null +++ b/specs/test_formats/sanity/blocks.md @@ -0,0 +1,19 @@ +# Sanity blocks testing + +Sanity tests to cover a series of one or more blocks being processed, aiming to cover common changes. + +## Test case format + +```yaml +description: string -- description of test case, purely for debugging purposes +bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. +bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +pre: BeaconState -- state before running through the transitions triggered by the blocks. +blocks: [BeaconBlock] -- blocks to process, in given order, following the main transition function (i.e. process slot and epoch transitions in between blocks as normal) +post: BeaconState -- state after applying all the transitions triggered by the blocks. +``` + +## Condition + +The resulting state should match the expected `post` state, or if the `post` state is left blank, + the handler should reject the series of blocks as invalid. diff --git a/specs/test_formats/sanity/slots.md b/specs/test_formats/sanity/slots.md new file mode 100644 index 000000000..e8961608d --- /dev/null +++ b/specs/test_formats/sanity/slots.md @@ -0,0 +1,24 @@ +# Sanity slots testing + +Sanity tests to cover a series of one or more empty-slot transitions being processed, aiming to cover common changes. + +## Test case format + +```yaml +description: string -- description of test case, purely for debugging purposes +bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. +bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +pre: BeaconState -- state before running through the transitions. +slots: N -- amount of slots to process, N being a positive numer. +post: BeaconState -- state after applying all the transitions. +``` + +The transition with pure time, no blocks, is known as `state_transition_to(state, slot)` in the spec. +This runs state-caching (pure slot transition) and epoch processing (every E slots). + +To process the data, call `state_transition_to(pre, pre.slot + N)`. And see if `pre` mutated into the equivalent of `post`. + + +## Condition + +The resulting state should match the expected `post` state. From 57dd9fc4fffd9e9675ce35885423cad53199c8c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 01:15:49 +0200 Subject: [PATCH 091/118] fix syntax --- test_libs/gen_helpers/gen_from_tests/gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py index d954e0485..e7d801131 100644 --- a/test_libs/gen_helpers/gen_from_tests/gen.py +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -16,7 +16,7 @@ def generate_from_tests(src, bls_active=True): for name in fn_names: tfn = getattr(src, name) try: - test_case = tfn(generator_mode=True, bls_active=bls_active)) + test_case = tfn(generator_mode=True, bls_active=bls_active) # If no test case data is returned, the test is ignored. if test_case is not None: out.append(test_case) From f52b2282b30e60d664fd18ae14af77c2409f3b6f Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 24 May 2019 08:32:54 -0400 Subject: [PATCH 092/118] Update specs/test_formats/operations/README.md Co-Authored-By: Danny Ryan --- specs/test_formats/operations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 149d54501..0d187abbd 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -27,7 +27,7 @@ Operations: | *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | |-------------------------|----------------------|----------------------|--------------------------------------------------------| | `attestation` | `Attestation` | `attestation` | `process_deposit(state, attestation)` | -| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_deposit(state, attester_slashing)` | +| `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | | `block_header` | `Block` | `block` | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | From 4690bcf682950660b76d28a879a27b90085ec648 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 24 May 2019 08:33:06 -0400 Subject: [PATCH 093/118] Update specs/test_formats/operations/README.md Co-Authored-By: Danny Ryan --- specs/test_formats/operations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 0d187abbd..039e01a2f 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -26,7 +26,7 @@ Operations: | *`operation-name`* | *`operation-object`* | *`input name`* | *`processing call`* | |-------------------------|----------------------|----------------------|--------------------------------------------------------| -| `attestation` | `Attestation` | `attestation` | `process_deposit(state, attestation)` | +| `attestation` | `Attestation` | `attestation` | `process_attestation(state, attestation)` | | `attester_slashing` | `AttesterSlashing` | `attester_slashing` | `process_attester_slashing(state, attester_slashing)` | | `block_header` | `Block` | `block` | `process_block_header(state, block)` | | `deposit` | `Deposit` | `deposit` | `process_deposit(state, deposit)` | From f2e3cd01aad760384d8bda02485cb204a9187ac6 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 24 May 2019 08:34:14 -0400 Subject: [PATCH 094/118] Update test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py Co-Authored-By: Danny Ryan --- .../eth2spec/test/epoch_processing/test_process_crosslinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 58c7b669f..a0cc10251 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -104,7 +104,7 @@ def test_single_crosslink_update_from_previous_epoch(state): @spec_state_test def test_double_late_crosslink(state): - if spec.SLOTS_PER_EPOCH < spec.SHARD_COUNT: + if get_epoch_committee_count(state, get_current_epoch(state)) < spec.SHARD_COUNT: print("warning: ignoring test, test-assumptions are incompatible with configuration") return From 73f0f74fb012fd700e3d265ce5eff571dd262940 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 14:58:06 +0200 Subject: [PATCH 095/118] run process yield-from test pattern --- .../test_process_registry_updates.py | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 05f40218e..fa49561ab 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -5,10 +5,34 @@ from eth2spec.phase0.spec import ( is_active_validator, process_registry_updates ) +from eth2spec.phase0.state_transition import state_transition +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import spec_state_test +def run_process_registry_updates(state, valid=True): + """ + Run ``process_crosslinks``, yielding: + - pre-state ('pre') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # transition state to slot before state transition + slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 + block = build_empty_block_for_next_slot(state, signed=False) + block.slot = slot + sign_block(state, block) + state_transition(state, block) + + # cache state before epoch transition + spec.cache_state(state) + + yield 'pre', state + process_registry_updates(state) + yield 'post', state + + @spec_state_test def test_activation(state): index = 0 @@ -23,11 +47,7 @@ def test_activation(state): for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(state) - yield 'pre', state - - process_registry_updates(state) - - yield 'post', state + yield from run_process_registry_updates(state) assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH @@ -49,11 +69,7 @@ def test_ejection(state): for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(state) - yield 'pre', state - - process_registry_updates(state) - - yield 'post', state + yield from run_process_registry_updates(state) assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH assert not is_active_validator( From 321baf79f4aee96e56fe021ce3e9becc6e775d94 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 15:03:03 +0200 Subject: [PATCH 096/118] fix missing imports from earlier code suggestion --- .../eth2spec/test/epoch_processing/test_process_crosslinks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index a0cc10251..ad0e623be 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -104,7 +104,7 @@ def test_single_crosslink_update_from_previous_epoch(state): @spec_state_test def test_double_late_crosslink(state): - if get_epoch_committee_count(state, get_current_epoch(state)) < spec.SHARD_COUNT: + if spec.get_epoch_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: print("warning: ignoring test, test-assumptions are incompatible with configuration") return From b6b5787931b6adfb0f5bdb2bbce527b0737390df Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 24 May 2019 12:24:42 -0400 Subject: [PATCH 097/118] Update specs/test_formats/epoch_processing/README.md Co-Authored-By: Hsiao-Wei Wang --- specs/test_formats/epoch_processing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md index 041cb2fed..117be89d1 100644 --- a/specs/test_formats/epoch_processing/README.md +++ b/specs/test_formats/epoch_processing/README.md @@ -2,7 +2,7 @@ The different epoch sub-transitions are tested individually with test handlers. The format is similar to block-processing state-transition tests. -There is no "change" factor however, the transitions are a pure functions with just the pre-state as input. +There is no "change" factor however, the transitions are pure functions with just the pre-state as input. Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) ## Test case format From b12031b48d28db38fe81575c40653b1aad15b911 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 18:38:51 +0200 Subject: [PATCH 098/118] not signed by default --- .../test_process_attestation.py | 22 +++++++------- .../test_process_block_header.py | 6 ++-- .../block_processing/test_process_deposit.py | 8 ++--- .../block_processing/test_process_transfer.py | 2 +- .../test_process_voluntary_exit.py | 2 +- .../test_process_crosslinks.py | 4 +-- .../test_process_registry_updates.py | 2 +- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/sanity/test_blocks.py | 30 +++++++++---------- 9 files changed, 39 insertions(+), 39 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py index 0dae852f0..af6b39ef6 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py @@ -75,7 +75,7 @@ def test_success_previous_epoch(state): @always_bls @spec_state_test def test_invalid_attestation_signature(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY yield from run_attestation_processing(state, attestation, False) @@ -105,7 +105,7 @@ def test_old_source_epoch(state): state.finalized_epoch = 2 state.previous_justified_epoch = 3 state.current_justified_epoch = 4 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) # test logic sanity check: make sure the attestation is pointing to oldest known source epoch assert attestation.data.source_epoch == state.previous_justified_epoch @@ -120,7 +120,7 @@ def test_old_source_epoch(state): @spec_state_test def test_wrong_shard(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.shard += 1 @@ -132,7 +132,7 @@ def test_wrong_shard(state): @spec_state_test def test_new_source_epoch(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 1 @@ -144,7 +144,7 @@ def test_new_source_epoch(state): @spec_state_test def test_source_root_is_target_root(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = attestation.data.target_root @@ -165,7 +165,7 @@ def test_invalid_current_source_root(state): state.current_justified_epoch = 4 state.current_justified_root = b'\xff' * 32 - attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1, signed=False) + attestation = get_valid_attestation(state, slot=(spec.SLOTS_PER_EPOCH * 3) + 1) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY # Test logic sanity checks: @@ -182,7 +182,7 @@ def test_invalid_current_source_root(state): @spec_state_test def test_bad_source_root(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 @@ -194,7 +194,7 @@ def test_bad_source_root(state): @spec_state_test def test_non_zero_crosslink_data_root(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink_data_root = b'\x42' * 32 @@ -221,7 +221,7 @@ def test_bad_previous_crosslink(state): @spec_state_test def test_inconsistent_bitfields(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' @@ -233,7 +233,7 @@ def test_inconsistent_bitfields(state): @spec_state_test def test_non_empty_custody_bitfield(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) @@ -245,7 +245,7 @@ def test_non_empty_custody_bitfield(state): @spec_state_test def test_empty_aggregation_bitfield(state): - attestation = get_valid_attestation(state, signed=False) + attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py index 28e215a3a..454f557c5 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py @@ -50,13 +50,13 @@ def test_success_block_header(state): @always_bls @spec_state_test def test_invalid_sig_block_header(state): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) yield from run_block_header_processing(state, block, valid=False) @spec_state_test def test_invalid_slot_block_header(state): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot = state.slot + 2 # invalid slot sign_block(state, block) @@ -65,7 +65,7 @@ def test_invalid_slot_block_header(state): @spec_state_test def test_invalid_previous_block_root(state): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.previous_block_root = b'\12' * 32 # invalid prev root sign_block(state, block) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index b520c809f..336af3bf7 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -69,7 +69,7 @@ def test_invalid_sig_new_deposit(state): # fresh deposit = next validator index = validator appended to registry validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + deposit = prepare_state_and_deposit(state, validator_index, amount) yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=False) @@ -87,7 +87,7 @@ def test_success_top_up(state): def test_invalid_sig_top_up(state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + deposit = prepare_state_and_deposit(state, validator_index, amount) # invalid signatures, in top-ups, are allowed! yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) @@ -97,7 +97,7 @@ def test_invalid_sig_top_up(state): def test_wrong_index(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + deposit = prepare_state_and_deposit(state, validator_index, amount) # mess up deposit_index deposit.index = state.deposit_index + 1 @@ -114,7 +114,7 @@ def test_wrong_index(state): def test_bad_merkle_proof(state): validator_index = len(state.validator_registry) amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(state, validator_index, amount, signed=False) + deposit = prepare_state_and_deposit(state, validator_index, amount) # mess up merkle branch deposit.proof[-1] = spec.ZERO_HASH diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index e5f52e209..83af75574 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -86,7 +86,7 @@ def test_success_active_above_max_effective_fee(state): @always_bls @spec_state_test def test_invalid_signature(state): - transfer = get_valid_transfer(state, signed=False) + transfer = get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py index fe33fb631..53fb4e3f7 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py @@ -62,7 +62,7 @@ def test_invalid_signature(state): validator_index = get_active_validator_indices(state, current_epoch)[0] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey, signed=False) + voluntary_exit = build_voluntary_exit(state, current_epoch, validator_index, privkey) yield from run_voluntary_exit_processing(state, voluntary_exit, False) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index ad0e623be..cfbcd1883 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -34,7 +34,7 @@ def run_process_crosslinks(state, valid=True): """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot = slot sign_block(state, block) state_transition(state, block) @@ -119,7 +119,7 @@ def test_double_late_crosslink(state): add_attestation_to_state(state, attestation_1, state.slot + 1) for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = get_valid_attestation(state, signed=False) + attestation_2 = get_valid_attestation(state) if attestation_2.data.shard == attestation_1.data.shard: sign_attestation(state, attestation_2) break diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index fa49561ab..3f36f7107 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -20,7 +20,7 @@ def run_process_registry_updates(state, valid=True): """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot = slot sign_block(state, block) state_transition(state, block) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index e9b863463..b541e610f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -138,7 +138,7 @@ def fill_aggregate_attestation(state, attestation): def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot = slot block.body.attestations.append(attestation) state_transition_to(state, block.slot) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 01ffa304e..c9aadbf2a 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -53,7 +53,7 @@ def test_skipped_slots(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot += 3 sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -71,7 +71,7 @@ def test_empty_epoch_transition(state): pre_slot = state.slot yield 'pre', state - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -90,7 +90,7 @@ def test_empty_epoch_transition(state): # pre_state = deepcopy(state) # yield 'pre', state # -# block = build_empty_block_for_next_slot(state, signed=False) +# block = build_empty_block_for_next_slot(state) # block.slot += spec.SLOTS_PER_EPOCH * 5 # sign_block(state, block, proposer_index=0) # yield 'blocks', [block], [spec.BeaconBlock] @@ -118,7 +118,7 @@ def test_proposer_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.body.proposer_slashings.append(proposer_slashing) sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -151,7 +151,7 @@ def test_attester_slashing(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.body.attester_slashings.append(attester_slashing) sign_block(state, block) yield 'blocks', [block], [spec.BeaconBlock] @@ -187,7 +187,7 @@ def test_deposit_in_block(state): yield 'pre', state - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.body.deposits.append(deposit) sign_block(state, block) @@ -214,7 +214,7 @@ def test_deposit_top_up(state): yield 'pre', state - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.body.deposits.append(deposit) sign_block(state, block) @@ -238,7 +238,7 @@ def test_attestation(state): # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block_for_next_slot(state, signed=False) + attestation_block = build_empty_block_for_next_slot(state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) sign_block(state, attestation_block) @@ -249,7 +249,7 @@ def test_attestation(state): # Epoch transition should move to previous_epoch_attestations pre_current_attestations_root = spec.hash_tree_root(state.current_epoch_attestations) - epoch_block = build_empty_block_for_next_slot(state, signed=False) + epoch_block = build_empty_block_for_next_slot(state) epoch_block.slot += spec.SLOTS_PER_EPOCH sign_block(state, epoch_block) state_transition(state, epoch_block) @@ -287,7 +287,7 @@ def test_voluntary_exit(state): ) # Add to state via block transition - initiate_exit_block = build_empty_block_for_next_slot(state, signed=False) + initiate_exit_block = build_empty_block_for_next_slot(state) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) sign_block(state, initiate_exit_block) state_transition(state, initiate_exit_block) @@ -295,7 +295,7 @@ def test_voluntary_exit(state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH # Process within epoch transition - exit_block = build_empty_block_for_next_slot(state, signed=False) + exit_block = build_empty_block_for_next_slot(state) exit_block.slot += spec.SLOTS_PER_EPOCH sign_block(state, exit_block) state_transition(state, exit_block) @@ -324,7 +324,7 @@ def test_transfer(state): yield 'pre', state # Add to state via block transition - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.body.transfers.append(transfer) sign_block(state, block) @@ -352,7 +352,7 @@ def test_balance_driven_status_transitions(state): yield 'pre', state # trigger epoch transition - block = build_empty_block_for_next_slot(state, signed=False) + block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH sign_block(state, block) state_transition(state, block) @@ -390,13 +390,13 @@ def test_historical_batch(state): # # blocks = [] # for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): -# block = build_empty_block_for_next_slot(state, signed=False) +# block = build_empty_block_for_next_slot(state) # state_transition(state, block) # expected_votes += 1 # assert len(state.eth1_data_votes) == expected_votes # blocks.append(block) # -# block = build_empty_block_for_next_slot(state, signed=False) +# block = build_empty_block_for_next_slot(state) # blocks.append(block) # # state_transition(state, block) From e9a01a276d3c6dce26442d081dd4e878edbebd5b Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 24 May 2019 18:39:12 +0200 Subject: [PATCH 099/118] epoch processing formatting --- test_generators/epoch_processing/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 14df53aba..8f067e4a3 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -1,14 +1,13 @@ from typing import Callable, Iterable +from eth2spec.phase0 import spec from eth2spec.test.epoch_processing import ( test_process_crosslinks, test_process_registry_updates ) - from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests from preset_loader import loader -from eth2spec.phase0 import spec def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ @@ -26,13 +25,14 @@ def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], runner="epoch_processing", handler=transition_name, test_cases=get_cases())) + return suite_definition if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), - create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), + create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), + create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)), create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)), ]) From cff9c7fe6791b2dab9938ade1249b545ed2d7b39 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 23 May 2019 15:12:29 -0700 Subject: [PATCH 100/118] update version used in test generators to get SHA-256 hash --- test_generators/bls/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 329a4ce15..6d83bdfb5 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ -py-ecc==1.6.0 +py-ecc==1.7.0 eth-utils==1.6.0 ../../test_libs/gen_helpers From d805fb508446c8b9a6a0299cf5ac585b2597362d Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 25 May 2019 00:35:17 +0300 Subject: [PATCH 101/118] Simplify deposits --- specs/core/0_beacon-chain.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a..48a686735 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -471,8 +471,6 @@ The types are defined topologically to aid in facilitating an executable version { # Branch in the deposit tree 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], - # Index in the deposit tree - 'index': 'uint64', # Data 'data': DepositData, } @@ -1763,12 +1761,11 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit.index, + index=state.deposit_index, root=state.latest_eth1_data.deposit_root, ) # Deposits must be processed in order - assert deposit.index == state.deposit_index state.deposit_index += 1 pubkey = deposit.data.pubkey From edf0b9d05fd77d3c1a1035f4e97252cfc18d9d8e Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sat, 25 May 2019 07:59:35 +1000 Subject: [PATCH 102/118] Addressed some of @djrtwo's suggestions. - Rewording of specification goal paragraph - Clarify get duties description regarding chain reorgs. - Add epoch parameter to get duties, and new error 406 - Block publishing action is clarified around validation, with a new status code 202 - The validator pubkey and PoC bit are passed to produce attestation - Attestation publishing action is clarified around validation, new status code 202 - Rewording of genesis_time, 'block' -> 'slot' - Update Crosslink to latest spec - Added missing signature field to IndexedAttestation --- .../validator/0_beacon-node-validator-api.md | 2 +- specs/validator/beacon_node_oapi.yaml | 69 +++++++++++++------ 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index afecc395c..c53ce37ac 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -15,7 +15,7 @@ The validator client is a conceptually separate entity which utilizes private ke Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator clients independet of beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client plugging into the same API. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client which utilises the same API. This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 900b8eefa..c98fbd048 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -92,7 +92,7 @@ paths: tags: - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronized." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganisation (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganisations are recognised, and to ensure that the beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -103,6 +103,11 @@ paths: items: $ref: '#/components/schemas/pubkey' minItems: 1 + - name: epoch + in: query + required: false + schema: + type: integer responses: 200: description: Success response @@ -114,6 +119,8 @@ paths: $ref: '#/components/schemas/ValidatorDuty' 400: $ref: '#/components/responses/InvalidRequest' + 406: + description: "Duties cannot be provided for the requested epoch." 500: $ref: '#/components/responses/InternalError' 503: @@ -157,7 +164,7 @@ paths: tags: - MinimalSet summary: "Publish a signed block." - description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + description: "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202)" parameters: - name: beacon_block in: query @@ -167,7 +174,9 @@ paths: $ref: '#/components/schemas/BeaconBlock' responses: 200: - $ref: '#/components/responses/Success' + description: "The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The block failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." 400: $ref: '#/components/responses/InvalidRequest' 500: @@ -182,6 +191,18 @@ paths: summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: + - name: validator_pubkey + in: query + required: true + description: "Uniquely identifying which validator this attestation is to be produced for." + schema: + $ref: '#/components/schemas/pubkey' + - name: poc_bit + in: query + required: true + description: "The proof-of-custody bit that is reported by this " + schema: + # Still need to establish a schema for this. - name: slot in: query required: true @@ -210,8 +231,8 @@ paths: post: tags: - MinimalSet - summary: "Published a signed attestation." - description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + summary: "Publish a signed attestation." + description: "Instructs the beacon node to broadcast a newly signed IndexedAttestation object to the intended shard subnet. The beacon node is not required to validate the signed IndexedAttestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" parameters: - name: attestation in: query @@ -221,7 +242,9 @@ paths: $ref: '#/components/schemas/IndexedAttestation' responses: 200: - $ref: '#/components/responses/Success' + description: "The attestation was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The attestation failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." 400: $ref: '#/components/responses/InvalidRequest' 500: @@ -244,17 +267,13 @@ components: genesis_time: type: integer format: uint64 - description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + description: "The genesis_time configured for the beacon node, which is the unix time at which the Eth2.0 chain began." example: 1557716289 ValidatorDuty: type: object properties: validator_pubkey: $ref: '#/components/schemas/pubkey' - committee_index: - type: integer - format: uint64 - description: "The index of the validator in the committee." attestation_slot: type: integer format: uint64 @@ -272,18 +291,18 @@ components: type: object nullable: true properties: - starting_block: + starting_slot: type: integer format: uint64 - description: "The block at which syncing started (will only be reset after the sync reached its head)" - current_block: + description: "The slot at which syncing started (will only be reset after the sync reached its head)" + current_slot: type: integer format: uint64 - description: "The current highest block sync'd by the beacon node." - highest_block: + description: "The most recent slot sync'd by the beacon node." + highest_slot: type: integer format: uint64 - description: "The estimated highest block, or current target block number." + description: "Globally, the estimated most recent slot number, or current target slot number." BeaconBlock: description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." @@ -543,6 +562,12 @@ components: items: type: integer format: uint64 + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `IndexedAttestation`, created by the validator of the attestation." data: $ref: '#/components/schemas/AttestationData' AttestationData: @@ -575,16 +600,20 @@ components: crosslink: title: CrossLink type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." properties: shard: type: integer format: uint64 description: "The shard number." - epoch: + start_epoch: type: integer format: uint64 - description: "The epoch number." + description: "The first epoch which the crosslinking data references." + end_epoch: + type: integer + format: uint64 + description: "The 'end' epoch referred to by the crosslinking data; no data in this Crosslink should refer to the `end_epoch` since it is not included in the crosslinking data interval." parent_root: type: string format: byte From cdfb886c22473f8c145b97e726f6182d68495f57 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 26 May 2019 18:42:37 +0300 Subject: [PATCH 103/118] Avoid divisions by zero Possible fix to avoid four cases of divisions by zero: * `return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH` * `rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance` * `validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance` * `rewards[index] += base_reward * attesting_balance // committee_balance` See also #1107. --- specs/core/0_beacon-chain.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a..104f3eb92 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -963,9 +963,9 @@ def bytes_to_int(data: bytes) -> int: ```python def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: """ - Return the combined effective balance of an array of ``validators``. + Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return sum([state.validator_registry[index].effective_balance for index in indices]) + return max(sum([state.validator_registry[index].effective_balance for index in indices]), 1) ``` ### `get_domain` @@ -1413,10 +1413,9 @@ def process_crosslinks(state: BeaconState) -> None: ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - adjusted_quotient = integer_squareroot(get_total_active_balance(state)) // BASE_REWARD_QUOTIENT - if adjusted_quotient == 0: - return 0 - return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH + total_balance = get_total_active_balance(state) + effective_balance = state.validator_registry[index].effective_balance + return effective_balance * BASE_REWARD_QUOTIENT // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH ``` ```python @@ -1531,10 +1530,9 @@ def process_registry_updates(state: BeaconState) -> None: ```python def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - active_validator_indices = get_active_validator_indices(state, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) + total_balance = get_total_active_balance(state) - # Compute `total_penalties` + # Compute slashed balances in the current epoch total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] total_penalties = total_at_end - total_at_start From af798a3065f2fe293a1823ebe843ada31b11863a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 27 May 2019 11:21:23 +1000 Subject: [PATCH 104/118] Minor updates. - Fixed spelling (and made American English) - Clarified the schema for the new poc_bit field, and description. --- specs/validator/beacon_node_oapi.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index c98fbd048..6dca1a107 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -83,7 +83,7 @@ paths: chain_id: type: integer format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analagous to Eth1.0 JSON-RPC net_version." + description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analogous to Eth1.0 JSON-RPC net_version." 500: $ref: '#/components/responses/InternalError' @@ -92,7 +92,7 @@ paths: tags: - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganisation (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganisations are recognised, and to ensure that the beacon node is properly synchronized." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganization (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganizations are recognized, and to ensure that the beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -200,9 +200,12 @@ paths: - name: poc_bit in: query required: true - description: "The proof-of-custody bit that is reported by this " + description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `IndexedAttestation`." schema: - # Still need to establish a schema for this. + type: integer + format: uint32 + minimum: 0 + maximum: 1 - name: slot in: query required: true From fa177e03553f444c1ba1ae845f23470f9e9d7b1d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 27 May 2019 13:01:36 +1000 Subject: [PATCH 105/118] Bumped API version number to 0.2.0 --- specs/validator/beacon_node_oapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 6dca1a107..940f82137 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1.2" + version: "0.2.0" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" From 8d420c0780a5e9170c53f162b962f5c48c9e4ddd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 May 2019 08:38:37 -0600 Subject: [PATCH 106/118] fix prestate for process registry updates --- .../test/epoch_processing/test_process_registry_updates.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index 3f36f7107..71bf89c70 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -28,6 +28,11 @@ def run_process_registry_updates(state, valid=True): # cache state before epoch transition spec.cache_state(state) + # process components of epoch transition before registry update + spec.process_justification_and_finalization(state) + spec.process_crosslinks(state) + spec.process_rewards_and_penalties(state) + yield 'pre', state process_registry_updates(state) yield 'post', state From 405b34225fd8b9648d7141dd8628719c1287a59f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 May 2019 09:37:30 -0600 Subject: [PATCH 107/118] D be list instead of set in validator eth1 data counting --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d82bf6e56..b9953882d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -224,7 +224,7 @@ epoch_signature = bls_sign( `block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.latest_eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. -* Let `D` be the set of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: +* Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. From 47f3df6d0a6296f950289057f3082c6c95d907be Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 May 2019 16:40:00 -0600 Subject: [PATCH 108/118] fix deposit test for new index handling --- .../block_processing/test_process_deposit.py | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index bbfb390ef..62b058a70 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -83,33 +83,45 @@ def test_success_top_up(state): return pre_state, deposit, post_state -def test_wrong_index(state): +def test_wrong_deposit_for_deposit_count(state): pre_state = deepcopy(state) deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( + # build root for deposit_1 + index_1 = len(deposit_data_leaves) + pubkey_1 = pubkeys[index_1] + privkey_1 = privkeys[index_1] + deposit_1, root_1, deposit_data_leaves = build_deposit( pre_state, deposit_data_leaves, - pubkey, - privkey, + pubkey_1, + privkey_1, + spec.MAX_EFFECTIVE_BALANCE, + ) + deposit_count_1 = len(deposit_data_leaves) + + # build root for deposit_2 + index_2 = len(deposit_data_leaves) + pubkey_2 = pubkeys[index_2] + privkey_2 = privkeys[index_2] + deposit_2, root_2, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey_2, + privkey_2, spec.MAX_EFFECTIVE_BALANCE, ) - # mess up deposit_index - deposit.index = pre_state.deposit_index + 1 - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + # state has root for deposit_2 but is at deposit_count for deposit_1 + pre_state.latest_eth1_data.deposit_root = root_2 + pre_state.latest_eth1_data.deposit_count = deposit_count_1 post_state = deepcopy(pre_state) with pytest.raises(AssertionError): - process_deposit(post_state, deposit) + process_deposit(post_state, deposit_2) - return pre_state, deposit, None + return pre_state, deposit_2, None def test_bad_merkle_proof(state): From 56698602ab325871d703bacf3bb12a392421cbf1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 28 May 2019 10:48:41 +1000 Subject: [PATCH 109/118] Updated all absolute URLs to the eth2.0-specs repo so that they point to the master branch (instead of dev). --- specs/validator/beacon_node_oapi.yaml | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 940f82137..74be21fac 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -308,7 +308,7 @@ components: description: "Globally, the estimated most recent slot number, or current target slot number." BeaconBlock: - description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -316,7 +316,7 @@ components: body: $ref: '#/components/schemas/BeaconBlockBody' BeaconBlockHeader: - description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." + description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -352,7 +352,7 @@ components: description: "The BLS signature of the `BeaconBlock` made by the validator of the block." BeaconBlockBody: type: object - description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." + description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." properties: randao_reveal: type: string @@ -362,7 +362,7 @@ components: eth1_data: title: Eth1Data type: object - description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." properties: deposit_root: type: string @@ -387,7 +387,7 @@ components: items: title: ProposerSlashings type: object - description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." + description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." properties: proposer_index: type: integer @@ -402,7 +402,7 @@ components: items: title: AttesterSlashings type: object - description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." + description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." properties: attestation_1: $ref: '#/components/schemas/IndexedAttestation' @@ -413,7 +413,7 @@ components: items: title: Attestation type: object - description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." + description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: aggregation_bitfield: type: string @@ -437,7 +437,7 @@ components: items: title: Deposit type: object - description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." + description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." properties: proof: type: array @@ -455,7 +455,7 @@ components: data: title: DepositData type: object - description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." + description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." properties: pubkey: $ref: '#/components/schemas/pubkey' @@ -478,7 +478,7 @@ components: items: title: VoluntaryExit type: object - description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." + description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." properties: epoch: type: integer @@ -498,7 +498,7 @@ components: items: title: Transfer type: object - description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." + description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." properties: sender: type: integer @@ -533,7 +533,7 @@ components: Fork: type: object - description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." + description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." properties: previous_version: type: string @@ -551,7 +551,7 @@ components: description: "Fork epoch number." IndexedAttestation: type: object - description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." + description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: custody_bit_0_indices: type: array @@ -575,7 +575,7 @@ components: $ref: '#/components/schemas/AttestationData' AttestationData: type: object - description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." + description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: type: string @@ -603,7 +603,7 @@ components: crosslink: title: CrossLink type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." properties: shard: type: integer From c32328fdf2d13796343cc5cd33bd1c5a1c9043e7 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 28 May 2019 10:49:40 +1000 Subject: [PATCH 110/118] Fixed swagger URL so that the version number isn't specified (defaults to latest). --- specs/validator/0_beacon-node-validator-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index c53ce37ac..2a5fe7fcd 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -25,4 +25,4 @@ This specification is derived from a proposal and discussion on Issues [#1011](h The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: -[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1) +[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator) From 6ffd41650b11467f9c225b40a2523e1d7112eb3e Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 15:35:00 +0200 Subject: [PATCH 111/118] suggestion from hww, reduce into one bls_setting key --- specs/test_formats/README.md | 12 ++++++++++++ specs/test_formats/epoch_processing/README.md | 3 +-- specs/test_formats/operations/README.md | 3 +-- specs/test_formats/sanity/blocks.md | 3 +-- specs/test_formats/sanity/slots.md | 3 +-- test_libs/pyspec/eth2spec/test/context.py | 4 ++-- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 273659ce9..d245fcfa4 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -176,6 +176,18 @@ To prevent parsing of hundreds of different YAML files to test a specific test t ... <--- more test types ``` +## Common test-case properties + +Some test-case formats share some common key-value pair patterns, and these are documented here: + +``` +bls_setting: int -- optional, can have 3 different values: + 0: (default, applies if key-value pair is absent). Free to choose either BLS ON or OFF. + Tests are generated with valid BLS data in this case, + but there is no change of outcome when running the test if BLS is ON or OFF. + 1: known as "BLS required" - if the test validity is strictly dependent on BLS being ON + 2: known as "BLS ignored" - if the test validity is strictly dependent on BLS being OFF +``` ## Note for implementers diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md index 117be89d1..d3dbfcbd9 100644 --- a/specs/test_formats/epoch_processing/README.md +++ b/specs/test_formats/epoch_processing/README.md @@ -9,8 +9,7 @@ Hence, the format is shared between each test-handler. (See test condition docum ```yaml description: string -- description of test case, purely for debugging purposes -bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. -bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +bls_setting: int -- see general test-format spec. pre: BeaconState -- state before running the sub-transition post: BeaconState -- state after applying the epoch sub-transition. ``` diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 039e01a2f..04e24ddad 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -6,8 +6,7 @@ The different kinds of operations ("transactions") are tested individually with ```yaml description: string -- description of test case, purely for debugging purposes -bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. -bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +bls_setting: int -- see general test-format spec. pre: BeaconState -- state before applying the operation : -- the YAML encoded operation, e.g. a "ProposerSlashing", or "Deposit". post: BeaconState -- state after applying the operation. No value if operation processing is aborted. diff --git a/specs/test_formats/sanity/blocks.md b/specs/test_formats/sanity/blocks.md index 696813740..3004a6de7 100644 --- a/specs/test_formats/sanity/blocks.md +++ b/specs/test_formats/sanity/blocks.md @@ -6,8 +6,7 @@ Sanity tests to cover a series of one or more blocks being processed, aiming to ```yaml description: string -- description of test case, purely for debugging purposes -bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. -bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +bls_setting: int -- see general test-format spec. pre: BeaconState -- state before running through the transitions triggered by the blocks. blocks: [BeaconBlock] -- blocks to process, in given order, following the main transition function (i.e. process slot and epoch transitions in between blocks as normal) post: BeaconState -- state after applying all the transitions triggered by the blocks. diff --git a/specs/test_formats/sanity/slots.md b/specs/test_formats/sanity/slots.md index e8961608d..81866d47b 100644 --- a/specs/test_formats/sanity/slots.md +++ b/specs/test_formats/sanity/slots.md @@ -6,8 +6,7 @@ Sanity tests to cover a series of one or more empty-slot transitions being proce ```yaml description: string -- description of test case, purely for debugging purposes -bls_required: bool -- optional, true if the test validity is strictly dependent on BLS being ON. False otherwise. -bls_ignored: bool -- optional, true if the test validity is strictly dependent on BLS being OFF. False otherwise. +bls_setting: int -- see general test-format spec. pre: BeaconState -- state before running through the transitions. slots: N -- amount of slots to process, N being a positive numer. post: BeaconState -- state after applying all the transitions. diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index a484cc995..2be9322de 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -39,7 +39,7 @@ def expect_assertion_error(fn): # Tags a test to be ignoring BLS for it to pass. -bls_ignored = with_tags({'bls_ignored': True}) +bls_ignored = with_tags({'bls_setting': 2}) def never_bls(fn): @@ -54,7 +54,7 @@ def never_bls(fn): # Tags a test to be requiring BLS for it to pass. -bls_required = with_tags({'bls_required': True}) +bls_required = with_tags({'bls_setting': 1}) def always_bls(fn): From 8737984e19773d11233873713bcc7b21874ba3b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 16:05:25 +0200 Subject: [PATCH 112/118] introduce back assert, modified to allow valid but non-effective calls --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e873aa4ed..9dc052ec4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1248,6 +1248,7 @@ def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root ```python def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot <= slot while state.slot < slot: process_slot(state) # Process epoch on the first slot of the next epoch From d036b5a87a319f463259ecbc3f89b20526b0b849 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 18:02:09 +0200 Subject: [PATCH 113/118] Remove some old comments about BLS setting. Description is shared in main readme now. --- specs/test_formats/epoch_processing/README.md | 4 ---- specs/test_formats/operations/README.md | 4 ---- 2 files changed, 8 deletions(-) diff --git a/specs/test_formats/epoch_processing/README.md b/specs/test_formats/epoch_processing/README.md index d3dbfcbd9..6384a0eda 100644 --- a/specs/test_formats/epoch_processing/README.md +++ b/specs/test_formats/epoch_processing/README.md @@ -14,10 +14,6 @@ pre: BeaconState -- state before running the sub-transition post: BeaconState -- state after applying the epoch sub-transition. ``` -Note: if both `bls_required` and `bls_ignored` are false (or simply not included), - then the test consumer can freely choose to run with BLS ON or OFF. -One may choose for OFF for performance reasons during repeated testing. Otherwise it is recommended to run with BLS ON. - ## Condition A handler of the `epoch_processing` test-runner should process these cases, diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 04e24ddad..32cf880b3 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -12,10 +12,6 @@ pre: BeaconState -- state before applying the operation post: BeaconState -- state after applying the operation. No value if operation processing is aborted. ``` -Note: if both `bls_required` and `bls_ignored` are false (or simply not included), - then the test consumer can freely choose to run with BLS ON or OFF. -One may choose for OFF for performance reasons during repeated testing. Otherwise it is recommended to run with BLS ON. - ## Condition A handler of the `operations` test-runner should process these cases, From 3781614f201bfadee08697121d54e906b76f8814 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 18:28:37 +0200 Subject: [PATCH 114/118] SSZ format update, to facilitate more efficient parsing --- specs/test_formats/ssz_static/core.md | 10 +++++----- test_generators/ssz_static/main.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/specs/test_formats/ssz_static/core.md b/specs/test_formats/ssz_static/core.md index 1d470c338..0f26e0f9c 100644 --- a/specs/test_formats/ssz_static/core.md +++ b/specs/test_formats/ssz_static/core.md @@ -9,11 +9,11 @@ This test-format ensures these direct serializations are covered. ## Test case format ```yaml -type_name: string -- string, object name, formatted as in spec. E.g. "BeaconBlock" -value: dynamic -- the YAML-encoded value, of the type specified by type_name. -serialized: bytes -- string, SSZ-serialized data, hex encoded, with prefix 0x -root: bytes32 -- string, hash-tree-root of the value, hex encoded, with prefix 0x -signing_root: bytes32 -- string, signing-root of the value, hex encoded, with prefix 0x. Optional, present if type contains ``signature`` field +SomeObjectName: -- key, object name, formatted as in spec. E.g. "BeaconBlock". + value: dynamic -- the YAML-encoded value, of the type specified by type_name. + serialized: bytes -- string, SSZ-serialized data, hex encoded, with prefix 0x + root: bytes32 -- string, hash-tree-root of the value, hex encoded, with prefix 0x + signing_root: bytes32 -- string, signing-root of the value, hex encoded, with prefix 0x. Optional, present if type contains ``signature`` field ``` ## Condition diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 1234294db..e8995b918 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -18,10 +18,7 @@ MAX_LIST_LENGTH = 10 @to_dict -def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): - typ = spec.get_ssz_type_by_name(name) - value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) - yield "type_name", name +def create_test_case_contents(value, typ): yield "value", encode.encode(value, typ) yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() @@ -29,6 +26,13 @@ def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMod yield "signing_root", '0x' + signing_root(value).hex() +@to_dict +def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): + typ = spec.get_ssz_type_by_name(name) + value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) + yield name, create_test_case_contents(value, typ) + + @to_tuple def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int): for type_name in spec.ssz_types: From 12a7e264532800670d12c31e08dc2d0d69f47b31 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 28 May 2019 20:57:18 -0700 Subject: [PATCH 115/118] Update README.md --- specs/test_formats/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index d245fcfa4..f0acb444e 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -25,7 +25,8 @@ Test formats: - [`bls`](./bls/README.md) - [`operations`](./operations/README.md) - [`shuffling`](./shuffling/README.md) -- [`ssz`](./ssz/README.md) +- [`ssz_generic`](./ssz_generic/README.md) +- [`ssz_static`](./ssz_static/README.md) - More formats are planned, see tracking issues for CI/testing ## Glossary From 1c416541e1478af60ee3695f263f6499275a374c Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 29 May 2019 23:40:46 +0300 Subject: [PATCH 116/118] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 104f3eb92..b394d59e1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -17,7 +17,7 @@ - [Initial values](#initial-values) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - - [Reward and penalty quotients](#reward-and-penalty-quotients) + - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - [Data structures](#data-structures) @@ -103,7 +103,7 @@ - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) - [Crosslinks](#crosslinks) - - [Rewards and penalties](#rewards-and-penalties) + - [Rewards and penalties](#rewards-and-penalties-1) - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) @@ -220,17 +220,17 @@ These configurations are updated for releases, but may be out of sync during `de | `LATEST_ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `LATEST_SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -### Reward and penalty quotients +### Rewards and penalties | Name | Value | | - | - | -| `BASE_REWARD_QUOTIENT` | `2**5` (= 32) | +| `BASE_REWARD_FACTOR` | `2**5` (= 32) | | `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* **The `BASE_REWARD_QUOTIENT` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** +* **The `BASE_REWARD_FACTOR` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** * The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -1415,7 +1415,7 @@ def process_crosslinks(state: BeaconState) -> None: def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validator_registry[index].effective_balance - return effective_balance * BASE_REWARD_QUOTIENT // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH + return effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH ``` ```python From 6ec59867ef8eb56329e8ae10437f72edc0f0976d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 09:53:46 +0800 Subject: [PATCH 117/118] Fix linter errors --- scripts/phase0/build_spec.py | 12 ++++++++++-- specs/core/0_beacon-chain.md | 3 ++- .../test/block_processing/test_process_transfer.py | 10 ++++++++-- test_libs/pyspec/eth2spec/test/conftest.py | 1 + test_libs/pyspec/eth2spec/test/helpers/block.py | 1 - test_libs/pyspec/eth2spec/test/sanity/test_slots.py | 1 - 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 36d039c2c..457a13da0 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,9 +13,17 @@ from typing import ( NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import * +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, +) from eth2spec.utils.hash_function import hash -from eth2spec.utils.bls import * +from eth2spec.utils.bls import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) # stub, will get overwritten by real var diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9dc052ec4..a01ba785d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1776,7 +1776,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession). - # Invalid signatures are allowed by the deposit contract, and hence included on-chain, but must not be processed. + # Invalid signatures are allowed by the deposit contract, + # and hence included on-chain, but must not be processed. # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 83af75574..bd435d67a 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -89,7 +89,7 @@ def test_invalid_signature(state): transfer = get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - + yield from run_transfer_processing(state, transfer, False) @@ -140,7 +140,13 @@ def test_insufficient_balance(state): def test_no_dust_sender(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0, signed=True) + transfer = get_valid_transfer( + state, + sender_index=sender_index, + amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, + fee=0, + signed=True, + ) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py index dadb0d5d0..5e8ec708a 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -3,6 +3,7 @@ from eth2spec.phase0 import spec # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. + def module_exists(module_name): try: __import__(module_name) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index c557e0245..715cf82db 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -77,4 +77,3 @@ def build_empty_block(state, slot=None, signed=False): def build_empty_block_for_next_slot(state, signed=False): return build_empty_block(state, state.slot + 1, signed=signed) - diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py index 92e0251ca..4c3897a6c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py @@ -55,4 +55,3 @@ def test_over_epoch_boundary(state): yield 'slots', slots process_slots(state, state.slot + slots) yield 'post', state - From cc5b172da3d981f974e999dc21018bba4f95bcac Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 30 May 2019 12:38:55 +1000 Subject: [PATCH 118/118] Test deposit top-up with inconsistent withdrawal credentials (#1133) * Simplify deposits * Avoid divisions by zero Possible fix to avoid four cases of divisions by zero: * `return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH` * `rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance` * `validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance` * `rewards[index] += base_reward * attesting_balance // committee_balance` See also #1107. * fix deposit test for new index handling * tests: deposit with inconsistent withdrawal credentials * Update README.md * Update 0_beacon-chain.md * Fix linter errors * Update test_process_deposit.py * fix deposit test * fix lint --- .../block_processing/test_process_deposit.py | 18 ++++++++++++++++++ .../pyspec/eth2spec/test/helpers/deposits.py | 16 +++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 0ef2e509a..0430dd12f 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -94,6 +94,22 @@ def test_invalid_sig_top_up(state): yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) +@spec_state_test +def test_invalid_withdrawal_credentials_top_up(state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(b"junk")[1:] + deposit = prepare_state_and_deposit( + state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials + ) + + # inconsistent withdrawal credentials, in top-ups, are allowed! + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) + + @spec_state_test def test_wrong_index(state): validator_index = len(state.validator_registry) @@ -122,6 +138,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_1, privkey_1, spec.MAX_EFFECTIVE_BALANCE, + withdrawal_credentials=b'\x00'*32, signed=True, ) deposit_count_1 = len(deposit_data_leaves) @@ -136,6 +153,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_2, privkey_2, spec.MAX_EFFECTIVE_BALANCE, + withdrawal_credentials=b'\x00'*32, signed=True, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index c5deb124e..2db3ae03c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -8,11 +8,10 @@ from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merk from eth2spec.utils.minimal_ssz import signing_root -def build_deposit_data(state, pubkey, privkey, amount, signed=False): +def build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = DepositData( pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], + withdrawal_credentials=withdrawal_credentials, amount=amount, ) if signed: @@ -37,8 +36,9 @@ def build_deposit(state, pubkey, privkey, amount, + withdrawal_credentials, signed): - deposit_data = build_deposit_data(state, pubkey, privkey, amount, signed) + deposit_data = build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -57,7 +57,7 @@ def build_deposit(state, return deposit, root, deposit_data_leaves -def prepare_state_and_deposit(state, validator_index, amount, signed=False): +def prepare_state_and_deposit(state, validator_index, amount, withdrawal_credentials=None, signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ @@ -67,12 +67,18 @@ def prepare_state_and_deposit(state, validator_index, amount, signed=False): pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] + + # insecurely use pubkey as withdrawal key if no credentials provided + if withdrawal_credentials is None: + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] + deposit, root, deposit_data_leaves = build_deposit( state, deposit_data_leaves, pubkey, privkey, amount, + withdrawal_credentials, signed )