From 94c231cf986a268320d658a8c6dd96c14a0b57e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 26 Jun 2020 01:12:36 +0800 Subject: [PATCH] Fix `ShardTransition.shard_data_roots` and add test 1. Fix `ShardTransition.shard_data_roots`: use `get_block_data_merkle_root` helper to calculate it. 2. Rework `get_valid_custody_chunk_response` testing helper: accept `block_length_or_custody_data` 3. Add `test_with_custody_challenge_and_response` test --- setup.py | 7 ++ specs/phase1/validator.md | 4 +- .../pyspec/eth2spec/test/helpers/custody.py | 21 +++--- .../test_process_chunk_challenge.py | 9 ++- .../test_process_custody_final_updates.py | 3 +- .../test/phase1/sanity/test_blocks.py | 66 +++++++++++++++++++ 6 files changed, 94 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index cc3b00da9..8661f5e52 100644 --- a/setup.py +++ b/setup.py @@ -221,6 +221,13 @@ get_attesting_indices = cache_this( PHASE1_SUNDRY_FUNCTIONS = ''' + +def get_block_data_merkle_root(data: ByteList) -> Root: + # To get the Merkle root of the block data, we need the Merkle root without the length Mixing + # The below implements this in the Remerkleable framework + return data.get_backing().get_left().merkle_root() + + _get_start_shard = get_start_shard get_start_shard = cache_this( lambda state, slot: (state.validators.hash_tree_root(), slot), diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index 130765268..aa5e07f1a 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -275,6 +275,8 @@ Set `attestation_data.shard_head_root = hash_tree_root(shard_head_block)`. Set `shard_transition` to the value returned by `get_shard_transition(head_state, shard, shard_blocks)`. +`get_block_data_merkle_root(data: ByteList) -> Root` is the function that returns the Merkle root of the block data without the length mixing. + ```python def get_shard_transition_fields( beacon_state: BeaconState, @@ -294,7 +296,7 @@ def get_shard_transition_fields( for slot in offset_slots: if slot in shard_block_slots: shard_block = shard_blocks[shard_block_slots.index(slot)] - shard_data_roots.append(hash_tree_root(shard_block.message.body)) + shard_data_roots.append(get_block_data_merkle_root(shard_block.message.body)) else: shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard)) shard_data_roots.append(Root()) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index d25b91a41..d535a2385 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -136,17 +136,16 @@ def build_proof(anchor, leaf_index): return list(reversed(proof)) -def get_block_data_merkle_root(data_as_bytelist): - # To get the Merkle root of the block data, we need the Merkle root without the length Mixing - # The below implements this in the Remerkleable framework - return data_as_bytelist.get_backing().get_left().merkle_root() - - -def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index, +def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_index, + block_length_or_custody_data, invalid_chunk_data=False): - custody_data = get_custody_test_vector(block_length) + if isinstance(block_length_or_custody_data, int): + custody_data = get_custody_test_vector(block_length_or_custody_data) + else: + custody_data = block_length_or_custody_data + custody_data_block = ByteList[spec.MAX_SHARD_BLOCK_SIZE](custody_data) - chunks = custody_chunkify(spec, custody_data) + chunks = custody_chunkify(spec, custody_data_block) chunk_index = chunk_challenge.chunk_index @@ -166,7 +165,7 @@ def get_custody_test_vector(bytelength, offset=0): def get_sample_shard_transition(spec, start_slot, block_lengths): - b = [get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) + b = [spec.get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) for x in block_lengths] shard_transition = spec.ShardTransition( start_slot=start_slot, @@ -201,5 +200,5 @@ def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, cust slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, block_lengths[0], slashable=slashable) block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) - shard_transition.shard_data_roots[0] = get_block_data_merkle_root(block_data) + shard_transition.shard_data_roots[0] = spec.get_block_data_merkle_root(block_data) return shard_transition, slashable_test_vector diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index c9adabb62..4ef1d667a 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -242,7 +242,8 @@ def test_custody_response(spec, state): chunk_challenge_index = state.custody_chunk_challenge_index - 1 - custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3) yield from run_custody_chunk_response_processing(spec, state, custody_response) @@ -270,7 +271,8 @@ def test_custody_response_multiple_epochs(spec, state): chunk_challenge_index = state.custody_chunk_challenge_index - 1 - custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3) yield from run_custody_chunk_response_processing(spec, state, custody_response) @@ -298,6 +300,7 @@ def test_custody_response_many_epochs(spec, state): chunk_challenge_index = state.custody_chunk_challenge_index - 1 - custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3) yield from run_custody_chunk_response_processing(spec, state, custody_response) diff --git a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py index 630ddc3a7..93fea19cf 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py @@ -159,7 +159,8 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state) assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH chunk_challenge_index = state.custody_chunk_challenge_index - 1 - custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3) _, _, _ = run_custody_chunk_response_processing(spec, state, custody_response) diff --git a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py index 2c391a660..2f4c0b67b 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py @@ -7,8 +7,13 @@ from eth2spec.test.context import ( ) from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_valid_custody_chunk_response, +) from eth2spec.test.helpers.shard_block import ( build_shard_block, + get_committee_index_of_shard, get_sample_shard_block_body, get_shard_transitions, ) @@ -16,6 +21,25 @@ from eth2spec.test.helpers.shard_transitions import is_full_crosslink from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to +def run_beacon_block(spec, state, block, valid=True): + yield 'pre', state.copy() + + if not valid: + state_transition_and_sign_block(spec, state, block, expect_fail=True) + yield 'block', block + yield 'post', None + return + + state_transition_and_sign_block(spec, state, block) + yield 'block', block + yield 'post', state + + +# +# Beacon block with non-empty shard transitions +# + + def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): transition_to(spec, state, state.slot + target_len_offset_slot) @@ -102,3 +126,45 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): assert state.shard_states[shard].slot == state.slot - 1 yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) + + +# +# Beacon block with custody operations +# + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_with_custody_challenge_and_response(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + if not is_full_crosslink(spec, state): + # skip + return + + state = transition_to_valid_shard_slot(spec, state) + + # build shard block + shard = 0 + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + body = get_sample_shard_block_body(spec) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + shard_transitions = get_shard_transitions(spec, state, shard_block_dict) + attestation = get_valid_on_time_attestation( + spec, state, index=committee_index, + shard_transition=shard_transitions[shard], signed=True, + ) + + block = build_empty_block(spec, state, slot=state.slot + 1) + block.body.attestations = [attestation] + block.body.shard_transitions = shard_transitions + + # Custody operations + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transitions[shard]) + block.body.chunk_challenges = [challenge] + chunk_challenge_index = state.custody_chunk_challenge_index + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=body) + block.body.chunk_challenge_responses = [custody_response] + + yield from run_beacon_block(spec, state, block)