Merge pull request #1937 from ethereum/hwwhww/custody-block-test

Fix `ShardTransition.shard_data_roots` and add custody game block-level tests

setup.py

@@ -216,11 +216,21 @@ get_matching_head_attestations = cache_this(
 
 _get_attesting_indices = get_attesting_indices
 get_attesting_indices = cache_this(
-    lambda state, data, bits: (state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root()),
+    lambda state, data, bits: (
+        state.randao_mixes.hash_tree_root(),
+        state.validators.hash_tree_root(), data.hash_tree_root(), bits.hash_tree_root()
+    ),
     _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)'''
 
 
 PHASE1_SUNDRY_FUNCTIONS = '''
+
+def get_block_data_merkle_root(data: ByteList) -> Root:
+    # To get the Merkle root of the block data, we need the Merkle root without the length mix-in
+    # The below implements this in the Remerkleable framework
+    return data.get_backing().get_left().merkle_root()
+
+
 _get_start_shard = get_start_shard
 get_start_shard = cache_this(
     lambda state, slot: (state.validators.hash_tree_root(), slot),

specs/phase1/custody-game.md

@@ -25,6 +25,7 @@
     - [`CustodyKeyReveal`](#custodykeyreveal)
     - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal)
 - [Helpers](#helpers)
+  - [`get_block_data_merkle_root`](#get_block_data_merkle_root)
   - [`replace_empty_or_append`](#replace_empty_or_append)
   - [`legendre_bit`](#legendre_bit)
   - [`get_custody_atoms`](#get_custody_atoms)
@@ -182,6 +183,10 @@ class EarlyDerivedSecretReveal(Container):
 
 ## Helpers
 
+### `get_block_data_merkle_root`
+
+`get_block_data_merkle_root(data: ByteList) -> Root` is the function that returns the Merkle root of the block data without the length mix-in.
+
 ### `replace_empty_or_append`
 
 ```python
@@ -515,7 +520,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed
     assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root
     # Verify that the provided data matches the shard-transition
     assert (
-        custody_slashing.data.get_backing().get_left().merkle_root()
+        get_block_data_merkle_root(custody_slashing.data)
         == shard_transition.shard_data_roots[custody_slashing.data_index]
     )
     assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index]

specs/phase1/validator.md

@@ -294,7 +294,7 @@ def get_shard_transition_fields(
     for slot in offset_slots:
         if slot in shard_block_slots:
             shard_block = shard_blocks[shard_block_slots.index(slot)]
-            shard_data_roots.append(hash_tree_root(shard_block.message.body))
+            shard_data_roots.append(get_block_data_merkle_root(shard_block.message.body))
         else:
             shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard))
             shard_data_roots.append(Root())

tests/core/pyspec/eth2spec/test/helpers/custody.py

@@ -136,17 +136,16 @@ def build_proof(anchor, leaf_index):
     return list(reversed(proof))
 
 
-def get_block_data_merkle_root(data_as_bytelist):
-    # To get the Merkle root of the block data, we need the Merkle root without the length Mixing
-    # The below implements this in the Remerkleable framework
-    return data_as_bytelist.get_backing().get_left().merkle_root()
-
-
-def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index,
+def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_index,
+                                     block_length_or_custody_data,
                                      invalid_chunk_data=False):
-    custody_data = get_custody_test_vector(block_length)
+    if isinstance(block_length_or_custody_data, int):
+        custody_data = get_custody_test_vector(block_length_or_custody_data)
+    else:
+        custody_data = block_length_or_custody_data
+
     custody_data_block = ByteList[spec.MAX_SHARD_BLOCK_SIZE](custody_data)
-    chunks = custody_chunkify(spec, custody_data)
+    chunks = custody_chunkify(spec, custody_data_block)
 
     chunk_index = chunk_challenge.chunk_index
 
@@ -166,7 +165,7 @@ def get_custody_test_vector(bytelength, offset=0):
 
 
 def get_sample_shard_transition(spec, start_slot, block_lengths):
-    b = [get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)))
+    b = [spec.get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x)))
          for x in block_lengths]
     shard_transition = spec.ShardTransition(
         start_slot=start_slot,
@@ -192,7 +191,7 @@ def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=Tr
     offset = 0
     while spec.compute_custody_bit(custody_secret, test_vector) != slashable:
         offset += 1
-        test_vector = test_vector = get_custody_test_vector(length, offset)
+        test_vector = get_custody_test_vector(length, offset)
     return test_vector
 
 
@@ -201,5 +200,5 @@ def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, cust
     slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret,
                                                               block_lengths[0], slashable=slashable)
     block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector)
-    shard_transition.shard_data_roots[0] = get_block_data_merkle_root(block_data)
+    shard_transition.shard_data_roots[0] = spec.get_block_data_merkle_root(block_data)
     return shard_transition, slashable_test_vector

tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py

@@ -242,7 +242,8 @@ def test_custody_response(spec, state):
 
     chunk_challenge_index = state.custody_chunk_challenge_index - 1
 
-    custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index)
+    custody_response = get_valid_custody_chunk_response(
+        spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3)
 
     yield from run_custody_chunk_response_processing(spec, state, custody_response)
 
@@ -270,7 +271,8 @@ def test_custody_response_multiple_epochs(spec, state):
 
     chunk_challenge_index = state.custody_chunk_challenge_index - 1
 
-    custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index)
+    custody_response = get_valid_custody_chunk_response(
+        spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3)
 
     yield from run_custody_chunk_response_processing(spec, state, custody_response)
 
@@ -298,6 +300,7 @@ def test_custody_response_many_epochs(spec, state):
 
     chunk_challenge_index = state.custody_chunk_challenge_index - 1
 
-    custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index)
+    custody_response = get_valid_custody_chunk_response(
+        spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3)
 
     yield from run_custody_chunk_response_processing(spec, state, custody_response)

tests/core/pyspec/eth2spec/test/phase1/epoch_processing/test_process_custody_final_updates.py

@@ -159,7 +159,8 @@ def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state)
     assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH
 
     chunk_challenge_index = state.custody_chunk_challenge_index - 1
-    custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index)
+    custody_response = get_valid_custody_chunk_response(
+        spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3)
 
     _, _, _ = run_custody_chunk_response_processing(spec, state, custody_response)
 

tests/core/pyspec/eth2spec/test/phase1/sanity/test_blocks.py

@@ -7,8 +7,18 @@ from eth2spec.test.context import (
 )
 from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import build_empty_block
+from eth2spec.test.helpers.custody import (
+    get_custody_secret,
+    get_custody_slashable_test_vector,
+    get_valid_chunk_challenge,
+    get_valid_custody_chunk_response,
+    get_valid_custody_key_reveal,
+    get_valid_custody_slashing,
+    get_valid_early_derived_secret_reveal,
+)
 from eth2spec.test.helpers.shard_block import (
     build_shard_block,
+    get_committee_index_of_shard,
     get_sample_shard_block_body,
     get_shard_transitions,
 )
@@ -16,6 +26,25 @@ from eth2spec.test.helpers.shard_transitions import is_full_crosslink
 from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to
 
 
+def run_beacon_block(spec, state, block, valid=True):
+    yield 'pre', state.copy()
+
+    if not valid:
+        signed_beacon_block = state_transition_and_sign_block(spec, state, block, expect_fail=True)
+        yield 'block', signed_beacon_block
+        yield 'post', None
+        return
+
+    signed_beacon_block = state_transition_and_sign_block(spec, state, block)
+    yield 'block', signed_beacon_block
+    yield 'post', state
+
+
+#
+# Beacon block with non-empty shard transitions
+#
+
+
 def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True):
     transition_to(spec, state, state.slot + target_len_offset_slot)
 
@@ -49,8 +78,8 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm
         yield 'post', None
         return
 
-    state_transition_and_sign_block(spec, state, beacon_block)
-    yield 'block', beacon_block
+    signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block)
+    yield 'block', signed_beacon_block
     yield 'post', state
 
     for shard in range(spec.get_active_shard_count(state)):
@@ -102,3 +131,112 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
     assert state.shard_states[shard].slot == state.slot - 1
 
     yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard)
+
+
+#
+# Beacon block with custody operations
+#
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_with_shard_transition_with_custody_challenge_and_response(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
+    if not is_full_crosslink(spec, state):
+        # skip
+        return
+
+    state = transition_to_valid_shard_slot(spec, state)
+
+    # build shard block
+    shard = 0
+    committee_index = get_committee_index_of_shard(spec, state, state.slot, shard)
+    body = get_sample_shard_block_body(spec)
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
+    shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
+    shard_transitions = get_shard_transitions(spec, state, shard_block_dict)
+    attestation = get_valid_on_time_attestation(
+        spec, state, index=committee_index,
+        shard_transition=shard_transitions[shard], signed=True,
+    )
+
+    block = build_empty_block(spec, state, slot=state.slot + 1)
+    block.body.attestations = [attestation]
+    block.body.shard_transitions = shard_transitions
+
+    # CustodyChunkChallenge operation
+    challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transitions[shard])
+    block.body.chunk_challenges = [challenge]
+    # CustodyChunkResponse operation
+    chunk_challenge_index = state.custody_chunk_challenge_index
+    custody_response = get_valid_custody_chunk_response(
+        spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=body)
+    block.body.chunk_challenge_responses = [custody_response]
+
+    yield from run_beacon_block(spec, state, block)
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_custody_key_reveal(spec, state):
+    state = transition_to_valid_shard_slot(spec, state)
+    transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH)
+
+    block = build_empty_block(spec, state, slot=state.slot + 1)
+    custody_key_reveal = get_valid_custody_key_reveal(spec, state)
+    block.body.custody_key_reveals = [custody_key_reveal]
+
+    yield from run_beacon_block(spec, state, block)
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_early_derived_secret_reveal(spec, state):
+    state = transition_to_valid_shard_slot(spec, state)
+    block = build_empty_block(spec, state, slot=state.slot + 1)
+    early_derived_secret_reveal = get_valid_early_derived_secret_reveal(spec, state)
+    block.body.early_derived_secret_reveals = [early_derived_secret_reveal]
+
+    yield from run_beacon_block(spec, state, block)
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_custody_slashing(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
+    if not is_full_crosslink(spec, state):
+        # skip
+        return
+
+    state = transition_to_valid_shard_slot(spec, state)
+
+    # Build shard block
+    shard = 0
+    committee_index = get_committee_index_of_shard(spec, state, state.slot, shard)
+    # Create slashable shard block body
+    validator_index = spec.get_beacon_committee(state, state.slot, committee_index)[0]
+    custody_secret = get_custody_secret(spec, state, validator_index)
+    slashable_body = get_custody_slashable_test_vector(spec, custody_secret, length=100, slashable=True)
+    shard_block = build_shard_block(spec, state, shard, body=slashable_body, slot=state.slot, signed=True)
+    shard_block_dict: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
+    shard_transitions = get_shard_transitions(spec, state, shard_block_dict)
+
+    attestation = get_valid_on_time_attestation(
+        spec, state, index=committee_index,
+        shard_transition=shard_transitions[shard], signed=True,
+    )
+    block = build_empty_block(spec, state, slot=state.slot + 1)
+    block.body.attestations = [attestation]
+    block.body.shard_transitions = shard_transitions
+
+    _, _, _ = run_beacon_block(spec, state, block)
+
+    transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1))
+
+    block = build_empty_block(spec, state, slot=state.slot + 1)
+    custody_slashing = get_valid_custody_slashing(
+        spec, state, attestation, shard_transitions[shard], custody_secret, slashable_body
+    )
+    block.body.custody_slashings = [custody_slashing]
+
+    yield from run_beacon_block(spec, state, block)