bugfix custody bit index lookup + lint fixes

scripts/build_spec.py

@@ -15,7 +15,7 @@ apply_constants_preset(globals())
 
 PHASE0_IMPORTS = '''from eth2spec.config.apply_config import apply_constants_preset
 from typing import (
-    Dict, Set, Sequence, Tuple, Optional
+    Any, Callable, Dict, Set, Sequence, Tuple, Optional
 )
 
 from dataclasses import (
@@ -40,7 +40,7 @@ from eth2spec.utils.hash_function import hash
 PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0
 from eth2spec.config.apply_config import apply_constants_preset
 from typing import (
-    Dict, Set, Sequence, NewType, Tuple, Union,
+    Any, Callable, Dict, Set, Sequence, NewType, Tuple, Union,
 )
 from math import (
     log2,
@@ -110,13 +110,6 @@ def compute_committee(indices: Sequence[ValidatorIndex],  # type: ignore
     return committee_cache[param_hash]'''
 
 
-def remove_for_phase1(functions: Dict[str, str]):
-    for key, value in functions.items():
-        lines = value.split("\n")
-        lines = filter(lambda s: "[to be removed in phase 1]" not in s, lines)
-        functions[key] = "\n".join(lines)
-
-
 def objects_to_spec(functions: Dict[str, str],
                     custom_types: Dict[str, str],
                     constants: Dict[str, str],
@@ -172,10 +165,10 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
 
 
 ignored_dependencies = [
-    'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'ByteList', 'ByteVector'
+    'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature',
     'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
-    'bytes', 'byte', 'ByteVector'  # to be removed after updating spec doc
+    'bytes', 'byte', 'Bytes', 'BytesN'  # to be removed after updating spec doc
 ]
 
 
@@ -209,7 +202,6 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]
     """
     for key, value in new_objects.items():
         old_objects[key] = value
-    dependency_order_ssz_objects(old_objects, custom_types)
     return old_objects
 
 
@@ -226,6 +218,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     return functions, custom_types, constants, ssz_objects
 
 
+def dependency_order_spec(objs: SpecObject):
+    functions, custom_types, constants, ssz_objects = objs
+    dependency_order_ssz_objects(ssz_objects, custom_types)
+
+
 def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
                       v_guide_sourcefile: str, outfile: str=None) -> Optional[str]:
     phase0_spec = get_spec(phase0_sourcefile)
@@ -234,6 +231,7 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
     spec_objects = phase0_spec
     for value in [fork_choice_spec, v_guide]:
         spec_objects = combine_spec_objects(spec_objects, value)
+    dependency_order_spec(spec_objects)
     spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
@@ -259,11 +257,10 @@ def build_phase1_spec(phase0_beacon_sourcefile: str,
         phase1_fork_sourcefile,
     )
     all_spescs = [get_spec(spec) for spec in all_sourcefiles]
-    for spec in all_spescs:
-        remove_for_phase1(spec[0])
     spec_objects = all_spescs[0]
     for value in all_spescs[1:]:
         spec_objects = combine_spec_objects(spec_objects, value)
+    dependency_order_spec(spec_objects)
     spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:

specs/core/0_beacon-chain.md

@@ -1419,7 +1419,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None:
     # Verify that outstanding deposits are processed up to the maximum number of deposits
     assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index)
 
-    def for_ops(operations, fn):
+    def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None:
         for operation in operations:
             fn(state, operation)
 

specs/core/1_beacon-chain.md

@@ -39,11 +39,11 @@ Configuration is not namespaced. Instead it is strictly an extension;
 | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours |
 | `SHARD_COMMITTEE_PERIOD` | `2**8` (= 256) | epochs | ~27 hours |
 | `SHARD_BLOCK_CHUNK_SIZE` | `2**18` (= 262,144) | |
-| `SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | |
+| `MAX_SHARD_BLOCK_CHUNKS` | `2**2` (= 4) | |
 | `TARGET_SHARD_BLOCK_SIZE` | `3 * 2**16` (= 196,608) | |
 | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | |
 | `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | |
-| `EMPTY_CHUNK_ROOT` | `hash_tree_root(ByteVector[SHARD_BLOCK_CHUNK_SIZE]())` | |
+| `EMPTY_CHUNK_ROOT` | `hash_tree_root(BytesN[SHARD_BLOCK_CHUNK_SIZE]())` | |
 | `MAX_GASPRICE` | `2**14` (= 16,384) | Gwei | |
 | `MIN_GASPRICE` | `2**5` (= 32) | Gwei | |
 | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | |
@@ -62,7 +62,7 @@ class ShardBlockWrapper(Container):
     shard_parent_root: Hash
     beacon_parent_root: Hash
     slot: Slot
-    body: ByteVector[MAX_SHARD_BLOCK_SIZE]
+    body: BytesN[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]
     signature: BLSSignature
 ```
 
@@ -315,7 +315,7 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid
 ```python
 def chunks_to_body_root(chunks):
     return hash_tree_root(Vector[Hash, MAX_SHARD_BLOCK_CHUNKS](
-            chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))
+        chunks + [EMPTY_CHUNK_ROOT] * (MAX_SHARD_BLOCK_CHUNKS - len(chunks))
     ))
 ```
 
@@ -376,11 +376,11 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation)
 def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei:
     if length > TARGET_SHARD_BLOCK_SIZE:
         delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) 
-                    // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
+                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
         return min(prev_gasprice + delta, MAX_GASPRICE)
     else:
         delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length)
-                    // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
+                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
         return max(prev_gasprice, MIN_GASPRICE + delta) - delta
 ```
 
@@ -465,7 +465,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None:
     # Verify that outstanding deposits are processed up to the maximum number of deposits
     assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index)
     
-    def for_ops(operations, fn):
+    def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None:
         for operation in operations:
             fn(state, operation)
     

specs/core/1_custody-game.md

@@ -100,7 +100,7 @@ class CustodySlashing(Container):
     whistleblower_index: ValidatorIndex
     shard_transition: ShardTransition
     attestation: Attestation
-    data: ByteList[MAX_SHARD_BLOCK_SIZE]
+    data: Bytes[MAX_SHARD_BLOCK_CHUNKS * SHARD_BLOCK_CHUNK_SIZE]
     signature: BLSSignature
 ```
 
@@ -178,13 +178,13 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]:
 ### `compute_custody_bit`
 
 ```python
-def compute_custody_bit(key: BLSSignature, data: bytes) -> bool:
+def compute_custody_bit(key: BLSSignature, data: bytes) -> bit:
     full_G2_element = bls_signature_to_G2(key)
     s = full_G2_element[0].coeffs
     bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(atom, "little"), BLS12_381_Q)
             for i, atom in enumerate(get_custody_atoms(data))]
     # XOR all atom bits
-    return bool(sum(bits) % 2)
+    return bit(sum(bits) % 2)
 ```
 
 ### `get_randao_epoch_for_custody_period`
@@ -212,7 +212,7 @@ def get_custody_period_for_validator(validator_index: ValidatorIndex, epoch: Epo
 
 ```python
 def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) -> None:
-    def for_ops(operations, fn):
+    def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None:
         for operation in operations:
             fn(state, operation)
 
@@ -374,13 +374,14 @@ def process_custody_slashing(state: BeaconState, custody_slashing: CustodySlashi
     shard_chunk_roots = shard_transition.shard_data_roots[custody_slashing.data_index]
     assert hash_tree_root(custody_slashing.data) == chunks_to_body_root(shard_chunk_roots)
 
-    # Verify existence of claimed malefactor
+    # Verify existence and participation of claimed malefactor
     attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits)
     assert custody_slashing.malefactor_index in attesters
     
     # Get the custody bit
     custody_bits = attestation.custody_bits[custody_slashing.data_index]
-    claimed_custody_bit = custody_bits[attesters.index(custody_slashing.malefactor_index)]
+    committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index)
+    claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)]
     
     # Compute the custody bit
     computed_custody_bit = compute_custody_bit(custody_slashing.data)