Merge branch 'v08x' into testgen-reorg

scripts/build_spec.py

@@ -49,7 +49,7 @@ from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
     serialize,
-    is_empty,
+    is_zero,
 )
 from eth2spec.utils.ssz.ssz_typing import (
     bit, boolean, Container, List, Vector, Bytes, uint64,

specs/core/0_beacon-chain.md

@@ -1504,8 +1504,6 @@ def process_final_updates(state: BeaconState) -> None:
         HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2
         if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance:
             validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
-    # Update start shard
-    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Set active index root
     index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY)
     index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR
@@ -1522,6 +1520,8 @@ def process_final_updates(state: BeaconState) -> None:
     if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
         historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots)
         state.historical_roots.append(hash_tree_root(historical_batch))
+    # Update start shard
+    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Rotate current/previous epoch attestations
     state.previous_epoch_attestations = state.current_epoch_attestations
     state.current_epoch_attestations = []
@@ -1654,6 +1654,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
     attestation_slot = get_attestation_data_slot(state, data)
     assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH
 
+    committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard)
+    assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len(committee)
+
     pending_attestation = PendingAttestation(
         data=data,
         aggregation_bits=attestation.aggregation_bits,

specs/core/0_fork-choice.md

@@ -91,8 +91,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store:
 ```python
 def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
     block = store.blocks[root]
-    assert block.slot >= slot
+    if block.slot > slot:
-    return root if block.slot == slot else get_ancestor(store, block.parent_root, slot)
+        return get_ancestor(store, block.parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        return Bytes32()  # root is older than queried slot: no results. 
 ```
 
 #### `get_latest_attesting_balance`

specs/core/1_custody-game.md

@@ -328,7 +328,7 @@ def get_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch
 ```python
 def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int:
     for i in range(len(list)):
-        if is_empty(list[i]):
+        if is_zero(list[i]):
             list[i] = new_element
             return i
     list.append(new_element)

specs/simple-serialize.md

@@ -14,7 +14,7 @@
         - [Variable-size and fixed-size](#variable-size-and-fixed-size)
         - [Aliases](#aliases)
         - [Default values](#default-values)
-            - [`is_empty`](#is_empty)
+            - [`is_zero`](#is_zero)
         - [Illegal types](#illegal-types)
     - [Serialization](#serialization)
         - [`uintN`](#uintn)
@@ -75,19 +75,21 @@ For convenience we alias:
 * `bit` to `boolean`
 * `byte` to `uint8` (this is a basic type)
 * `BytesN` to `Vector[byte, N]` (this is *not* a basic type)
-* `null`: `{}`, i.e. the empty container
+* `null`: `{}`
 
 ### Default values
 
 The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union.
 
-#### `is_empty`
+#### `is_zero`
 
-An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it is equal to the default value for that type.
+An SSZ object is called zeroed (and thus, `is_zero(object)` returns true) if it is equal to the default value for that type.
 
 ### Illegal types
 
-Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero).
+- Empty vector types (`Vector[type, 0]`, `Bitvector[0]`) are illegal.
+- Containers with no fields are illegal.
+- The `null` type is only legal as the first type in a union subtype (i.e. with type index zero).
 
 ## Serialization
 

test_libs/pyspec/eth2spec/fuzzing/decoder.py

@@ -18,14 +18,11 @@ def translate_typ(typ) -> ssz.BaseSedes:
     elif issubclass(typ, spec_ssz.Vector):
         return ssz.Vector(translate_typ(typ.elem_type), typ.length)
     elif issubclass(typ, spec_ssz.List):
-        # TODO: Make py-ssz List support the new fixed length list
+        return ssz.List(translate_typ(typ.elem_type), typ.length)
-        return ssz.List(translate_typ(typ.elem_type))
     elif issubclass(typ, spec_ssz.Bitlist):
-        # TODO: Once Bitlist implemented in py-ssz, use appropriate type
+        return ssz.Bitlist(typ.length)
-        return ssz.List(translate_typ(typ.elem_type))
     elif issubclass(typ, spec_ssz.Bitvector):
-        # TODO: Once Bitvector implemented in py-ssz, use appropriate type
+        return ssz.Bitvector(typ.length)
-        return ssz.Vector(translate_typ(typ.elem_type), typ.length)
     elif issubclass(typ, spec_ssz.boolean):
         return ssz.boolean
     elif issubclass(typ, spec_ssz.uint):

test_libs/pyspec/eth2spec/fuzzing/test_decoder.py

@@ -9,9 +9,7 @@ def test_decoder():
     rng = Random(123)
 
     # check these types only, Block covers a lot of operation types already.
-    # TODO: Once has Bitlists and Bitvectors, add back
+    for typ in [spec.AttestationDataAndCustodyBit, spec.BeaconState, spec.BeaconBlock]:
-    #       spec.BeaconState and spec.BeaconBlock
-    for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]:
         # create a random pyspec value
         original = random_value.get_random_ssz_object(rng, typ, 100, 10,
                                                       mode=random_value.RandomizationMode.mode_random,
@@ -32,4 +30,6 @@ def test_decoder():
         block = translate_value(raw_value, typ)
 
         # and see if the hash-tree-root of the original matches the hash-tree-root of the decoded & translated value.
-        assert spec_ssz_impl.hash_tree_root(original) == spec_ssz_impl.hash_tree_root(block)
+        original_hash_tree_root = spec_ssz_impl.hash_tree_root(original)
+        assert original_hash_tree_root == spec_ssz_impl.hash_tree_root(block)
+        assert original_hash_tree_root == block_sedes.get_hash_tree_root(raw_value)

test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py

@@ -398,3 +398,61 @@ def test_empty_aggregation_bits(spec, state):
     sign_attestation(spec, state, attestation)
 
     yield from run_attestation_processing(spec, state, attestation)
+
+
+@with_all_phases
+@spec_state_test
+def test_too_many_aggregation_bits(spec, state):
+    attestation = get_valid_attestation(spec, state, signed=True)
+    state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+
+    # one too many bits
+    attestation.aggregation_bits.append(0b0)
+
+    yield from run_attestation_processing(spec, state, attestation, False)
+
+
+@with_all_phases
+@spec_state_test
+def test_too_few_aggregation_bits(spec, state):
+    attestation = get_valid_attestation(spec, state)
+    state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+
+    attestation.aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](
+        *([0b1] + [0b0] * (len(attestation.aggregation_bits) - 1)))
+
+    sign_attestation(spec, state, attestation)
+
+    # one too few bits
+    attestation.aggregation_bits = attestation.aggregation_bits[:-1]
+
+    yield from run_attestation_processing(spec, state, attestation, False)
+
+
+@with_all_phases
+@spec_state_test
+def test_too_many_custody_bits(spec, state):
+    attestation = get_valid_attestation(spec, state, signed=True)
+    state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+
+    # one too many bits
+    attestation.custody_bits.append(0b0)
+
+    yield from run_attestation_processing(spec, state, attestation, False)
+
+
+@with_all_phases
+@spec_state_test
+def test_too_few_custody_bits(spec, state):
+    attestation = get_valid_attestation(spec, state)
+    state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+
+    attestation.custody_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](
+        *([0b1] + [0b0] * (len(attestation.custody_bits) - 1)))
+
+    sign_attestation(spec, state, attestation)
+
+    # one too few bits
+    attestation.custody_bits = attestation.custody_bits[:-1]
+
+    yield from run_attestation_processing(spec, state, attestation, False)

test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py

@@ -89,3 +89,20 @@ def test_historical_root_accumulator(spec, state):
     yield from run_process_final_updates(spec, state)
 
     assert len(state.historical_roots) == history_len + 1
+
+
+@with_all_phases
+@spec_state_test
+def test_compact_committees_root(spec, state):
+    assert spec.SLOTS_PER_ETH1_VOTING_PERIOD > spec.SLOTS_PER_EPOCH
+    # skip ahead to the end of the epoch
+    state.slot = spec.SLOTS_PER_EPOCH - 1
+
+    next_epoch = spec.get_current_epoch(state) + 1
+
+    # ensure that order in which items are processed in final_updates
+    # does not alter the expected_root
+    expected_root = spec.get_compact_committees_root(state, next_epoch)
+    yield from run_process_final_updates(spec, state)
+
+    assert state.compact_committees_roots[next_epoch % spec.EPOCHS_PER_HISTORICAL_VECTOR] == expected_root

test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py

@@ -33,7 +33,7 @@ def deserialize_basic(value, typ: BasicType):
         raise Exception(f"Type not supported: {typ}")
 
 
-def is_empty(obj: SSZValue):
+def is_zero(obj: SSZValue):
     return type(obj).default() == obj
 
 

test_libs/pyspec/requirements.txt

@@ -3,4 +3,4 @@ eth-typing>=2.1.0,<3.0.0
 pycryptodome==3.7.3
 py_ecc==1.7.1
 dataclasses==0.6
-ssz==0.1.0a10
+ssz==0.1.3

test_libs/pyspec/setup.py

@@ -9,7 +9,7 @@ setup(
         "eth-typing>=2.1.0,<3.0.0",
         "pycryptodome==3.7.3",
         "py_ecc==1.7.1",
-        "ssz==0.1.0a10",
+        "ssz==0.1.3",
         "dataclasses==0.6",
     ]
 )