Merge branch 'dev' into JustinDrake-patch-8

.circleci/config.yml

@@ -0,0 +1,41 @@
+# Python CircleCI 2.0 configuration file
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/python:3.6
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "requirements.txt" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install -r requirements.txt
+      - run:
+          name: build phase0 spec
+          command: make build/phase0
+
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: run tests
+          command: |
+            . venv/bin/activate
+            pytest tests
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports

.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+/__pycache__
+/venv
+/.pytest_cache
+
+build/

LICENSE

@@ -0,0 +1,116 @@
+CC0 1.0 Universal
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific
+works ("Commons") that the public can reliably and without fear of later
+claims of infringement build upon, modify, incorporate in other works, reuse
+and redistribute as freely as possible in any form whatsoever and for any
+purposes, including without limitation commercial purposes. These owners may
+contribute to the Commons to promote the ideal of a free culture and the
+further production of creative, cultural and scientific works, or to gain
+reputation or greater distribution for their Work in part through the use and
+efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation
+of additional consideration or compensation, the person associating CC0 with a
+Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
+and publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not limited
+to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display, communicate,
+  and translate a Work;
+
+  ii. moral rights retained by the original author(s) and/or performer(s);
+
+  iii. publicity and privacy rights pertaining to a person's image or likeness
+  depicted in a Work;
+
+  iv. rights protecting against unfair competition in regards to a Work,
+  subject to the limitations in paragraph 4(a), below;
+
+  v. rights protecting the extraction, dissemination, use and reuse of data in
+  a Work;
+
+  vi. database rights (such as those arising under Directive 96/9/EC of the
+  European Parliament and of the Council of 11 March 1996 on the legal
+  protection of databases, and under any national implementation thereof,
+  including any amended or successor version of such directive); and
+
+  vii. other similar, equivalent or corresponding rights throughout the world
+  based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
+and Related Rights and associated claims and causes of action, whether now
+known or unknown (including existing as well as future claims and causes of
+action), in the Work (i) in all territories worldwide, (ii) for the maximum
+duration provided by applicable law or treaty (including future time
+extensions), (iii) in any current or future medium and for any number of
+copies, and (iv) for any purpose whatsoever, including without limitation
+commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
+the Waiver for the benefit of each member of the public at large and to the
+detriment of Affirmer's heirs and successors, fully intending that such Waiver
+shall not be subject to revocation, rescission, cancellation, termination, or
+any other legal or equitable action to disrupt the quiet enjoyment of the Work
+by the public as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free,
+non transferable, non sublicensable, non exclusive, irrevocable and
+unconditional license to exercise Affirmer's Copyright and Related Rights in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "License"). The License shall be deemed effective as
+of the date CC0 was applied by Affirmer to the Work. Should any part of the
+License for any reason be judged legally invalid or ineffective under
+applicable law, such partial invalidity or ineffectiveness shall not
+invalidate the remainder of the License, and in such case Affirmer hereby
+affirms that he or she will not (i) exercise any of his or her remaining
+Copyright and Related Rights in the Work or (ii) assert any associated claims
+and causes of action with respect to the Work, in either case contrary to
+Affirmer's express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+  a. No trademark or patent rights held by Affirmer are waived, abandoned,
+  surrendered, licensed or otherwise affected by this document.
+
+  b. Affirmer offers the Work as-is and makes no representations or warranties
+  of any kind concerning the Work, express, implied, statutory or otherwise,
+  including without limitation warranties of title, merchantability, fitness
+  for a particular purpose, non infringement, or the absence of latent or
+  other defects, accuracy, or the present or absence of errors, whether or not
+  discoverable, all to the greatest extent permissible under applicable law.
+
+  c. Affirmer disclaims responsibility for clearing rights of other persons
+  that may apply to the Work or any use thereof, including without limitation
+  any person's Copyright and Related Rights in the Work. Further, Affirmer
+  disclaims responsibility for obtaining any necessary consents, permissions
+  or other rights required for any use of the Work.
+
+  d. Affirmer understands and acknowledges that Creative Commons is not a
+  party to this document and has no duty or obligation with respect to this
+  CC0 or use of the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>

Makefile

@@ -0,0 +1,29 @@
+SPEC_DIR = ./specs
+SCRIPT_DIR = ./scripts
+BUILD_DIR = ./build
+UTILS_DIR = ./utils
+
+
+.PHONY: clean all test
+
+
+all: $(BUILD_DIR)/phase0
+
+
+clean:
+	rm -rf $(BUILD_DIR)
+
+
+# runs a limited set of tests against a minimal config
+# run pytest with `-m` option to full suite
+test:
+	pytest -m "sanity and minimal_config" tests/
+
+
+$(BUILD_DIR)/phase0:
+	mkdir -p $@
+	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py
+	mkdir -p $@/utils
+	cp $(UTILS_DIR)/phase0/* $@/utils
+	cp $(UTILS_DIR)/phase0/state_transition.py $@
+	touch $@/__init__.py $@/utils/__init__.py

requirements.txt

@@ -0,0 +1,6 @@
+eth-utils>=1.3.0,<2
+eth-typing>=2.1.0,<3.0.0
+oyaml==0.7
+pycryptodome==3.7.3
+py_ecc>=1.6.0
+pytest>=3.6,<3.7

scripts/phase0/build_spec.py

@@ -0,0 +1,79 @@
+import sys
+import function_puller
+
+
+def build_spec(sourcefile, outfile):
+    code_lines = []
+
+    code_lines.append("from build.phase0.utils.minimal_ssz import *")
+    code_lines.append("from build.phase0.utils.bls_stub import *")
+    for i in (1, 2, 3, 4, 8, 32, 48, 96):
+        code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i))
+    code_lines.append("SLOTS_PER_EPOCH = 64")  # stub, will get overwritten by real var
+    code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH")
+
+    code_lines.append("""
+from typing import (
+    Any,
+    Callable,
+    List,
+    NewType,
+    Tuple,
+)
+
+
+Slot = NewType('Slot', int)  # uint64
+Epoch = NewType('Epoch', int)  # uint64
+Shard = NewType('Shard', int)  # uint64
+ValidatorIndex = NewType('ValidatorIndex', int)  # uint64
+Gwei = NewType('Gwei', int)  # uint64
+Bytes32 = NewType('Bytes32', bytes)  # bytes32
+BLSPubkey = NewType('BLSPubkey', bytes)  # bytes48
+BLSSignature = NewType('BLSSignature', bytes)  # bytes96
+Any = None
+Store = None
+    """)
+
+    code_lines += function_puller.get_lines(sourcefile)
+
+    code_lines.append("""
+# Monkey patch validator get committee code
+_compute_committee = compute_committee
+committee_cache = {}
+def compute_committee(validator_indices: List[ValidatorIndex],
+                      seed: Bytes32,
+                      index: int,
+                      total_committees: int) -> List[ValidatorIndex]:
+
+    param_hash = (hash_tree_root(validator_indices), seed, index, total_committees)
+
+    if param_hash in committee_cache:
+        # print("Cache hit, epoch={0}".format(epoch))
+        return committee_cache[param_hash]
+    else:
+        # print("Cache miss, epoch={0}".format(epoch))
+        ret = _compute_committee(validator_indices, seed, index, total_committees)
+        committee_cache[param_hash] = ret
+        return ret
+
+
+# Monkey patch hash cache
+_hash = hash
+hash_cache = {}
+def hash(x):
+    if x in hash_cache:
+        return hash_cache[x]
+    else:
+        ret = _hash(x)
+        hash_cache[x] = ret
+        return ret
+    """)
+
+    with open(outfile, 'w') as out:
+        out.write("\n".join(code_lines))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print("Error: spec source and outfile must defined")
+    build_spec(sys.argv[1], sys.argv[2])

scripts/phase0/function_puller.py

@@ -0,0 +1,46 @@
+import sys
+
+
+def get_lines(file_name):
+    code_lines = []
+    pulling_from = None
+    current_name = None
+    processing_typedef = False
+    for linenum, line in enumerate(open(sys.argv[1]).readlines()):
+        line = line.rstrip()
+        if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
+            current_name = line[line[:-1].rfind('`') + 1: -1]
+        if line[:9] == '```python':
+            assert pulling_from is None
+            pulling_from = linenum + 1
+        elif line[:3] == '```':
+            if pulling_from is None:
+                pulling_from = linenum
+            else:
+                if processing_typedef:
+                    assert code_lines[-1] == '}'
+                    code_lines[-1] = '})'
+                pulling_from = None
+                processing_typedef = False
+        else:
+            if pulling_from == linenum and line == '{':
+                code_lines.append('%s = SSZType({' % current_name)
+                processing_typedef = True
+            elif pulling_from is not None:
+                code_lines.append(line)
+            elif pulling_from is None and len(line) > 0 and line[0] == '|':
+                row = line[1:].split('|')
+                if len(row) >= 2:
+                    for i in range(2):
+                        row[i] = row[i].strip().strip('`')
+                        if '`' in row[i]:
+                            row[i] = row[i][:row[i].find('`')]
+                    eligible = True
+                    if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_':
+                        eligible = False
+                    for c in row[0]:
+                        if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789':
+                            eligible = False
+                    if eligible:
+                        code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890')))
+    return code_lines

specs/bls_signature.md

@@ -110,11 +110,11 @@ def modular_squareroot(value: Fq2) -> Fq2:
 
 ### `bls_aggregate_pubkeys`
 
-Let `bls_aggregate_pubkeys(pubkeys: List[Bytes48]) -> Bytes48` return `pubkeys[0] + .... + pubkeys[len(pubkeys)-1]`, where `+` is the elliptic curve addition operation over the G1 curve.
+Let `bls_aggregate_pubkeys(pubkeys: List[Bytes48]) -> Bytes48` return `pubkeys[0] + .... + pubkeys[len(pubkeys)-1]`, where `+` is the elliptic curve addition operation over the G1 curve. (When `len(pubkeys) == 0` the empty sum is the G1 point at infinity.)
 
 ### `bls_aggregate_signatures`
 
-Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `signatures[0] + .... + signatures[len(signatures)-1]`, where `+` is the elliptic curve addition operation over the G2 curve.
+Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `signatures[0] + .... + signatures[len(signatures)-1]`, where `+` is the elliptic curve addition operation over the G2 curve. (When `len(signatures) == 0` the empty sum is the G2 point at infinity.)
 
 ## Signature verification
 

specs/core/0_beacon-chain.md

@@ -61,9 +61,9 @@
         - [`is_active_validator`](#is_active_validator)
         - [`get_active_validator_indices`](#get_active_validator_indices)
         - [`get_permuted_index`](#get_permuted_index)
-        - [`split`](#split)
+        - [`get_split_offset`](#get_split_offset)
         - [`get_epoch_committee_count`](#get_epoch_committee_count)
-        - [`get_shuffling`](#get_shuffling)
+        - [`compute_committee`](#compute_committee)
         - [`get_previous_epoch_committee_count`](#get_previous_epoch_committee_count)
         - [`get_current_epoch_committee_count`](#get_current_epoch_committee_count)
         - [`get_next_epoch_committee_count`](#get_next_epoch_committee_count)
@@ -186,7 +186,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code.
 | `MAX_EXIT_DEQUEUES_PER_EPOCH` | `2**2` (= 4) |
 | `SHUFFLE_ROUND_COUNT` | 90 |
 
-* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.)
+* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.)
 
 ### Deposit contract
 
@@ -208,7 +208,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code.
 
 | Name | Value |
 | - | - |
-| `GENESIS_FORK_VERSION` | `0` |
+| `GENESIS_FORK_VERSION` | `int_to_bytes4(0)` |
 | `GENESIS_SLOT` | `2**32` |
 | `GENESIS_EPOCH` | `slot_to_epoch(GENESIS_SLOT)` |
 | `GENESIS_START_SHARD` | `0` |
@@ -290,9 +290,9 @@ The types are defined topologically to aid in facilitating an executable version
 ```python
 {
     # Previous fork version
-    'previous_version': 'uint64',
+    'previous_version': 'bytes4',
     # Current fork version
-    'current_version': 'uint64',
+    'current_version': 'bytes4',
     # Fork epoch number
     'epoch': 'uint64',
 }
@@ -335,22 +335,19 @@ The types are defined topologically to aid in facilitating an executable version
 
 ```python
 {
-    # Slot number
+    # LMD GHOST vote
     'slot': 'uint64',
-    # Shard number
-    'shard': 'uint64',
-    # Root of the signed beacon block
     'beacon_block_root': 'bytes32',
-    # Root of the ancestor at the epoch boundary
-    'epoch_boundary_root': 'bytes32',
-    # Data from the shard since the last attestation
+
+    # FFG vote
+    'source_epoch': 'uint64',
+    'source_root': 'bytes32',
+    'target_root': 'bytes32',
+
+    # Crosslink vote
+    'shard': 'uint64',
+    'previous_crosslink': Crosslink,
     'crosslink_data_root': 'bytes32',
-    # Last crosslink
-    'latest_crosslink': Crosslink,
-    # Last justified epoch in the beacon state
-    'justified_epoch': 'uint64',
-    # Hash of the last justified beacon block
-    'justified_block_root': 'bytes32',
 }
 ```
 
@@ -612,9 +609,12 @@ The types are defined topologically to aid in facilitating an executable version
     'previous_epoch_attestations': [PendingAttestation],
     'current_epoch_attestations': [PendingAttestation],
     'previous_justified_epoch': 'uint64',
-    'justified_epoch': 'uint64',
+    'current_justified_epoch': 'uint64',
+    'previous_justified_root': 'bytes32',
+    'current_justified_root': 'bytes32',
     'justification_bitfield': 'uint64',
     'finalized_epoch': 'uint64',
+    'finalized_root': 'bytes32',
 
     # Recent state
     'latest_crosslinks': [Crosslink, SHARD_COUNT],
@@ -628,7 +628,7 @@ The types are defined topologically to aid in facilitating an executable version
     # Ethereum 1.0 chain data
     'latest_eth1_data': Eth1Data,
     'eth1_data_votes': [Eth1DataVote],
-    'deposit_index': 'uint64'
+    'deposit_index': 'uint64',
 }
 ```
 
@@ -773,18 +773,11 @@ def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int:
     return index
 ```
 
-### `split`
+### `get_split_offset`
 
 ```python
-def split(values: List[Any], split_count: int) -> List[List[Any]]:
-    """
-    Splits ``values`` into ``split_count`` pieces.
-    """
-    list_length = len(values)
-    return [
-        values[(list_length * i // split_count): (list_length * (i + 1) // split_count)]
-        for i in range(split_count)
-    ]
+def get_split_offset(list_length: int, split_count: int, index: int) -> int:
+    return (list_length * index) // split_count
 ```
 
 ### `get_epoch_committee_count`
@@ -803,28 +796,26 @@ def get_epoch_committee_count(active_validator_count: int) -> int:
     ) * SLOTS_PER_EPOCH
 ```
 
-### `get_shuffling`
+### `compute_committee`
 
 ```python
-def get_shuffling(seed: Bytes32,
-                  validators: List[Validator],
-                  epoch: Epoch) -> List[List[ValidatorIndex]]:
+def compute_committee(validator_indices: List[ValidatorIndex],
+                      seed: Bytes32,
+                      index: int,
+                      total_committees: int) -> List[ValidatorIndex]:
     """
-    Shuffle active validators and split into crosslink committees.
-    Return a list of committees (each a list of validator indices).
+    Return the ``index``'th shuffled committee out of a total ``total_committees``
+    using ``validator_indices`` and ``seed``.
     """
-    # Shuffle active validator indices
-    active_validator_indices = get_active_validator_indices(validators, epoch)
-    length = len(active_validator_indices)
-    shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)]
-
-    # Split the shuffled active validator indices
-    return split(shuffled_indices, get_epoch_committee_count(length))
+    start_offset = get_split_offset(len(validator_indices), total_committees, index)
+    end_offset = get_split_offset(len(validator_indices), total_committees, index + 1)
+    return [
+        validator_indices[get_permuted_index(i, len(validator_indices), seed)]
+        for i in range(start_offset, end_offset)
+    ]
 ```
 
-**Invariant**: if `get_shuffling(seed, validators, epoch)` returns some value `x` for some `epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY`, it should return the same value `x` for the same `seed` and `epoch` and possible future modifications of `validators` forever in phase 0, and until the ~1 year deletion delay in phase 2 and in the future.
-
-**Note**: this definition and the next few definitions make heavy use of repetitive computing. Production implementations are expected to appropriately use caching/memoization to avoid redoing work.
+**Note**: this definition and the next few definitions are highly inefficient as algorithms as they re-calculate many sub-expressions. Production implementations are expected to appropriately use caching/memoization to avoid redoing work.
 
 ### `get_previous_epoch_committee_count`
 
@@ -916,18 +907,14 @@ def get_crosslink_committees_at_slot(state: BeaconState,
             shuffling_epoch = state.current_shuffling_epoch
             shuffling_start_shard = state.current_shuffling_start_shard
 
-    shuffling = get_shuffling(
-        seed,
-        state.validator_registry,
-        shuffling_epoch,
-    )
-    offset = slot % SLOTS_PER_EPOCH
+    indices = get_active_validator_indices(state.validator_registry, shuffling_epoch)
     committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH
+    offset = slot % SLOTS_PER_EPOCH
     slot_start_shard = (shuffling_start_shard + committees_per_slot * offset) % SHARD_COUNT
 
     return [
         (
-            shuffling[committees_per_slot * offset + i],
+            compute_committee(indices, seed, committees_per_slot * offset + i, committees_per_epoch),
             (slot_start_shard + i) % SHARD_COUNT,
         )
         for i in range(committees_per_slot)
@@ -1015,7 +1002,7 @@ def get_beacon_proposer_index(state: BeaconState,
     assert previous_epoch <= epoch <= next_epoch
 
     first_committee, _ = get_crosslink_committees_at_slot(state, slot, registry_change)[0]
-    return first_committee[slot % len(first_committee)]
+    return first_committee[epoch % len(first_committee)]
 ```
 
 ### `verify_merkle_branch`
@@ -1042,7 +1029,7 @@ def get_attestation_participants(state: BeaconState,
                                  attestation_data: AttestationData,
                                  bitfield: bytes) -> List[ValidatorIndex]:
     """
-    Return the participant indices at for the ``attestation_data`` and ``bitfield``.
+    Return the participant indices corresponding to ``attestation_data`` and ``bitfield``.
     """
     # Find the committee in the list with the desired shard
     crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot)
@@ -1106,7 +1093,7 @@ def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> G
 
 ```python
 def get_fork_version(fork: Fork,
-                     epoch: Epoch) -> int:
+                     epoch: Epoch) -> bytes:
     """
     Return the fork version of the given ``epoch``.
     """
@@ -1125,8 +1112,7 @@ def get_domain(fork: Fork,
     """
     Get the domain number that represents the fork meta and signature domain.
     """
-    fork_version = get_fork_version(fork, epoch)
-    return fork_version * 2**32 + domain_type
+    return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type))
 ```
 
 ### `get_bitfield_bit`
@@ -1223,8 +1209,8 @@ def is_surround_vote(attestation_data_1: AttestationData,
     """
     Check if ``attestation_data_1`` surrounds ``attestation_data_2``.
     """
-    source_epoch_1 = attestation_data_1.justified_epoch
-    source_epoch_2 = attestation_data_2.justified_epoch
+    source_epoch_1 = attestation_data_1.source_epoch
+    source_epoch_2 = attestation_data_2.source_epoch
     target_epoch_1 = slot_to_epoch(attestation_data_1.slot)
     target_epoch_2 = slot_to_epoch(attestation_data_2.slot)
 
@@ -1288,7 +1274,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None:
     serialized_deposit_data = serialize(deposit.deposit_data)
     # Deposits must be processed in order
     assert deposit.index == state.deposit_index
-    
+
     # Verify the Merkle branch
     merkle_branch_is_valid = verify_merkle_branch(
         leaf=hash(serialized_deposit_data),
@@ -1298,27 +1284,12 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None:
         root=state.latest_eth1_data.deposit_root,
     )
     assert merkle_branch_is_valid
-        
+
     # Increment the next deposit index we are expecting. Note that this
     # needs to be done here because while the deposit contract will never
     # create an invalid Merkle branch, it may admit an invalid deposit
     # object, and we need to be able to skip over it
     state.deposit_index += 1
-    
-    # Verify the proof of possession
-    proof_is_valid = bls_verify(
-        pubkey=deposit_input.pubkey,
-        message_hash=signed_root(deposit_input),
-        signature=deposit_input.proof_of_possession,
-        domain=get_domain(
-            state.fork,
-            get_current_epoch(state),
-            DOMAIN_DEPOSIT,
-        )
-    )
-    
-    if not proof_is_valid:
-        return
 
     validator_pubkeys = [v.pubkey for v in state.validator_registry]
     pubkey = deposit_input.pubkey
@@ -1326,6 +1297,20 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None:
     withdrawal_credentials = deposit_input.withdrawal_credentials
 
     if pubkey not in validator_pubkeys:
+        # Verify the proof of possession
+        proof_is_valid = bls_verify(
+            pubkey=deposit_input.pubkey,
+            message_hash=signed_root(deposit_input),
+            signature=deposit_input.proof_of_possession,
+            domain=get_domain(
+                state.fork,
+                get_current_epoch(state),
+                DOMAIN_DEPOSIT,
+            )
+        )
+        if not proof_is_valid:
+            return
+
         # Add new validator
         validator = Validator(
             pubkey=pubkey,
@@ -1342,10 +1327,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None:
         state.validator_balances.append(amount)
     else:
         # Increase balance by deposit amount
-        index = validator_pubkeys.index(pubkey)
-        assert state.validator_registry[index].withdrawal_credentials == withdrawal_credentials
-
-        state.validator_balances[index] += amount
+        state.validator_balances[validator_pubkeys.index(pubkey)] += amount
 ```
 
 ### Routines for updating validator status
@@ -1382,17 +1364,14 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None:
 ```python
 def exit_validator(state: BeaconState, index: ValidatorIndex) -> None:
     """
-    Exit the validator of the given ``index``.
+    Exit the validator with the given ``index``.
     Note that this function mutates ``state``.
     """
     validator = state.validator_registry[index]
-    delayed_activation_exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state))
 
-    # The following updates only occur if not previous exited
-    if validator.exit_epoch <= delayed_activation_exit_epoch:
-        return
-    else:
-        validator.exit_epoch = delayed_activation_exit_epoch
+    # Update validator exit epoch if not previously exited
+    if validator.exit_epoch == FAR_FUTURE_EPOCH:
+        validator.exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state))
 ```
 
 #### `slash_validator`
@@ -1533,10 +1512,10 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit],
         validator_registry_update_epoch=GENESIS_EPOCH,
 
         # Randomness and committees
-        latest_randao_mixes=[ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)],
+        latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]),
         previous_shuffling_start_shard=GENESIS_START_SHARD,
         current_shuffling_start_shard=GENESIS_START_SHARD,
-        previous_shuffling_epoch=GENESIS_EPOCH,
+        previous_shuffling_epoch=GENESIS_EPOCH - 1,
         current_shuffling_epoch=GENESIS_EPOCH,
         previous_shuffling_seed=ZERO_HASH,
         current_shuffling_seed=ZERO_HASH,
@@ -1544,17 +1523,20 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit],
         # Finality
         previous_epoch_attestations=[],
         current_epoch_attestations=[],
-        previous_justified_epoch=GENESIS_EPOCH,
-        justified_epoch=GENESIS_EPOCH,
+        previous_justified_epoch=GENESIS_EPOCH - 1,
+        current_justified_epoch=GENESIS_EPOCH,
+        previous_justified_root=ZERO_HASH,
+        current_justified_root=ZERO_HASH,
         justification_bitfield=0,
         finalized_epoch=GENESIS_EPOCH,
+        finalized_root=ZERO_HASH,
 
         # Recent state
-        latest_crosslinks=[Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)],
-        latest_block_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)],
-        latest_state_roots=[ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)],
-        latest_active_index_roots=[ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)],
-        latest_slashed_balances=[0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)],
+        latest_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]),
+        latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]),
+        latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]),
+        latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]),
+        latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]),
         latest_block_header=get_temporary_block_header(get_empty_block()),
         historical_roots=[],
 
@@ -1721,7 +1703,7 @@ def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestat
 ```
 
 ```python
-def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]:
+def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei:
     return get_total_balance(state, get_attesting_indices(state, attestations))
 ```
 
@@ -1729,7 +1711,7 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat
 def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]:
     return [
         a for a in state.current_epoch_attestations
-        if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state)))
+        if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state)))
     ]
 ```
 
@@ -1737,7 +1719,7 @@ def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingA
 def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]:
     return [
         a for a in state.previous_epoch_attestations
-        if a.data.epoch_boundary_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state)))
+        if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state)))
     ]
 ```
 
@@ -1755,7 +1737,7 @@ def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[Pe
 def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]:
     all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations
     valid_attestations = [
-        a for a in all_attestations if a.data.latest_crosslink == state.latest_crosslinks[shard]
+        a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard]
     ]
     all_roots = [a.data.crosslink_data_root for a in valid_attestations]
 
@@ -1798,7 +1780,9 @@ Run the following function:
 
 ```python
 def update_justification_and_finalization(state: BeaconState) -> None:
-    new_justified_epoch = state.justified_epoch
+    new_justified_epoch = state.current_justified_epoch
+    new_finalized_epoch = state.finalized_epoch
+
     # Rotate the justification bitfield up one epoch to make room for the current epoch
     state.justification_bitfield <<= 1
     # If the previous epoch gets justified, fill the second last bit
@@ -1817,20 +1801,26 @@ def update_justification_and_finalization(state: BeaconState) -> None:
     current_epoch = get_current_epoch(state)
     # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source
     if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3:
-        state.finalized_epoch = state.previous_justified_epoch
+        new_finalized_epoch = state.previous_justified_epoch
     # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source
     if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2:
-        state.finalized_epoch = state.previous_justified_epoch
+        new_finalized_epoch = state.previous_justified_epoch
     # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source
-    if (bitfield >> 0) % 8 == 0b111 and state.justified_epoch == current_epoch - 2:
-        state.finalized_epoch = state.justified_epoch
+    if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2:
+        new_finalized_epoch = state.current_justified_epoch
     # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source
-    if (bitfield >> 0) % 4 == 0b11 and state.justified_epoch == current_epoch - 1:
-        state.finalized_epoch = state.justified_epoch
+    if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1:
+        new_finalized_epoch = state.current_justified_epoch
 
-    # Rotate justified epochs
-    state.previous_justified_epoch = state.justified_epoch
-    state.justified_epoch = new_justified_epoch
+    # Update state jusification/finality fields
+    state.previous_justified_epoch = state.current_justified_epoch
+    state.previous_justified_root = state.current_justified_root
+    if new_justified_epoch != state.current_justified_epoch:
+        state.current_justified_epoch = new_justified_epoch
+        state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch))
+    if new_finalized_epoch != state.finalized_epoch:
+        state.finalized_epoch = new_finalized_epoch
+        state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch))
 ```
 
 #### Crosslinks
@@ -2044,7 +2034,7 @@ def process_ejections(state: BeaconState) -> None:
     """
     for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)):
         if state.validator_balances[index] < EJECTION_BALANCE:
-            exit_validator(state, index)
+            initiate_validator_exit(state, index)
 ```
 
 #### Validator registry and shuffling seed data
@@ -2096,16 +2086,21 @@ def update_validator_registry(state: BeaconState) -> None:
             activate_validator(state, index, is_genesis=False)
 
     # Exit validators within the allowable balance churn
-    balance_churn = 0
-    for index, validator in enumerate(state.validator_registry):
-        if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit:
-            # Check the balance churn would be within the allowance
-            balance_churn += get_effective_balance(state, index)
-            if balance_churn > max_balance_churn:
-                break
+    if current_epoch < state.validator_registry_update_epoch + LATEST_SLASHED_EXIT_LENGTH:
+        balance_churn = (
+            state.latest_slashed_balances[state.validator_registry_update_epoch % LATEST_SLASHED_EXIT_LENGTH] -
+            state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH]
+        )
 
-            # Exit validator
-            exit_validator(state, index)
+        for index, validator in enumerate(state.validator_registry):
+            if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit:
+                # Check the balance churn would be within the allowance
+                balance_churn += get_effective_balance(state, index)
+                if balance_churn > max_balance_churn:
+                    break
+
+                # Exit validator
+                exit_validator(state, index)
 
     state.validator_registry_update_epoch = current_epoch
 ```
@@ -2307,8 +2302,8 @@ def process_proposer_slashing(state: BeaconState,
     Note that this function mutates ``state``.
     """
     proposer = state.validator_registry[proposer_slashing.proposer_index]
-    # Verify that the slot is the same
-    assert proposer_slashing.header_1.slot == proposer_slashing.header_2.slot
+    # Verify that the epoch is the same
+    assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot)
     # But the headers are different
     assert proposer_slashing.header_1 != proposer_slashing.header_2
     # Proposer is not yet slashed
@@ -2371,75 +2366,49 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
     Process ``Attestation`` transaction.
     Note that this function mutates ``state``.
     """
-    # Can't submit attestations that are too far in history (or in prehistory) 
-    assert attestation.data.slot >= GENESIS_SLOT
-    assert state.slot <= attestation.data.slot + SLOTS_PER_EPOCH
-    # Can't submit attestations too quickly
-    assert attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot
-    # Verify that the justified epoch is correct, case 1: current epoch attestations
-    if slot_to_epoch(attestation.data.slot) >= get_current_epoch(state):
-        assert attestation.data.justified_epoch == state.justified_epoch
-    # Case 2: previous epoch attestations
-    else:
-        assert attestation.data.justified_epoch == state.previous_justified_epoch
-    # Check that the justified block root is correct
-    assert attestation.data.justified_block_root == get_block_root(
-        state, get_epoch_start_slot(attestation.data.justified_epoch)
-    )
-    # Check that the crosslink data is valid
-    acceptable_crosslink_data = {
-        # Case 1: Latest crosslink matches the one in the state
-        attestation.data.latest_crosslink,
-        # Case 2: State has already been updated, state's latest crosslink matches the crosslink
-        # the attestation is trying to create
-        Crosslink(
-            crosslink_data_root=attestation.data.crosslink_data_root,
-            epoch=slot_to_epoch(attestation.data.slot)
-        )
-    }
-    assert state.latest_crosslinks[attestation.data.shard] in acceptable_crosslink_data
-    # Attestation must be nonempty!
-    assert attestation.aggregation_bitfield != b'\x00' * len(attestation.aggregation_bitfield)
-    # Custody must be empty (to be removed in phase 1)
-    assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield)
-    # Get the committee for the specific shard that this attestation is for
-    crosslink_committee = [
-        committee for committee, shard in get_crosslink_committees_at_slot(state, attestation.data.slot)
-        if shard == attestation.data.shard
-    ][0]
-    # Custody bitfield must be a subset of the attestation bitfield
-    for i in range(len(crosslink_committee)):
-        if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b0:
-            assert get_bitfield_bit(attestation.custody_bitfield, i) == 0b0
-    # Verify aggregate signature
-    participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield)
-    custody_bit_1_participants = get_attestation_participants(state, attestation.data, attestation.custody_bitfield)
-    custody_bit_0_participants = [i for i in participants if i not in custody_bit_1_participants]
+    assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot
+    assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY
 
-    assert bls_verify_multiple(
-        pubkeys=[
-            bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_participants]),
-            bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_participants]),
-        ],
-        message_hashes=[
-            hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)),
-            hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b1)),
-        ],
+    # Check target epoch, source epoch, and source root
+    target_epoch = slot_to_epoch(attestation.data.slot)
+    assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in {
+        (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), 
+        (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root),
+    }
+
+    # Check crosslink data
+    assert attestation.data.crosslink_data_root == ZERO_HASH  # [to be removed in phase 1]
+    assert state.latest_crosslinks[attestation.data.shard] in {
+        attestation.data.previous_crosslink,  # Case 1: latest crosslink matches previous crosslink
+        Crosslink(                            # Case 2: latest crosslink matches current crosslink
+            crosslink_data_root=attestation.data.crosslink_data_root,
+            epoch=target_epoch,
+        ),
+    }
+
+    # Check custody bits [to be generalised in phase 1]
+    assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield)
+
+    # Check aggregate signature [to be generalised in phase 1]
+    participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield)
+    assert len(participants) != 0
+    assert bls_verify(
+        pubkey=bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in participants]),
+        message_hash=hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)),
         signature=attestation.aggregate_signature,
-        domain=get_domain(state.fork, slot_to_epoch(attestation.data.slot), DOMAIN_ATTESTATION),
+        domain=get_domain(state.fork, target_epoch, DOMAIN_ATTESTATION),
     )
-    # Crosslink data root is zero (to be removed in phase 1)
-    assert attestation.data.crosslink_data_root == ZERO_HASH
-    # Apply the attestation
+
+    # Cache pending attestation
     pending_attestation = PendingAttestation(
         data=attestation.data,
         aggregation_bitfield=attestation.aggregation_bitfield,
         custody_bitfield=attestation.custody_bitfield,
         inclusion_slot=state.slot
     )
-    if slot_to_epoch(attestation.data.slot) == get_current_epoch(state):
+    if target_epoch == get_current_epoch(state):
         state.current_epoch_attestations.append(pending_attestation)
-    elif slot_to_epoch(attestation.data.slot) == get_previous_epoch(state):
+    else:
         state.previous_epoch_attestations.append(pending_attestation)
 ```
 

specs/core/1_shard-data-chains.md

@@ -17,41 +17,51 @@ At the current stage, Phase 1, while fundamentally feature-complete, is still su
             - [Time parameters](#time-parameters)
             - [Max operations per block](#max-operations-per-block)
             - [Signature domains](#signature-domains)
-    - [Shard chains and crosslink data](#shard-chains-and-crosslink-data)
-      - [Helper functions](#helper-functions)
+- [Shard chains and crosslink data](#shard-chains-and-crosslink-data)
+    - [Helper functions](#helper-functions)
             - [`get_split_offset`](#get_split_offset)
             - [`get_shuffled_committee`](#get_shuffled_committee)
             - [`get_persistent_committee`](#get_persistent_committee)
             - [`get_shard_proposer_index`](#get_shard_proposer_index)
-      - [Data Structures](#data-structures)
+    - [Data Structures](#data-structures)
         - [Shard chain blocks](#shard-chain-blocks)
-      - [Shard block processing](#shard-block-processing)
+    - [Shard block processing](#shard-block-processing)
         - [Verifying shard block data](#verifying-shard-block-data)
         - [Verifying a crosslink](#verifying-a-crosslink)
         - [Shard block fork choice rule](#shard-block-fork-choice-rule)
-    - [Updates to the beacon chain](#updates-to-the-beacon-chain)
-      - [Data structures](#data-structures)
+- [Updates to the beacon chain](#updates-to-the-beacon-chain)
+    - [Data structures](#data-structures)
         - [`Validator`](#validator)
         - [`BeaconBlockBody`](#beaconblockbody)
+        - [`BeaconState`](#beaconstate)
         - [`BranchChallenge`](#branchchallenge)
         - [`BranchResponse`](#branchresponse)
         - [`BranchChallengeRecord`](#branchchallengerecord)
+        - [`InteractiveCustodyChallengeRecord`](#interactivecustodychallengerecord)
+        - [`InteractiveCustodyChallengeInitiation`](#interactivecustodychallengeinitiation)
+        - [`InteractiveCustodyChallengeResponse`](#interactivecustodychallengeresponse)
+        - [`InteractiveCustodyChallengeContinuation`](#interactivecustodychallengecontinuation)
         - [`SubkeyReveal`](#subkeyreveal)
     - [Helpers](#helpers)
-        - [`get_attestation_data_merkle_depth`](#get_attestation_data_merkle_depth)
+        - [`get_branch_challenge_record_by_id`](#get_branch_challenge_record_by_id)
+        - [`get_custody_challenge_record_by_id`](#get_custody_challenge_record_by_id)
+        - [`get_attestation_merkle_depth`](#get_attestation_merkle_depth)
         - [`epoch_to_custody_period`](#epoch_to_custody_period)
         - [`slot_to_custody_period`](#slot_to_custody_period)
         - [`get_current_custody_period`](#get_current_custody_period)
         - [`verify_custody_subkey_reveal`](#verify_custody_subkey_reveal)
-        - [`prepare_validator_for_withdrawal`](#prepare_validator_for_withdrawal)
+        - [`verify_signed_challenge_message`](#verify_signed_challenge_message)
         - [`penalize_validator`](#penalize_validator)
-      - [Per-slot processing](#per-slot-processing)
+    - [Per-slot processing](#per-slot-processing)
         - [Operations](#operations)
             - [Branch challenges](#branch-challenges)
             - [Branch responses](#branch-responses)
             - [Subkey reveals](#subkey-reveals)
-      - [Per-epoch processing](#per-epoch-processing)
-      - [One-time phase 1 initiation transition](#one-time-phase-1-initiation-transition)
+            - [Interactive custody challenge initiations](#interactive-custody-challenge-initiations)
+            - [Interactive custody challenge responses](#interactive-custody-challenge-responses)
+            - [Interactive custody challenge continuations](#interactive-custody-challenge-continuations)
+    - [Per-epoch processing](#per-epoch-processing)
+    - [One-time phase 1 initiation transition](#one-time-phase-1-initiation-transition)
 
 <!-- /TOC -->
 
@@ -118,9 +128,9 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md
 def get_split_offset(list_size: int, chunks: int, index: int) -> int:
   """
   Returns a value such that for a list L, chunk count k and index i,
-  split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k+1, i)]
+  split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k, i+1)]
   """
-  return (len(list_size) * index) // chunks
+  return (list_size * index) // chunks
 ````
 
 #### `get_shuffled_committee`
@@ -128,16 +138,27 @@ def get_split_offset(list_size: int, chunks: int, index: int) -> int:
 ```python
 def get_shuffled_committee(state: BeaconState,
                            shard: Shard,
-                           committee_start_epoch: Epoch) -> List[ValidatorIndex]:
+                           committee_start_epoch: Epoch,
+                           index: int,
+                           committee_count: int) -> List[ValidatorIndex]:
     """
     Return shuffled committee.
     """
-    validator_indices = get_active_validator_indices(state.validators, committee_start_epoch)
+    active_validator_indices = get_active_validator_indices(state.validator_registry, committee_start_epoch)
+    length = len(active_validator_indices)
     seed = generate_seed(state, committee_start_epoch)
-    start_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard)
-    end_offset = get_split_offset(len(validator_indices), SHARD_COUNT, shard + 1)
+    start_offset = get_split_offset(
+        length,
+        SHARD_COUNT * committee_count,
+        shard * committee_count + index,
+    )
+    end_offset = get_split_offset(
+        length,
+        SHARD_COUNT * committee_count,
+        shard * committee_count + index + 1,
+    )
     return [
-        validator_indices[get_permuted_index(i, len(validator_indices), seed)]
+        active_validator_indices[get_permuted_index(i, length, seed)]
         for i in range(start_offset, end_offset)
     ]
 ```
@@ -147,15 +168,24 @@ def get_shuffled_committee(state: BeaconState,
 ```python
 def get_persistent_committee(state: BeaconState,
                              shard: Shard,
-                             epoch: Epoch) -> List[ValidatorIndex]:
+                             slot: Slot) -> List[ValidatorIndex]:
     """
-    Return the persistent committee for the given ``shard`` at the given ``epoch``.
+    Return the persistent committee for the given ``shard`` at the given ``slot``.
     """
-    earlier_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2
-    earlier_committee = get_shuffled_committee(state, shard, earlier_committee_start_epoch)
+        
+    earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2
+    later_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD
 
-    later_committee_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD
-    later_committee = get_shuffled_committee(state, shard, later_committee_start_epoch)
+    committee_count = max(
+        len(get_active_validator_indices(state.validator_registry, earlier_start_epoch)) //
+        (SHARD_COUNT * TARGET_COMMITTEE_SIZE),
+        len(get_active_validator_indices(state.validator_registry, later_start_epoch)) //
+        (SHARD_COUNT * TARGET_COMMITTEE_SIZE),
+    ) + 1
+    
+    index = slot % committee_count
+    earlier_committee = get_shuffled_committee(state, shard, earlier_start_epoch, index, committee_count)
+    later_committee = get_shuffled_committee(state, shard, later_start_epoch, index, committee_count)
 
     def get_switchover_epoch(index):
         return (
@@ -170,6 +200,7 @@ def get_persistent_committee(state: BeaconState,
         [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)]
     )))
 ```
+
 #### `get_shard_proposer_index`
 
 ```python
@@ -181,14 +212,14 @@ def get_shard_proposer_index(state: BeaconState,
         int_to_bytes8(shard) +
         int_to_bytes8(slot)
     )
-    persistent_committee = get_persistent_committee(state, shard, slot_to_epoch(slot))
+    persistent_committee = get_persistent_committee(state, shard, slot)
     # Default proposer
     index = bytes_to_int(seed[0:8]) % len(persistent_committee)
     # If default proposer exits, try the other proposers in order; if all are exited
     # return None (ie. no block can be proposed)
     validators_to_try = persistent_committee[index:] + persistent_committee[:index]
     for index in validators_to_try:
-        if is_active_validator(state.validators[index], get_current_epoch(state)):
+        if is_active_validator(state.validator_registry[index], get_current_epoch(state)):
             return index
     return None
 ```
@@ -233,14 +264,14 @@ To validate a block header on shard `shard_block.shard_id`, compute as follows:
 * Verify that `shard_block.beacon_chain_ref` is the hash of a block in the (canonical) beacon chain with slot less than or equal to `slot`.
 * Verify that `shard_block.beacon_chain_ref` is equal to or a descendant of the `shard_block.beacon_chain_ref` specified in the `ShardBlock` pointed to by `shard_block.parent_root`.
 * Let `state` be the state of the beacon chain block referred to by `shard_block.beacon_chain_ref`.
-* Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, slot_to_epoch(shard_block.slot))`.
+* Let `persistent_committee = get_persistent_committee(state, shard_block.shard_id, shard_block.slot)`.
 * Assert `verify_bitfield(shard_block.participation_bitfield, len(persistent_committee))`
-* For every `i in range(len(persistent_committee))` where `is_active_validator(state.validators[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0`
+* For every `i in range(len(persistent_committee))` where `is_active_validator(state.validator_registry[persistent_committee[i]], get_current_epoch(state))` returns `False`, verify that `get_bitfield_bit(shard_block.participation_bitfield, i) == 0`
 * Let `proposer_index = get_shard_proposer_index(state, shard_block.shard_id, shard_block.slot)`.
 * Verify that `proposer_index` is not `None`.
 * Let `msg` be the `shard_block` but with `shard_block.signature` set to `[0, 0]`.
 * Verify that `bls_verify(pubkey=validators[proposer_index].pubkey, message_hash=hash(msg), signature=shard_block.signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_PROPOSER))` passes.
-* Let `group_public_key = bls_aggregate_pubkeys([state.validators[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`.
+* Let `group_public_key = bls_aggregate_pubkeys([state.validator_registry[index].pubkey for i, index in enumerate(persistent_committee) if get_bitfield_bit(shard_block.participation_bitfield, i) is True])`.
 * Verify that `bls_verify(pubkey=group_public_key, message_hash=shard_block.parent_root, sig=shard_block.aggregate_signature, domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER))` passes.
 
 ### Verifying shard block data
@@ -556,7 +587,7 @@ def verify_custody_subkey_reveal(pubkey: bytes48,
 ```python
 def verify_signed_challenge_message(message: Any, pubkey: bytes48) -> bool:
     return bls_verify(
-        message_hash=signed_root(message, 'signature'),
+        message_hash=signed_root(message),
         pubkey=pubkey,
         signature=message.signature,
         domain=get_domain(state, get_current_epoch(state), DOMAIN_CUSTODY_INTERACTIVE)
@@ -607,8 +638,8 @@ Verify that `len(block.body.branch_challenges) <= MAX_BRANCH_CHALLENGES`.
 For each `challenge` in `block.body.branch_challenges`, run:
 
 ```python
-def process_branch_challenge(challenge: BranchChallenge,
-                             state: BeaconState):
+def process_branch_challenge(state: BeaconState,
+                             challenge: BranchChallenge) -> None:
     # Check that it's not too late to challenge
     assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_BRANCH_CHALLENGE_DELAY
     assert state.validator_registry[responder_index].exit_epoch >= get_current_epoch(state) - MAX_BRANCH_CHALLENGE_DELAY
@@ -643,8 +674,8 @@ Verify that `len(block.body.branch_responses) <= MAX_BRANCH_RESPONSES`.
 For each `response` in `block.body.branch_responses`, if `response.responding_to_custody_challenge == False`, run:
 
 ```python
-def process_branch_exploration_response(response: BranchResponse,
-                                        state: BeaconState):
+def process_branch_exploration_response(state: BeaconState,
+                                        response: BranchResponse) -> None:
     challenge = get_branch_challenge_record_by_id(response.challenge_id)
     assert verify_merkle_branch(
         leaf=response.data,
@@ -664,8 +695,8 @@ def process_branch_exploration_response(response: BranchResponse,
 If `response.responding_to_custody_challenge == True`, run:
 
 ```python
-def process_branch_custody_response(response: BranchResponse,
-                                    state: BeaconState):
+def process_branch_custody_response(state: BeaconState,
+                                    response: BranchResponse) -> None:
     challenge = get_custody_challenge_record_by_id(response.challenge_id)
     responder = state.validator_registry[challenge.responder_index]
     # Verify we're not too late
@@ -718,8 +749,8 @@ Verify that `len(block.body.interactive_custody_challenge_initiations) <= MAX_IN
 For each `initiation` in `block.body.interactive_custody_challenge_initiations`, use the following function to process it:
 
 ```python
-def process_initiation(initiation: InteractiveCustodyChallengeInitiation,
-                       state: BeaconState):
+def process_initiation(state: BeaconState,
+                       initiation: InteractiveCustodyChallengeInitiation) -> None:
     challenger = state.validator_registry[initiation.challenger_index]
     responder = state.validator_registry[initiation.responder_index]
     # Verify the signature
@@ -771,8 +802,8 @@ Verify that `len(block.body.interactive_custody_challenge_responses) <= MAX_INTE
 For each `response` in `block.body.interactive_custody_challenge_responses`, use the following function to process it:
 
 ```python
-def process_response(response: InteractiveCustodyChallengeResponse,
-                     state: State):
+def process_response(state: BeaconState,
+                     response: InteractiveCustodyChallengeResponse) -> None:
     challenge = get_custody_challenge_record_by_id(state, response.challenge_id)
     responder = state.validator_registry[challenge.responder_index]
     # Check that the right number of hashes was provided
@@ -804,8 +835,8 @@ Verify that `len(block.body.interactive_custody_challenge_continuations) <= MAX_
 For each `continuation` in `block.body.interactive_custody_challenge_continuations`, use the following function to process it:
 
 ```python
-def process_continuation(continuation: InteractiveCustodyChallengeContinuation,
-                         state: State):
+def process_continuation(state: BeaconState,
+                         continuation: InteractiveCustodyChallengeContinuation) -> None:
     challenge = get_custody_challenge_record_by_id(state, continuation.challenge_id)
     challenger = state.validator_registry[challenge.challenger_index]
     responder = state.validator_registry[challenge.responder_index]

specs/light_client/merkle_proofs.md

@@ -0,0 +1,134 @@
+### Generalized Merkle tree index
+
+In a binary Merkle tree, we define a "generalized index" of a node as `2**depth + index`. Visually, this looks as follows:
+
+```
+    1
+ 2     3
+4 5   6 7
+   ...
+```
+
+Note that the generalized index has the convenient property that the two children of node `k` are `2k` and `2k+1`, and also that it equals the position of a node in the linear representation of the Merkle tree that's computed by this function:
+
+```python
+def merkle_tree(leaves):
+    o = [0] * len(leaves) + leaves
+    for i in range(len(leaves)-1, 0, -1):
+        o[i] = hash(o[i*2] + o[i*2+1])
+    return o
+```
+
+We will define Merkle proofs in terms of generalized indices.
+
+### SSZ object to index
+
+We can describe the hash tree of any SSZ object, rooted in `hash_tree_root(object)`, as a binary Merkle tree whose depth may vary. For example, an object `{x: bytes32, y: List[uint64]}` would look as follows:
+
+```
+     root
+    /    \
+   x    y_root
+        /    \
+y_data_root  len(y)
+    / \
+   /\ /\
+  .......
+```
+
+We can now define a concept of a "path", a way of describing a function that takes as input an SSZ object and outputs some specific (possibly deeply nested) member. For example, `foo -> foo.x` is a path, as are `foo -> len(foo.y)` and `foo -> foo[5]`. We'll describe paths as lists: in these three cases they are `["x"]`, `["y", "len"]` and `["y", 5]` respectively. We can now define a function `get_generalized_indices(object: Any, path: List[str OR int], root=1: int) -> int` that converts an object and a path to a set of generalized indices (note that for constant-sized objects, there is only one generalized index and it only depends on the path, but for dynamically sized objects the indices may depend on the object itself too). For dynamically-sized objects, the set of indices will have more than one member because of the need to access an array's length to determine the correct generalized index for some array access.
+
+```python
+def get_generalized_indices(obj: Any, path: List[str or int], root=1) -> List[int]:
+    if len(path) == 0:
+        return [root]
+    elif isinstance(obj, StaticList):
+        items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1
+        new_root = root * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk
+        return get_generalized_indices(obj[path[0]], path[1:], new_root)
+    elif isinstance(obj, DynamicList) and path[0] == "len":
+        return [root * 2 + 1]
+    elif isinstance(obj, DynamicList) and isinstance(path[0], int):
+        assert path[0] < len(obj)
+        items_per_chunk = (32 // len(serialize(x))) if isinstance(x, int) else 1
+        new_root = root * 2 * next_power_of_2(len(obj) // items_per_chunk) + path[0] // items_per_chunk
+        return [root *2 + 1] + get_generalized_indices(obj[path[0]], path[1:], new_root)
+    elif hasattr(obj, "fields"):
+        index = list(fields.keys()).index(path[0])
+        new_root = root * next_power_of_2(len(fields)) + index
+        return get_generalized_indices(getattr(obj, path[0]), path[1:], new_root)
+    else:
+        raise Exception("Unknown type / path")
+```
+
+### Merkle multiproofs
+
+We define a Merkle multiproof as a minimal subset of nodes in a Merkle tree needed to fully authenticate that a set of nodes actually are part of a Merkle tree with some specified root, at a particular set of generalized indices. For example, here is the Merkle multiproof for positions 0, 1, 6 in an 8-node Merkle tree (ie. generalized indices 8, 9, 14):
+
+```
+       .
+   .       .
+ .   *   *   .
+x x . . . . x *
+```
+
+. are unused nodes, * are used nodes, x are the values we are trying to prove. Notice how despite being a multiproof for 3 values, it requires only 3 auxiliary nodes, only one node more than would be required to prove a single value. Normally the efficiency gains are not quite that extreme, but the savings relative to individual Merkle proofs are still significant. As a rule of thumb, a multiproof for k nodes at the same level of an n-node tree has size `k * (n/k + log(n/k))`.
+
+Here is code for creating and verifying a multiproof. First a helper:
+
+```python
+def log2(x):
+    return 0 if x == 1 else 1 + log2(x//2)
+```
+
+First, a method for computing the generalized indices of the auxiliary tree nodes that a proof of a given set of generalized indices will require:
+
+```python
+def get_proof_indices(tree_indices: List[int]) -> List[int]:
+    # Get all indices touched by the proof
+    maximal_indices = set({})
+    for i in tree_indices:
+        x = i
+        while x > 1:
+            maximal_indices.add(x ^ 1)
+            x //= 2
+    maximal_indices = tree_indices + sorted(list(maximal_indices))[::-1]
+    # Get indices that cannot be recalculated from earlier indices
+    redundant_indices = set({})
+    proof = []
+    for index in maximal_indices:
+        if index not in redundant_indices:
+            proof.append(index)
+            while index > 1:
+                redundant_indices.add(index)
+                if (index ^ 1) not in redundant_indices:
+                    break
+                index //= 2
+    return [i for i in proof if i not in tree_indices]
+````
+
+Generating a proof is simply a matter of taking the node of the SSZ hash tree with the union of the given generalized indices for each index given by `get_proof_indices`, and outputting the list of nodes in the same order.
+
+```python
+def verify_multi_proof(root, indices, leaves, proof):
+    tree = {}
+    for index, leaf in zip(indices, leaves):
+        tree[index] = leaf
+    for index, proofitem in zip(get_proof_indices(indices), proof):
+        tree[index] = proofitem
+    indexqueue = sorted(tree.keys())[:-1]
+    i = 0
+    while i < len(indexqueue):
+        index = indexqueue[i]
+        if index >= 2 and index^1 in tree:
+            tree[index//2] = hash(tree[index - index%2] + tree[index - index%2 + 1])
+            indexqueue.append(index//2)
+        i += 1
+    return (indices == []) or (1 in tree and tree[1] == root)
+```
+
+#### Proofs for execution
+
+We define `MerklePartial(f, arg1, arg2...)` as being a list of Merkle multiproofs of the sets of nodes in the hash trees of the SSZ objects that are needed to authenticate the values needed to compute some function `f(arg1, arg2...)`. An individual Merkle multiproof is given as a dynamic sized list of `bytes32` values, a `MerklePartial` is a fixed-size list of objects `{proof: ["bytes32"], value: "bytes32"}`, one for each `arg` to `f` (if some `arg` is a base type, then the multiproof is empty).
+
+Ideally, any function which accepts an SSZ object should also be able to accept a `MerklePartial` object as a substitute.

specs/light_client/sync_protocol.md

@@ -0,0 +1,172 @@
+# Beacon chain light client syncing
+
+One of the design goals of the eth2 beacon chain is light-client friendlines, both to allow low-resource clients (mobile phones, IoT, etc) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the eth2 beacon chain and other chains.
+
+### Preliminaries
+
+We define an "expansion" of an object as an object where a field in an object that is meant to represent the `hash_tree_root` of another object is replaced by the object. Note that defining expansions is not a consensus-layer-change; it is merely a "re-interpretation" of the object. Particularly, the `hash_tree_root` of an expansion of an object is identical to that of the original object, and we can define expansions where, given a complete history, it is always possible to compute the expansion of any object in the history. The opposite of an expansion is a "summary" (eg. `BeaconBlockHeader` is a summary of `BeaconBlock`).
+
+We define two expansions:
+
+* `ExtendedBeaconBlock`, which is identical to a `BeaconBlock` except `state_root` is replaced with the corresponding `state: ExtendedBeaconState`
+* `ExtendedBeaconState`, which is identical to a `BeaconState` except `latest_active_index_roots: List[Bytes32]` is replaced by `latest_active_indices: List[List[ValidatorIndex]]`, where `BeaconState.latest_active_index_roots[i] = hash_tree_root(ExtendedBeaconState.latest_active_indices[i])`
+
+Note that there is now a new way to compute `get_active_validator_indices`:
+
+```python
+def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]:
+    return state.latest_active_indices[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH]
+```
+
+Note that it takes `state` instead of `state.validator_registry` as an argument. This does not affect its use in `get_shuffled_committee`, because `get_shuffled_committee` has access to the full `state` as one of its arguments.
+
+A `MerklePartial(f, *args)` is an object that contains a minimal Merkle proof needed to compute `f(*args)`. A `MerklePartial` can be used in place of a regular SSZ object, though a computation would return an error if it attempts to access part of the object that is not contained in the proof.
+
+We add a data type `PeriodData` and four helpers:
+
+```python
+{
+    'validator_count': 'uint64',
+    'seed': 'bytes32',
+    'committee': [Validator]
+}
+```
+
+```python
+def get_earlier_start_epoch(slot: Slot) -> int:
+    return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD * 2
+
+def get_later_start_epoch(slot: Slot) -> int:
+    return slot - slot % PERSISTENT_COMMITTEE_PERIOD - PERSISTENT_COMMITTEE_PERIOD
+    
+def get_earlier_period_data(block: ExtendedBeaconBlock, shard_id: Shard) -> PeriodData:
+    period_start = get_earlier_start_epoch(header.slot)
+    validator_count = len(get_active_validator_indices(state, period_start))
+    committee_count = validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE) + 1
+    indices = get_shuffled_committee(block.state, shard_id, period_start, 0, committee_count)
+    return PeriodData(
+        validator_count,
+        generate_seed(block.state, period_start),
+        [block.state.validator_registry[i] for i in indices]
+    )
+    
+def get_later_period_data(block: ExtendedBeaconBlock, shard_id: Shard) -> PeriodData:
+    period_start = get_later_start_epoch(header.slot)
+    validator_count = len(get_active_validator_indices(state, period_start))
+    committee_count = validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE) + 1
+    indices = get_shuffled_committee(block.state, shard_id, period_start, 0, committee_count)
+    return PeriodData(
+        validator_count,
+        generate_seed(block.state, period_start),
+        [block.state.validator_registry[i] for i in indices]
+    )
+```
+
+### Light client state
+
+A light client will keep track of:
+
+* A random `shard_id` in `[0...SHARD_COUNT-1]` (selected once and retained forever)
+* A block header that they consider to be finalized (`finalized_header`) and do not expect to revert.
+* `later_period_data = get_maximal_later_committee(finalized_header, shard_id)`
+* `earlier_period_data = get_maximal_earlier_committee(finalized_header, shard_id)`
+
+We use the struct `validator_memory` to keep track of these variables.
+
+### Updating the shuffled committee
+
+If a client's `validator_memory.finalized_header` changes so that `header.slot // PERSISTENT_COMMITTEE_PERIOD` increases, then the client can ask the network for a `new_committee_proof = MerklePartial(get_maximal_later_committee, validator_memory.finalized_header, shard_id)`. It can then compute:
+
+```python
+earlier_period_data = later_period_data
+later_period_data = get_later_period_data(new_committee_proof, finalized_header, shard_id)
+```
+
+The maximum size of a proof is `128 * ((22-7) * 32 + 110) = 75520` bytes for validator records and `(22-7) * 32 + 128 * 8 = 1504` for the active index proof (much smaller because the relevant active indices are all beside each other in the Merkle tree). This needs to be done once per `PERSISTENT_COMMITTEE_PERIOD` epochs (2048 epochs / 9 days), or ~38 bytes per epoch.
+
+### Computing the current committee
+
+Here is a helper to compute the committee at a slot given the maximal earlier and later committees:
+
+```python
+def compute_committee(header: BeaconBlockHeader,
+                      validator_memory: ValidatorMemory):
+                      
+    earlier_validator_count = validator_memory.earlier_period_data.validator_count
+    later_validator_count = validator_memory.later_period_data.validator_count
+    earlier_committee = validator_memory.earlier_period_data.committee
+    later_committee = validator_memory.later_period_data.committee
+    earlier_start_epoch = get_earlier_start_epoch(header.slot)
+    later_start_epoch = get_later_start_epoch(header.slot)
+    epoch = slot_to_epoch(header.slot)
+    
+    actual_committee_count = max(
+        earlier_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE),
+        later_validator_count // (SHARD_COUNT * TARGET_COMMITTEE_SIZE),
+    ) + 1
+    
+    def get_offset(count, end:bool):
+        return get_split_offset(count,
+                                SHARD_COUNT * committee_count,
+                                validator_memory.shard_id * committee_count + (1 if end else 0))
+                                
+    actual_earlier_committee = maximal_earlier_committee[
+        0:get_offset(earlier_validator_count, True) - get_offset(earlier_validator_count, False)
+    ]
+    actual_later_committee = maximal_later_committee[
+        0:get_offset(later_validator_count, True) - get_offset(later_validator_count, False)
+    ]
+    def get_switchover_epoch(index):
+        return (
+            bytes_to_int(hash(validator_memory.earlier_period_data.seed + bytes3(index))[0:8]) %
+            PERSISTENT_COMMITTEE_PERIOD
+        )
+
+    # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from
+    # later committee; return a sorted list of the union of the two, deduplicated
+    return sorted(list(set(
+        [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(i)] +
+        [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(i)]
+    )))
+                    
+```
+
+Note that this method makes use of the fact that the committee for any given shard always starts and ends at the same validator index independently of the committee count (this is because the validator set is split into `SHARD_COUNT * committee_count` slices but the first slice of a shard is a multiple `committee_count * i`, so the start of the slice is `n * committee_count * i // (SHARD_COUNT * committee_count) = n * i // SHARD_COUNT`, using the slightly nontrivial algebraic identity `(x * a) // ab == x // b`).
+
+### Verifying blocks
+
+If a client wants to update its `finalized_header` it asks the network for a `BlockValidityProof`, which is simply:
+
+```python
+{
+    'header': BlockHeader,
+    'shard_aggregate_signature': 'bytes96',
+    'shard_bitfield': 'bytes',
+    'shard_parent_block': ShardBlock
+}
+```
+
+The verification procedure is as follows:
+
+```python
+def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: ValidatorMemory) -> bool:
+    assert proof.shard_parent_block.beacon_chain_ref == hash_tree_root(proof.header)
+    committee = compute_committee(proof.header, validator_memory)
+    # Verify that we have >=50% support
+    support_balance = sum([c.high_balance for i, c in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True])
+    total_balance = sum([c.high_balance for i, c in enumerate(committee)]
+    assert support_balance * 2 > total_balance
+    # Verify shard attestations
+    group_public_key = bls_aggregate_pubkeys([
+        v.pubkey for v, index in enumerate(committee) if
+        get_bitfield_bit(proof.shard_bitfield, i) is True
+    ])
+    assert bls_verify(
+        pubkey=group_public_key,
+        message_hash=hash_tree_root(shard_parent_block),
+        signature=shard_aggregate_signature,
+        domain=get_domain(state, slot_to_epoch(shard_block.slot), DOMAIN_SHARD_ATTESTER)
+    )
+```
+
+The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_ref, ShardBlock)`, which would cut off ~220 bytes.

specs/simple-serialize.md

@@ -12,7 +12,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio
 - [Serialization](#serialization)
     - [`"uintN"`](#uintn)
     - [`"bool"`](#bool)
-    - [Tuples, containers, lists](#tuples-containers-lists)
+    - [Vectors, containers, lists](#vectors-containers-lists)
 - [Deserialization](#deserialization)
 - [Merkleization](#merkleization)
 - [Self-signed containers](#self-signed-containers)
@@ -34,12 +34,14 @@ This is a **work in progress** describing typing, serialization and Merkleizatio
 ### Composite types
 
 * **container**: ordered heterogenous collection of values
-    * key-pair curly bracket notation `{}`, e.g. `{'foo': "uint64", 'bar': "bool"}`
-* **tuple**: ordered fixed-length homogeneous collection of values
+    * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}`
+* **vector**: ordered fixed-length homogeneous collection of values
     * angle bracket notation `[type, N]`, e.g. `["uint64", N]`
 * **list**: ordered variable-length homogenous collection of values
     * angle bracket notation `[type]`, e.g. `["uint64"]`
 
+We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size".
+
 ### Aliases
 
 For convenience we alias:
@@ -54,34 +56,34 @@ We recursively define the `serialize` function which consumes an object `value`
 
 *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signed_root`, etc.) objects implicitly carry their type.
 
-### `uintN`
+### `"uintN"`
 
 ```python
 assert N in [8, 16, 32, 64, 128, 256]
-return value.to_bytes(N // 8, 'little')
+return value.to_bytes(N // 8, "little")
 ```
 
-### `bool`
+### `"bool"`
 
 ```python
 assert value in (True, False)
-return b'\x01' if value is True else b'\x00'
+return b"\x01" if value is True else b"\x00"
 ```
 
-### Tuples, containers, lists
+### Vectors, containers, lists
 
-If `value` is fixed-length (i.e. does not embed a list):
+If `value` is fixed-size:
 
 ```python
-return ''.join([serialize(element) for element in value])
+return "".join([serialize(element) for element in value])
 ```
 
-If `value` is variable-length (i.e. embeds a list):
+If `value` is variable-size:
 
 ```python
-serialized_bytes = ''.join([serialize(element) for element in value])
+serialized_bytes = "".join([serialize(element) for element in value])
 assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX)
-serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
+serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, "little")
 return serialized_length + serialized_bytes
 ```
 
@@ -99,9 +101,9 @@ We first define helper functions:
 
 We now define Merkleization `hash_tree_root(value)` of an object `value` recursively:
 
-* `merkleize(pack(value))` if `value` is a basic object or a tuple of basic objects
+* `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects
 * `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects
-* `merkleize([hash_tree_root(element) for element in value])` if `value` is a tuple of composite objects or a container
+* `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container
 * `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects
 
 ## Self-signed containers

specs/validator/0_beacon-chain-validator.md

@@ -40,11 +40,11 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers
                 - [Slot](#slot-1)
                 - [Shard](#shard)
                 - [Beacon block root](#beacon-block-root)
-                - [Epoch boundary root](#epoch-boundary-root)
+                - [Target root](#target-root)
                 - [Crosslink data root](#crosslink-data-root)
                 - [Latest crosslink](#latest-crosslink)
-                - [Justified epoch](#justified-epoch)
-                - [Justified block root](#justified-block-root)
+                - [Source epoch](#source-epoch)
+                - [Source root](#source-root)
             - [Construct attestation](#construct-attestation)
                 - [Data](#data)
                 - [Aggregation bitfield](#aggregation-bitfield)
@@ -101,8 +101,7 @@ In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW
 To submit a deposit:
 
 * Pack the validator's [initialization parameters](#initialization) into `deposit_input`, a [`DepositInput`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#depositinput) SSZ object.
-* Set `deposit_input.proof_of_possession = EMPTY_SIGNATURE`.
-* Let `proof_of_possession` be the result of `bls_sign` of the `hash_tree_root(deposit_input)` with `domain=DOMAIN_DEPOSIT`.
+* Let `proof_of_possession` be the result of `bls_sign` of the `signed_root(deposit_input)` with `domain=DOMAIN_DEPOSIT`.
 * Set `deposit_input.proof_of_possession = proof_of_possession`.
 * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_DEPOSIT_AMOUNT`.
 * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `deposit` along with `serialize(deposit_input)` as the singular `bytes` input along with a deposit `amount` in Gwei.
@@ -121,11 +120,12 @@ Once a validator has been processed and added to the beacon state's `validator_r
 
 In normal operation, the validator is quickly activated at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes).
 
-The function [`is_active_validator`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows:
+The function [`is_active_validator`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given shuffling epoch. Note that the `BeaconState` contains a field `current_shuffling_epoch` which dictates from which epoch the current active validators are taken. Usage is as follows:
 
 ```python
+shuffling_epoch = state.current_shuffling_epoch
 validator = state.validator_registry[validator_index]
-is_active = is_active_validator(validator, epoch)
+is_active = is_active_validator(validator, shuffling_epoch)
 ```
 
 Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited.
@@ -138,7 +138,7 @@ A validator has two primary responsibilities to the beacon chain -- [proposing b
 
 ### Block proposal
 
-A validator is expected to propose a [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state, slot)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beacon-chain-state-transition-function).
+A validator is expected to propose a [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state, slot)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beacon-chain-state-transition-function).
 
 There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (eg. at 312500 validators = 10 million ETH, that's once per ~3 weeks).
 
@@ -152,13 +152,13 @@ _Note:_ there might be "skipped" slots between the `parent` and `block`. These s
 
 ##### Parent root
 
-Set `block.parent_root = hash_tree_root(parent)`.
+Set `block.previous_block_root = hash_tree_root(parent)`.
 
 ##### State root
 
 Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the `parent -> block` state transition.
 
-_Note_: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. It is useful to be able to run a state transition function that does _not_ validate signatures for this purpose.
+_Note_: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. It is useful to be able to run a state transition function that does _not_ validate signatures or state root for this purpose.
 
 ##### Randao reveal
 
@@ -166,8 +166,8 @@ Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined a
 
 ```python
 epoch_signature = bls_sign(
-    privkey=validator.privkey,  # privkey store locally, not in state
-    message_hash=int_to_bytes32(slot_to_epoch(block.slot)),
+    privkey=validator.privkey,  # privkey stored locally, not in state
+    message_hash=hash_tree_root(slot_to_epoch(block.slot)),
     domain=get_domain(
         fork=fork,  # `fork` is the fork object at the slot `block.slot`
         epoch=slot_to_epoch(block.slot),
@@ -194,23 +194,16 @@ epoch_signature = bls_sign(
 
 ##### Signature
 
-Set `block.signature = signed_proposal_data` where `signed_proposal_data` is defined as:
+Set `block.signature = block_signature` where `block_signature` is defined as:
 
 ```python
-proposal_data = ProposalSignedData(
-    slot=slot,
-    shard=BEACON_CHAIN_SHARD_NUMBER,
-    block_root=hash_tree_root(block),  # where `block.sigature == EMPTY_SIGNATURE
-)
-proposal_root = hash_tree_root(proposal_data)
-
-signed_proposal_data = bls_sign(
+block_signature = bls_sign(
     privkey=validator.privkey,  # privkey store locally, not in state
-    message_hash=proposal_root,
+    message_hash=signed_root(block),
     domain=get_domain(
         fork=fork,  # `fork` is the fork object at the slot `block.slot`
         epoch=slot_to_epoch(block.slot),
-        domain_type=DOMAIN_PROPOSAL,
+        domain_type=DOMAIN_BEACON_BLOCK,
     )
 )
 ```
@@ -227,12 +220,14 @@ Up to `MAX_ATTESTER_SLASHINGS` [`AttesterSlashing`](https://github.com/ethereum/
 
 ##### Attestations
 
-Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestations-1). To maximize profit, the validator should attempt to create aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain.
+Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestations-1). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain.
 
 ##### Deposits
 
 Up to `MAX_DEPOSITS` [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#deposit) objects can be included in the `block`. These deposits are constructed from the `Deposit` logs from the [Eth1.0 deposit contract](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#ethereum-10-deposit-contract) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#deposits-1).
 
+The `proof` for each deposit must be constructed against the deposit root contained in `state.latest_eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `latest_eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation.
+
 ##### Voluntary exits
 
 Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#exits-1).
@@ -247,9 +242,12 @@ A validator should create and broadcast the attestation halfway through the `slo
 
 First the validator should construct `attestation_data`, an [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot.
 
+* Let `head_block` be the result of running the fork choice during the assigned slot.
+* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot.
+
 ##### Slot
 
-Set `attestation_data.slot = slot` where `slot` is the current slot of which the validator is a member of a committee.
+Set `attestation_data.slot = head_state.slot`.
 
 ##### Shard
 
@@ -257,15 +255,15 @@ Set `attestation_data.shard = shard` where `shard` is the shard associated with
 
 ##### Beacon block root
 
-Set `attestation_data.beacon_block_root = hash_tree_root(head)` where `head` is the validator's view of the `head` block of the beacon chain during `slot`.
+Set `attestation_data.beacon_block_root = hash_tree_root(head_block)`.
 
-##### Epoch boundary root
+##### Target root
 
-Set `attestation_data.epoch_boundary_root = hash_tree_root(epoch_boundary)` where `epoch_boundary` is the block at the most recent epoch boundary in the chain defined by `head` -- i.e. the `BeaconBlock` where `block.slot == get_epoch_start_slot(slot_to_epoch(head.slot))`.
+Set `attestation_data.target_root = hash_tree_root(epoch_boundary)` where `epoch_boundary` is the block at the most recent epoch boundary.
 
 _Note:_ This can be looked up in the state using:
-* Let `epoch_start_slot = get_epoch_start_slot(slot_to_epoch(head.slot))`.
-* Set `epoch_boundary_root = hash_tree_root(head) if epoch_start_slot == head.slot else get_block_root(state, epoch_start_slot)`.
+* Let `epoch_start_slot = get_epoch_start_slot(get_current_epoch(head_state))`.
+* Set `epoch_boundary = head if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`.
 
 ##### Crosslink data root
 
@@ -275,17 +273,15 @@ _Note:_ This is a stub for phase 0.
 
 ##### Latest crosslink
 
-Set `attestation_data.latest_crosslink = state.latest_crosslinks[shard]` where `state` is the beacon state at `head` and `shard` is the validator's assigned shard.
+Set `attestation_data.previous_crosslink = head_state.latest_crosslinks[shard]`.
 
-##### Justified epoch
+##### Source epoch
 
-Set `attestation_data.justified_epoch = state.justified_epoch` where `state` is the beacon state at `head`.
+Set `attestation_data.source_epoch = head_state.justified_epoch`.
 
-##### Justified block root
+##### Source root
 
-Set `attestation_data.justified_block_root = hash_tree_root(justified_block)` where `justified_block` is the block at the slot `get_epoch_start_slot(state.justified_epoch)` in the chain defined by `head`.
-
-_Note:_ This can be looked up in the state using `get_block_root(state, get_epoch_start_slot(state.justified_epoch))`.
+Set `attestation_data.source_root = head_state.current_justified_root`.
 
 #### Construct attestation
 
@@ -320,11 +316,11 @@ attestation_data_and_custody_bit = AttestationDataAndCustodyBit(
     data=attestation.data,
     custody_bit=0b0,
 )
-attestation_message_to_sign = hash_tree_root(attestation_data_and_custody_bit)
+attestation_message = hash_tree_root(attestation_data_and_custody_bit)
 
 signed_attestation_data = bls_sign(
-    privkey=validator.privkey,  # privkey store locally, not in state
-    message_hash=attestation_message_to_sign,
+    privkey=validator.privkey,  # privkey stored locally, not in state
+    message_hash=attestation_message,
     domain=get_domain(
         fork=fork,  # `fork` is the fork object at the slot, `attestation_data.slot`
         epoch=slot_to_epoch(attestation_data.slot),
@@ -402,12 +398,12 @@ _Note_: Signed data must be within a sequential `Fork` context to conflict. Mess
 
 ### Proposer slashing
 
-To avoid "proposer slashings", a validator must not sign two conflicting [`ProposalSignedData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposalsigneddata) where conflicting is defined as having the same `slot` and `shard` but a different `block_root`. In phase 0, proposals are only made for the beacon chain (`shard == BEACON_CHAIN_SHARD_NUMBER`).
+To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposalsigneddata) where conflicting is defined as two distinct blocks within the same epoch.
 
-_In phase 0, as long as the validator does not sign two different beacon chain proposals for the same slot, the validator is safe against proposer slashings._
+_In phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings._
 
 Specifically, when signing an `BeaconBlock`, a validator should perform the following steps in the following order:
-1. Save a record to hard disk that an beacon block has been signed for the `slot=slot` and `shard=BEACON_CHAIN_SHARD_NUMBER`.
+1. Save a record to hard disk that an beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`.
 2. Generate and broadcast the block.
 
 If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the _potentially_ signed/broadcast block and can effectively avoid slashing.
@@ -417,7 +413,7 @@ If the software crashes at some point within this routine, then when the validat
 To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestationdata) objects where conflicting is defined as a set of two attestations that satisfy either [`is_double_vote`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#is_double_vote) or [`is_surround_vote`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#is_surround_vote).
 
 Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order:
-1. Save a record to hard disk that an attestation has been signed for source -- `attestation_data.justified_epoch` -- and target -- `slot_to_epoch(attestation_data.slot)`.
+1. Save a record to hard disk that an attestation has been signed for source -- `attestation_data.source_epoch` -- and target -- `slot_to_epoch(attestation_data.slot)`.
 2. Generate and broadcast attestation.
 
 If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the _potentially_ signed/broadcast attestation and can effectively avoid slashing.

tests/phase0/conftest.py

@@ -0,0 +1,82 @@
+import pytest
+
+from build.phase0 import spec
+
+from tests.phase0.helpers import (
+    privkeys_list,
+    pubkeys_list,
+    create_genesis_state,
+)
+
+
+DEFAULT_CONFIG = {}  # no change
+
+MINIMAL_CONFIG = {
+    "SHARD_COUNT": 8,
+    "MIN_ATTESTATION_INCLUSION_DELAY": 2,
+    "TARGET_COMMITTEE_SIZE": 4,
+    "SLOTS_PER_EPOCH": 8,
+    "GENESIS_EPOCH": spec.GENESIS_SLOT // 8,
+    "SLOTS_PER_HISTORICAL_ROOT": 64,
+    "LATEST_RANDAO_MIXES_LENGTH": 64,
+    "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64,
+    "LATEST_SLASHED_EXIT_LENGTH": 64,
+}
+
+
+@pytest.fixture
+def privkeys():
+    return privkeys_list
+
+
+@pytest.fixture
+def pubkeys():
+    return pubkeys_list
+
+
+def overwrite_spec_config(config):
+    for field in config:
+        setattr(spec, field, config[field])
+        if field == "LATEST_RANDAO_MIXES_LENGTH":
+            spec.BeaconState.fields['latest_randao_mixes'][1] = config[field]
+        elif field == "SHARD_COUNT":
+            spec.BeaconState.fields['latest_crosslinks'][1] = config[field]
+        elif field == "SLOTS_PER_HISTORICAL_ROOT":
+            spec.BeaconState.fields['latest_block_roots'][1] = config[field]
+            spec.BeaconState.fields['latest_state_roots'][1] = config[field]
+            spec.HistoricalBatch.fields['block_roots'][1] = config[field]
+            spec.HistoricalBatch.fields['state_roots'][1] = config[field]
+        elif field == "LATEST_ACTIVE_INDEX_ROOTS_LENGTH":
+            spec.BeaconState.fields['latest_active_index_roots'][1] = config[field]
+        elif field == "LATEST_SLASHED_EXIT_LENGTH":
+            spec.BeaconState.fields['latest_slashed_balances'][1] = config[field]
+
+
+@pytest.fixture(
+    params=[
+        pytest.param(MINIMAL_CONFIG, marks=pytest.mark.minimal_config),
+        DEFAULT_CONFIG,
+    ]
+)
+def config(request):
+    return request.param
+
+
+@pytest.fixture(autouse=True)
+def overwrite_config(config):
+    overwrite_spec_config(config)
+
+
+@pytest.fixture
+def num_validators():
+    return 100
+
+
+@pytest.fixture
+def deposit_data_leaves():
+    return list()
+
+
+@pytest.fixture
+def state(num_validators, deposit_data_leaves):
+    return create_genesis_state(num_validators, deposit_data_leaves)

tests/phase0/helpers.py

@@ -0,0 +1,145 @@
+from copy import deepcopy
+
+from py_ecc import bls
+
+import build.phase0.spec as spec
+from build.phase0.utils.minimal_ssz import signed_root
+from build.phase0.spec import (
+    # constants
+    EMPTY_SIGNATURE,
+    # SSZ
+    AttestationData,
+    Deposit,
+    DepositInput,
+    DepositData,
+    Eth1Data,
+    # functions
+    get_block_root,
+    get_current_epoch,
+    get_domain,
+    get_empty_block,
+    get_epoch_start_slot,
+    get_genesis_beacon_state,
+    verify_merkle_branch,
+    hash,
+)
+from build.phase0.utils.merkle_minimal import (
+    calc_merkle_tree_from_leaves,
+    get_merkle_proof,
+    get_merkle_root,
+)
+
+
+privkeys_list = [i + 1 for i in range(1000)]
+pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list]
+pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)}
+
+
+def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves):
+    deposit_timestamp = 0
+    proof_of_possession = b'\x33' * 96
+
+    deposit_data_list = []
+    for i in range(num_validators):
+        pubkey = pubkeys_list[i]
+        deposit_data = DepositData(
+            amount=spec.MAX_DEPOSIT_AMOUNT,
+            timestamp=deposit_timestamp,
+            deposit_input=DepositInput(
+                pubkey=pubkey,
+                # insecurely use pubkey as withdrawal key as well
+                withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:],
+                proof_of_possession=proof_of_possession,
+            ),
+        )
+        item = hash(deposit_data.serialize())
+        deposit_data_leaves.append(item)
+        tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
+        root = get_merkle_root((tuple(deposit_data_leaves)))
+        proof = list(get_merkle_proof(tree, item_index=i))
+        assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root)
+        deposit_data_list.append(deposit_data)
+
+    genesis_validator_deposits = []
+    for i in range(num_validators):
+        genesis_validator_deposits.append(Deposit(
+            proof=list(get_merkle_proof(tree, item_index=i)),
+            index=i,
+            deposit_data=deposit_data_list[i]
+        ))
+    return genesis_validator_deposits, root
+
+
+def create_genesis_state(num_validators, deposit_data_leaves):
+    initial_deposits, deposit_root = create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves)
+    return get_genesis_beacon_state(
+        initial_deposits,
+        genesis_time=0,
+        genesis_eth1_data=Eth1Data(
+            deposit_root=deposit_root,
+            block_hash=spec.ZERO_HASH,
+        ),
+    )
+
+
+def build_empty_block_for_next_slot(state):
+    empty_block = get_empty_block()
+    empty_block.slot = state.slot + 1
+    previous_block_header = deepcopy(state.latest_block_header)
+    if previous_block_header.state_root == spec.ZERO_HASH:
+        previous_block_header.state_root = state.hash_tree_root()
+    empty_block.previous_block_root = previous_block_header.hash_tree_root()
+    return empty_block
+
+
+def build_deposit_data(state, pubkey, privkey, amount):
+    deposit_input = DepositInput(
+        pubkey=pubkey,
+        # insecurely use pubkey as withdrawal key as well
+        withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:],
+        proof_of_possession=EMPTY_SIGNATURE,
+    )
+    proof_of_possession = bls.sign(
+        message_hash=signed_root(deposit_input),
+        privkey=privkey,
+        domain=get_domain(
+            state.fork,
+            get_current_epoch(state),
+            spec.DOMAIN_DEPOSIT,
+        )
+    )
+    deposit_input.proof_of_possession = proof_of_possession
+    deposit_data = DepositData(
+        amount=amount,
+        timestamp=0,
+        deposit_input=deposit_input,
+    )
+    return deposit_data
+
+
+def build_attestation_data(state, slot, shard):
+    assert state.slot >= slot
+
+    block_root = build_empty_block_for_next_slot(state).previous_block_root
+
+    epoch_start_slot = get_epoch_start_slot(get_current_epoch(state))
+    if epoch_start_slot == slot:
+        epoch_boundary_root = block_root
+    else:
+        get_block_root(state, epoch_start_slot)
+
+    if slot < epoch_start_slot:
+        justified_block_root = state.previous_justified_root
+    else:
+        justified_block_root = state.current_justified_root
+
+    return AttestationData(
+        slot=slot,
+        shard=shard,
+        beacon_block_root=block_root,
+        source_epoch=state.current_justified_epoch,
+        source_root=justified_block_root,
+        target_root=epoch_boundary_root,
+        crosslink_data_root=spec.ZERO_HASH,
+        previous_crosslink=deepcopy(state.latest_crosslinks[shard]),
+    )

tests/phase0/test_sanity.py

@@ -0,0 +1,499 @@
+from copy import deepcopy
+
+import pytest
+
+from py_ecc import bls
+import build.phase0.spec as spec
+
+from build.phase0.utils.minimal_ssz import signed_root
+from build.phase0.spec import (
+    # constants
+    EMPTY_SIGNATURE,
+    ZERO_HASH,
+    # SSZ
+    Attestation,
+    AttestationDataAndCustodyBit,
+    BeaconBlockHeader,
+    Deposit,
+    Transfer,
+    ProposerSlashing,
+    VoluntaryExit,
+    # functions
+    get_active_validator_indices,
+    get_attestation_participants,
+    get_block_root,
+    get_crosslink_committees_at_slot,
+    get_current_epoch,
+    get_domain,
+    get_state_root,
+    advance_slot,
+    cache_state,
+    verify_merkle_branch,
+    hash,
+)
+from build.phase0.state_transition import (
+    state_transition,
+)
+from build.phase0.utils.merkle_minimal import (
+    calc_merkle_tree_from_leaves,
+    get_merkle_proof,
+    get_merkle_root,
+)
+from tests.phase0.helpers import (
+    build_attestation_data,
+    build_deposit_data,
+    build_empty_block_for_next_slot,
+)
+
+
+# mark entire file as 'sanity'
+pytestmark = pytest.mark.sanity
+
+
+def test_slot_transition(state):
+    test_state = deepcopy(state)
+    cache_state(test_state)
+    advance_slot(test_state)
+    assert test_state.slot == state.slot + 1
+    assert get_state_root(test_state, state.slot) == state.hash_tree_root()
+    return test_state
+
+
+def test_empty_block_transition(state):
+    test_state = deepcopy(state)
+
+    block = build_empty_block_for_next_slot(test_state)
+    state_transition(test_state, block)
+
+    assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1
+    assert get_block_root(test_state, state.slot) == block.previous_block_root
+
+    return state, [block], test_state
+
+
+def test_skipped_slots(state):
+    test_state = deepcopy(state)
+    block = build_empty_block_for_next_slot(test_state)
+    block.slot += 3
+
+    state_transition(test_state, block)
+
+    assert test_state.slot == block.slot
+    for slot in range(state.slot, test_state.slot):
+        assert get_block_root(test_state, slot) == block.previous_block_root
+
+    return state, [block], test_state
+
+
+def test_empty_epoch_transition(state):
+    test_state = deepcopy(state)
+    block = build_empty_block_for_next_slot(test_state)
+    block.slot += spec.SLOTS_PER_EPOCH
+
+    state_transition(test_state, block)
+
+    assert test_state.slot == block.slot
+    for slot in range(state.slot, test_state.slot):
+        assert get_block_root(test_state, slot) == block.previous_block_root
+
+    return state, [block], test_state
+
+
+def test_empty_epoch_transition_not_finalizing(state):
+    test_state = deepcopy(state)
+    block = build_empty_block_for_next_slot(test_state)
+    block.slot += spec.SLOTS_PER_EPOCH * 5
+
+    state_transition(test_state, block)
+
+    assert test_state.slot == block.slot
+    assert test_state.finalized_epoch < get_current_epoch(test_state) - 4
+
+    return state, [block], test_state
+
+
+def test_proposer_slashing(state, pubkeys, privkeys):
+    test_state = deepcopy(state)
+    current_epoch = get_current_epoch(test_state)
+    validator_index = get_active_validator_indices(test_state.validator_registry, current_epoch)[-1]
+    privkey = privkeys[validator_index]
+    slot = spec.GENESIS_SLOT
+    header_1 = BeaconBlockHeader(
+        slot=slot,
+        previous_block_root=ZERO_HASH,
+        state_root=ZERO_HASH,
+        block_body_root=ZERO_HASH,
+        signature=EMPTY_SIGNATURE,
+    )
+    header_2 = deepcopy(header_1)
+    header_2.previous_block_root = b'\x02' * 32
+    header_2.slot = slot + 1
+
+    domain = get_domain(
+        fork=test_state.fork,
+        epoch=get_current_epoch(test_state),
+        domain_type=spec.DOMAIN_BEACON_BLOCK,
+    )
+    header_1.signature = bls.sign(
+        message_hash=signed_root(header_1),
+        privkey=privkey,
+        domain=domain,
+    )
+    header_2.signature = bls.sign(
+        message_hash=signed_root(header_2),
+        privkey=privkey,
+        domain=domain,
+    )
+
+    proposer_slashing = ProposerSlashing(
+        proposer_index=validator_index,
+        header_1=header_1,
+        header_2=header_2,
+    )
+
+    #
+    # Add to state via block transition
+    #
+    block = build_empty_block_for_next_slot(test_state)
+    block.body.proposer_slashings.append(proposer_slashing)
+    state_transition(test_state, block)
+
+    assert not state.validator_registry[validator_index].initiated_exit
+    assert not state.validator_registry[validator_index].slashed
+
+    slashed_validator = test_state.validator_registry[validator_index]
+    assert not slashed_validator.initiated_exit
+    assert slashed_validator.slashed
+    assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH
+    assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH
+    # lost whistleblower reward
+    assert test_state.validator_balances[validator_index] < state.validator_balances[validator_index]
+
+    return state, [block], test_state
+
+
+def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys):
+    pre_state = deepcopy(state)
+    test_deposit_data_leaves = deepcopy(deposit_data_leaves)
+
+    index = len(test_deposit_data_leaves)
+    pubkey = pubkeys[index]
+    privkey = privkeys[index]
+    deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_DEPOSIT_AMOUNT)
+
+    item = hash(deposit_data.serialize())
+    test_deposit_data_leaves.append(item)
+    tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves))
+    root = get_merkle_root((tuple(test_deposit_data_leaves)))
+    proof = list(get_merkle_proof(tree, item_index=index))
+    assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root)
+
+    deposit = Deposit(
+        proof=list(proof),
+        index=index,
+        deposit_data=deposit_data,
+    )
+
+    pre_state.latest_eth1_data.deposit_root = root
+    post_state = deepcopy(pre_state)
+    block = build_empty_block_for_next_slot(post_state)
+    block.body.deposits.append(deposit)
+
+    state_transition(post_state, block)
+    assert len(post_state.validator_registry) == len(state.validator_registry) + 1
+    assert len(post_state.validator_balances) == len(state.validator_balances) + 1
+    assert post_state.validator_registry[index].pubkey == pubkeys[index]
+
+    return pre_state, [block], post_state
+
+
+def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves):
+    pre_state = deepcopy(state)
+    test_deposit_data_leaves = deepcopy(deposit_data_leaves)
+
+    validator_index = 0
+    amount = spec.MAX_DEPOSIT_AMOUNT // 4
+    pubkey = pubkeys[validator_index]
+    privkey = privkeys[validator_index]
+    deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount)
+
+    merkle_index = len(test_deposit_data_leaves)
+    item = hash(deposit_data.serialize())
+    test_deposit_data_leaves.append(item)
+    tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves))
+    root = get_merkle_root((tuple(test_deposit_data_leaves)))
+    proof = list(get_merkle_proof(tree, item_index=merkle_index))
+    assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root)
+
+    deposit = Deposit(
+        proof=list(proof),
+        index=merkle_index,
+        deposit_data=deposit_data,
+    )
+
+    pre_state.latest_eth1_data.deposit_root = root
+    block = build_empty_block_for_next_slot(pre_state)
+    block.body.deposits.append(deposit)
+
+    pre_balance = pre_state.validator_balances[validator_index]
+    post_state = deepcopy(pre_state)
+    state_transition(post_state, block)
+    assert len(post_state.validator_registry) == len(pre_state.validator_registry)
+    assert len(post_state.validator_balances) == len(pre_state.validator_balances)
+    assert post_state.validator_balances[validator_index] == pre_balance + amount
+
+    return pre_state, [block], post_state
+
+
+def test_attestation(state, pubkeys, privkeys):
+    test_state = deepcopy(state)
+    slot = state.slot
+    shard = state.current_shuffling_start_shard
+    attestation_data = build_attestation_data(state, slot, shard)
+
+    crosslink_committees = get_crosslink_committees_at_slot(state, slot)
+    crosslink_committee = [committee for committee, _shard in crosslink_committees if _shard == attestation_data.shard][0]
+
+    committee_size = len(crosslink_committee)
+    bitfield_length = (committee_size + 7) // 8
+    aggregation_bitfield = b'\x01' + b'\x00' * (bitfield_length - 1)
+    custody_bitfield = b'\x00' * bitfield_length
+    attestation = Attestation(
+        aggregation_bitfield=aggregation_bitfield,
+        data=attestation_data,
+        custody_bitfield=custody_bitfield,
+        aggregate_signature=EMPTY_SIGNATURE,
+    )
+    participants = get_attestation_participants(
+        test_state,
+        attestation.data,
+        attestation.aggregation_bitfield,
+    )
+    assert len(participants) == 1
+
+    validator_index = participants[0]
+    privkey = privkeys[validator_index]
+
+    message_hash = AttestationDataAndCustodyBit(
+        data=attestation.data,
+        custody_bit=0b0,
+    ).hash_tree_root()
+
+    attestation.aggregation_signature = bls.sign(
+        message_hash=message_hash,
+        privkey=privkey,
+        domain=get_domain(
+            fork=test_state.fork,
+            epoch=get_current_epoch(test_state),
+            domain_type=spec.DOMAIN_ATTESTATION,
+        )
+    )
+
+    #
+    # Add to state via block transition
+    #
+    attestation_block = build_empty_block_for_next_slot(test_state)
+    attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+    attestation_block.body.attestations.append(attestation)
+    state_transition(test_state, attestation_block)
+
+    assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1
+
+    #
+    # Epoch transition should move to previous_epoch_attestations
+    #
+    pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations)
+
+    epoch_block = build_empty_block_for_next_slot(test_state)
+    epoch_block.slot += spec.SLOTS_PER_EPOCH
+    state_transition(test_state, epoch_block)
+
+    assert len(test_state.current_epoch_attestations) == 0
+    assert test_state.previous_epoch_attestations == pre_current_epoch_attestations
+
+    return state, [attestation_block, epoch_block], test_state
+
+
+def test_voluntary_exit(state, pubkeys, privkeys):
+    pre_state = deepcopy(state)
+    validator_index = get_active_validator_indices(
+        pre_state.validator_registry,
+        get_current_epoch(pre_state)
+    )[-1]
+
+    # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit
+    pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+    # artificially trigger registry update at next epoch transition
+    pre_state.finalized_epoch = get_current_epoch(pre_state) - 1
+    for crosslink in pre_state.latest_crosslinks:
+        crosslink.epoch = pre_state.finalized_epoch
+    pre_state.validator_registry_update_epoch = pre_state.finalized_epoch - 1
+
+    post_state = deepcopy(pre_state)
+
+    voluntary_exit = VoluntaryExit(
+        epoch=get_current_epoch(pre_state),
+        validator_index=validator_index,
+        signature=EMPTY_SIGNATURE,
+    )
+    voluntary_exit.signature = bls.sign(
+        message_hash=signed_root(voluntary_exit),
+        privkey=privkeys[validator_index],
+        domain=get_domain(
+            fork=pre_state.fork,
+            epoch=get_current_epoch(pre_state),
+            domain_type=spec.DOMAIN_VOLUNTARY_EXIT,
+        )
+    )
+
+    #
+    # Add to state via block transition
+    #
+    initiate_exit_block = build_empty_block_for_next_slot(post_state)
+    initiate_exit_block.body.voluntary_exits.append(voluntary_exit)
+    state_transition(post_state, initiate_exit_block)
+
+    assert not pre_state.validator_registry[validator_index].initiated_exit
+    assert post_state.validator_registry[validator_index].initiated_exit
+    assert post_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH
+
+    #
+    # Process within epoch transition
+    #
+    exit_block = build_empty_block_for_next_slot(post_state)
+    exit_block.slot += spec.SLOTS_PER_EPOCH
+    state_transition(post_state, exit_block)
+
+    assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH
+
+    return pre_state, [initiate_exit_block, exit_block], post_state
+
+
+def test_no_exit_too_long_since_change(state):
+    pre_state = deepcopy(state)
+    validator_index = get_active_validator_indices(
+        pre_state.validator_registry,
+        get_current_epoch(pre_state)
+    )[-1]
+
+    #
+    # setup pre_state
+    #
+    # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit
+    pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+    # artificially trigger registry update at next epoch transition
+    pre_state.finalized_epoch = get_current_epoch(pre_state) - 1
+    for crosslink in pre_state.latest_crosslinks:
+        crosslink.epoch = pre_state.finalized_epoch
+    # make epochs since registry update greater than LATEST_SLASHED_EXIT_LENGTH
+    pre_state.validator_registry_update_epoch = (
+        get_current_epoch(pre_state) - spec.LATEST_SLASHED_EXIT_LENGTH
+    )
+    # set validator to have previously initiated exit
+    pre_state.validator_registry[validator_index].initiated_exit = True
+
+    post_state = deepcopy(pre_state)
+
+    #
+    # Process registry change but ensure no exit
+    #
+    block = build_empty_block_for_next_slot(post_state)
+    block.slot += spec.SLOTS_PER_EPOCH
+    state_transition(post_state, block)
+
+    assert post_state.validator_registry_update_epoch == get_current_epoch(post_state) - 1
+    assert post_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH
+
+    return pre_state, [block], post_state
+
+
+def test_transfer(state, pubkeys, privkeys):
+    pre_state = deepcopy(state)
+    current_epoch = get_current_epoch(pre_state)
+    sender_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1]
+    recipient_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    transfer_pubkey = pubkeys[-1]
+    transfer_privkey = privkeys[-1]
+    amount = pre_state.validator_balances[sender_index]
+    pre_transfer_recipient_balance = pre_state.validator_balances[recipient_index]
+    transfer = Transfer(
+        sender=sender_index,
+        recipient=recipient_index,
+        amount=amount,
+        fee=0,
+        slot=pre_state.slot + 1,
+        pubkey=transfer_pubkey,
+        signature=EMPTY_SIGNATURE,
+    )
+    transfer.signature = bls.sign(
+        message_hash=signed_root(transfer),
+        privkey=transfer_privkey,
+        domain=get_domain(
+            fork=pre_state.fork,
+            epoch=get_current_epoch(pre_state),
+            domain_type=spec.DOMAIN_TRANSFER,
+        )
+    )
+
+    # ensure withdrawal_credentials reproducable
+    pre_state.validator_registry[sender_index].withdrawal_credentials = (
+        spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:]
+    )
+    # un-activate so validator can transfer
+    pre_state.validator_registry[sender_index].activation_epoch = spec.FAR_FUTURE_EPOCH
+
+    post_state = deepcopy(pre_state)
+    #
+    # Add to state via block transition
+    #
+    block = build_empty_block_for_next_slot(post_state)
+    block.body.transfers.append(transfer)
+    state_transition(post_state, block)
+
+    sender_balance = post_state.validator_balances[sender_index]
+    recipient_balance = post_state.validator_balances[recipient_index]
+    assert sender_balance == 0
+    assert recipient_balance == pre_transfer_recipient_balance + amount
+
+    return pre_state, [block], post_state
+
+
+def test_ejection(state):
+    pre_state = deepcopy(state)
+
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1]
+
+    assert pre_state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH
+
+    # set validator balance to below ejection threshold
+    pre_state.validator_balances[validator_index] = spec.EJECTION_BALANCE - 1
+
+    post_state = deepcopy(pre_state)
+    #
+    # trigger epoch transition
+    #
+    block = build_empty_block_for_next_slot(post_state)
+    block.slot += spec.SLOTS_PER_EPOCH
+    state_transition(post_state, block)
+
+    assert post_state.validator_registry[validator_index].initiated_exit == True
+
+    return pre_state, [block], post_state
+
+
+def test_historical_batch(state):
+    pre_state = deepcopy(state)
+    pre_state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (pre_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1
+
+    post_state = deepcopy(pre_state)
+
+    block = build_empty_block_for_next_slot(post_state)
+
+    state_transition(post_state, block)
+
+    assert post_state.slot == block.slot
+    assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0
+    assert len(post_state.historical_roots) == len(pre_state.historical_roots) + 1
+
+    return pre_state, [block], post_state

utils/phase0/bls_stub.py

@@ -0,0 +1,12 @@
+
+
+def bls_verify(pubkey, message_hash, signature, domain):
+    return True
+
+
+def bls_verify_multiple(pubkeys, message_hashes, signature, domain):
+    return True
+
+
+def bls_aggregate_pubkeys(pubkeys):
+    return b'\x42' * 96

utils/phase0/hash_function.py

@@ -0,0 +1,7 @@
+# from hashlib import sha256
+from eth_utils import keccak
+
+
+# def hash(x): return sha256(x).digest()
+def hash(x):
+    return keccak(x)

utils/phase0/merkle_minimal.py

@@ -0,0 +1,30 @@
+from .hash_function import hash
+
+
+zerohashes = [b'\x00' * 32]
+for layer in range(1, 32):
+    zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1]))
+
+
+# Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree
+def calc_merkle_tree_from_leaves(values):
+    values = list(values)
+    tree = [values[::]]
+    for h in range(32):
+        if len(values) % 2 == 1:
+            values.append(zerohashes[h])
+        values = [hash(values[i] + values[i + 1]) for i in range(0, len(values), 2)]
+        tree.append(values[::])
+    return tree
+
+
+def get_merkle_root(values):
+    return calc_merkle_tree_from_leaves(values)[-1][0]
+
+
+def get_merkle_proof(tree, item_index):
+    proof = []
+    for i in range(32):
+        subindex = (item_index // 2**i) ^ 1
+        proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i])
+    return proof

utils/phase0/minimal_ssz.py

@@ -0,0 +1,213 @@
+from .hash_function import hash
+
+
+BYTES_PER_CHUNK = 32
+BYTES_PER_LENGTH_PREFIX = 4
+ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK
+
+
+def SSZType(fields):
+    class SSZObject():
+        def __init__(self, **kwargs):
+            for f in fields:
+                if f not in kwargs:
+                    raise Exception("Missing constructor argument: %s" % f)
+                setattr(self, f, kwargs[f])
+
+        def __eq__(self, other):
+            return (
+                self.fields == other.fields and
+                self.serialize() == other.serialize()
+            )
+
+        def __hash__(self):
+            return int.from_bytes(self.hash_tree_root(), byteorder="little")
+
+        def __str__(self):
+            output = []
+            for field in self.fields:
+                output.append(f'{field}: {getattr(self, field)}')
+            return "\n".join(output)
+
+        def serialize(self):
+            return serialize_value(self, self.__class__)
+
+        def hash_tree_root(self):
+            return hash_tree_root(self, self.__class__)
+
+    SSZObject.fields = fields
+    return SSZObject
+
+
+class Vector():
+    def __init__(self, items):
+        self.items = items
+        self.length = len(items)
+
+    def __getitem__(self, key):
+        return self.items[key]
+
+    def __setitem__(self, key, value):
+        self.items[key] = value
+
+    def __iter__(self):
+        return iter(self.items)
+
+    def __len__(self):
+        return self.length
+
+
+def is_basic(typ):
+    return isinstance(typ, str) and (typ[:4] in ('uint', 'bool') or typ == 'byte')
+
+
+def is_constant_sized(typ):
+    if is_basic(typ):
+        return True
+    elif isinstance(typ, list) and len(typ) == 1:
+        return is_constant_sized(typ[0])
+    elif isinstance(typ, list) and len(typ) == 2:
+        return False
+    elif isinstance(typ, str) and typ[:5] == 'bytes':
+        return len(typ) > 5
+    elif hasattr(typ, 'fields'):
+        for subtype in typ.fields.values():
+            if not is_constant_sized(subtype):
+                return False
+        return True
+    else:
+        raise Exception("Type not recognized")
+
+
+def coerce_to_bytes(x):
+    if isinstance(x, str):
+        o = x.encode('utf-8')
+        assert len(o) == len(x)
+        return o
+    elif isinstance(x, bytes):
+        return x
+    else:
+        raise Exception("Expecting bytes")
+
+
+def serialize_value(value, typ=None):
+    if typ is None:
+        typ = infer_type(value)
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        length = int(typ[4:])
+        assert length in (8, 16, 32, 64, 128, 256)
+        return value.to_bytes(length // 8, 'little')
+    elif typ == 'bool':
+        assert value in (True, False)
+        return b'\x01' if value is True else b'\x00'
+    elif (isinstance(typ, list) and len(typ) == 1) or typ == 'bytes':
+        serialized_bytes = coerce_to_bytes(value) if typ == 'bytes' else b''.join([serialize_value(element, typ[0]) for element in value])
+        assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX)
+        serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
+        return serialized_length + serialized_bytes
+    elif isinstance(typ, list) and len(typ) == 2:
+        assert len(value) == typ[1]
+        return b''.join([serialize_value(element, typ[0]) for element in value])
+    elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes':
+        assert len(value) == int(typ[5:]), (value, int(typ[5:]))
+        return coerce_to_bytes(value)
+    elif hasattr(typ, 'fields'):
+        serialized_bytes = b''.join([serialize_value(getattr(value, field), subtype) for field, subtype in typ.fields.items()])
+        if is_constant_sized(typ):
+            return serialized_bytes
+        else:
+            assert len(serialized_bytes) < 2**(8 * BYTES_PER_LENGTH_PREFIX)
+            serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, 'little')
+            return serialized_length + serialized_bytes
+    else:
+        print(value, typ)
+        raise Exception("Type not recognized")
+
+
+def chunkify(bytez):
+    bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK)
+    return [bytez[i:i + 32] for i in range(0, len(bytez), 32)]
+
+
+def pack(values, subtype):
+    return chunkify(b''.join([serialize_value(value, subtype) for value in values]))
+
+
+def is_power_of_two(x):
+    return x > 0 and x & (x - 1) == 0
+
+
+def merkleize(chunks):
+    tree = chunks[::]
+    while not is_power_of_two(len(tree)):
+        tree.append(ZERO_CHUNK)
+    tree = [ZERO_CHUNK] * len(tree) + tree
+    for i in range(len(tree) // 2 - 1, 0, -1):
+        tree[i] = hash(tree[i * 2] + tree[i * 2 + 1])
+    return tree[1]
+
+
+def mix_in_length(root, length):
+    return hash(root + length.to_bytes(32, 'little'))
+
+
+def infer_type(value):
+    if hasattr(value.__class__, 'fields'):
+        return value.__class__
+    elif isinstance(value, Vector):
+        return [infer_type(value[0]) if len(value) > 0 else 'uint64', len(value)]
+    elif isinstance(value, list):
+        return [infer_type(value[0])] if len(value) > 0 else ['uint64']
+    elif isinstance(value, (bytes, str)):
+        return 'bytes'
+    elif isinstance(value, int):
+        return 'uint64'
+    else:
+        raise Exception("Failed to infer type")
+
+
+def hash_tree_root(value, typ=None):
+    if typ is None:
+        typ = infer_type(value)
+    if is_basic(typ):
+        return merkleize(pack([value], typ))
+    elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]):
+        return mix_in_length(merkleize(pack(value, typ[0])), len(value))
+    elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]):
+        return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value))
+    elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]):
+        assert len(value) == typ[1]
+        return merkleize(pack(value, typ[0]))
+    elif typ == 'bytes':
+        return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value))
+    elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5:
+        assert len(value) == int(typ[5:])
+        return merkleize(chunkify(coerce_to_bytes(value)))
+    elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]):
+        return merkleize([hash_tree_root(element, typ[0]) for element in value])
+    elif hasattr(typ, 'fields'):
+        return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()])
+    else:
+        raise Exception("Type not recognized")
+
+
+def truncate(container):
+    field_keys = list(container.fields.keys())
+    truncated_fields = {
+        key: container.fields[key]
+        for key in field_keys[:-1]
+    }
+    truncated_class = SSZType(truncated_fields)
+    kwargs = {
+        field: getattr(container, field)
+        for field in field_keys[:-1]
+    }
+    return truncated_class(**kwargs)
+
+
+def signed_root(container):
+    return hash_tree_root(truncate(container))
+
+
+def serialize(ssz_object):
+    return getattr(ssz_object, 'serialize')()

utils/phase0/state_transition.py

@@ -0,0 +1,100 @@
+from . import spec
+
+
+from typing import (  # noqa: F401
+    Any,
+    Callable,
+    List,
+    NewType,
+    Tuple,
+)
+
+from .spec import (
+    BeaconState,
+    BeaconBlock,
+)
+
+
+def process_transaction_type(state: BeaconState,
+                             transactions: List[Any],
+                             max_transactions: int,
+                             tx_fn: Callable[[BeaconState, Any], None]) -> None:
+    assert len(transactions) <= max_transactions
+    for transaction in transactions:
+        tx_fn(state, transaction)
+
+
+def process_transactions(state: BeaconState, block: BeaconBlock) -> None:
+    process_transaction_type(
+        state,
+        block.body.proposer_slashings,
+        spec.MAX_PROPOSER_SLASHINGS,
+        spec.process_proposer_slashing,
+    )
+    process_transaction_type(
+        state,
+        block.body.attester_slashings,
+        spec.MAX_ATTESTER_SLASHINGS,
+        spec.process_attester_slashing,
+    )
+    process_transaction_type(
+        state,
+        block.body.attestations,
+        spec.MAX_ATTESTATIONS,
+        spec.process_attestation,
+    )
+    process_transaction_type(
+        state,
+        block.body.deposits,
+        spec.MAX_DEPOSITS,
+        spec.process_deposit,
+    )
+    process_transaction_type(
+        state,
+        block.body.voluntary_exits,
+        spec.MAX_VOLUNTARY_EXITS,
+        spec.process_voluntary_exit,
+    )
+    assert len(block.body.transfers) == len(set(block.body.transfers))
+    process_transaction_type(
+        state,
+        block.body.transfers,
+        spec.MAX_TRANSFERS,
+        spec.process_transfer,
+    )
+
+
+def process_block(state: BeaconState,
+                  block: BeaconBlock,
+                  verify_state_root: bool=False) -> None:
+    spec.process_block_header(state, block)
+    spec.process_randao(state, block)
+    spec.process_eth1_data(state, block)
+
+    process_transactions(state, block)
+    if verify_state_root:
+        spec.verify_block_state_root(state, block)
+
+
+def process_epoch_transition(state: BeaconState) -> None:
+    spec.update_justification_and_finalization(state)
+    spec.process_crosslinks(state)
+    spec.maybe_reset_eth1_period(state)
+    spec.apply_rewards(state)
+    spec.process_ejections(state)
+    spec.update_registry_and_shuffling_data(state)
+    spec.process_slashings(state)
+    spec.process_exit_queue(state)
+    spec.finish_epoch_update(state)
+
+
+def state_transition(state: BeaconState,
+                     block: BeaconBlock,
+                     verify_state_root: bool=False) -> BeaconState:
+    while state.slot < block.slot:
+        spec.cache_state(state)
+        if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0:
+            process_epoch_transition(state)
+        spec.advance_slot(state)
+        if block.slot == state.slot:
+            process_block(state, block, verify_state_root)