merge dev, resolve minor merge conflicts

.circleci/config.yml

@@ -35,26 +35,26 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v2-pyspec
+          venv_name: v3-pyspec-bump2
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v2-pyspec
+          venv_name: v3-pyspec-bump2
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
           venv_path: ./test_libs/pyspec/venv
   restore_deposit_contract_cached_venv:
     description: "Restore the cache with deposit_contract keys"
     steps:
       - restore_cached_venv:
-          venv_name: v4-deposit-contract
+          venv_name: v6-deposit-contract
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
   save_deposit_contract_cached_venv:
     description: Save a venv into a cache with deposit_contract keys"
     steps:
       - save_cached_venv:
-          venv_name: v4-deposit-contract
+          venv_name: v6-deposit-contract
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
           venv_path: ./deposit_contract/venv
 jobs:

Makefile

@@ -79,10 +79,10 @@ test_deposit_contract:
 pyspec: $(PY_SPEC_ALL_TARGETS)
 
 $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@
 
 $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@
 
 CURRENT_DIR = ${CURDIR}
 

scripts/build_spec.py

@@ -16,6 +16,11 @@ PHASE0_IMPORTS = '''from typing import (
     List as TypingList,
 )
 
+from dataclasses import (
+    dataclass,
+    field,
+)
+
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
@@ -37,6 +42,11 @@ PHASE1_IMPORTS = '''from typing import (
     List as TypingList
 )
 
+from dataclasses import (
+    dataclass,
+    field,
+)
+
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
@@ -235,9 +245,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     return functions, custom_types, constants, ssz_objects, inserts
 
 
-def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
+def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]:
-    functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile)
+    phase0_spec = get_spec(phase0_sourcefile)
-    spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS)
+    fork_choice_spec = get_spec(fork_choice_sourcefile)
+    spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec)
+    spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)
@@ -247,12 +259,14 @@ def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
 def build_phase1_spec(phase0_sourcefile: str,
                       phase1_custody_sourcefile: str,
                       phase1_shard_sourcefile: str,
+                      fork_choice_sourcefile: str,
                       outfile: str=None) -> Optional[str]:
     phase0_spec = get_spec(phase0_sourcefile)
     phase1_custody = get_spec(phase1_custody_sourcefile)
     phase1_shard_data = get_spec(phase1_shard_sourcefile)
+    fork_choice_spec = get_spec(fork_choice_sourcefile)
     spec_objects = phase0_spec
-    for value in [phase1_custody, phase1_shard_data]:
+    for value in [phase1_custody, phase1_shard_data, fork_choice_spec]:
         spec_objects = combine_spec_objects(spec_objects, value)
     spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS)
     if outfile is not None:
@@ -266,13 +280,15 @@ if __name__ == '__main__':
 Build the specs from the md docs.
 If building phase 0:
     1st argument is input spec.md
-    2nd argument is output spec.py
+    2nd argument is input fork_choice.md
+    3rd argument is output spec.py
 
 If building phase 1:
     1st argument is input spec_phase0.md
     2nd argument is input spec_phase1_custody.md
     3rd argument is input spec_phase1_shard_data.md
-    4th argument is output spec.py
+    4th argument is input fork_choice.md
+    5th argument is output spec.py
 '''
     parser = ArgumentParser(description=description)
     parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
@@ -280,14 +296,14 @@ If building phase 1:
 
     args = parser.parse_args()
     if args.phase == 0:
-        if len(args.files) == 2:
+        if len(args.files) == 3:
             build_phase0_spec(*args.files)
         else:
-            print(" Phase 0 requires an output as well as an input file.")
+            print(" Phase 0 requires an output as well as spec and forkchoice files.")
     elif args.phase == 1:
-        if len(args.files) == 4:
+        if len(args.files) == 5:
             build_phase1_spec(*args.files)
         else:
-            print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)")
+            print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)")
     else:
         print("Invalid phase: {0}".format(args.phase))

scripts/function_puller.py

@@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject:
     inserts = {}
     function_matcher = re.compile(FUNCTION_REGEX)
     inserts_matcher = re.compile(BEGIN_INSERT_REGEX)
+    is_ssz = False
     custom_types = {}
     for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()

specs/core/0_fork-choice.md

@@ -8,23 +8,26 @@
 - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice)
     - [Table of contents](#table-of-contents)
     - [Introduction](#introduction)
-    - [Prerequisites](#prerequisites)
+    - [Constants](#constants)
-    - [Configuration](#configuration)
         - [Time parameters](#time-parameters)
-    - [Beacon chain processing](#beacon-chain-processing)
+    - [Fork choice](#fork-choice)
-        - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule)
+        - [Helpers](#helpers)
-    - [Implementation notes](#implementation-notes)
+            - [`Target`](#target)
-        - [Justification and finality at genesis](#justification-and-finality-at-genesis)
+            - [`Store`](#store)
+            - [`get_genesis_store`](#get_genesis_store)
+            - [`get_ancestor`](#get_ancestor)
+            - [`get_latest_attesting_balance`](#get_latest_attesting_balance)
+            - [`get_head`](#get_head)
+        - [Handlers](#handlers)
+            - [`on_tick`](#on_tick)
+            - [`on_block`](#on_block)
+            - [`on_attestation`](#on_attestation)
 
 <!-- /TOC -->
 
 ## Introduction
 
-This document represents the specification for the beacon chain fork choice rule, part of Ethereum 2.0 Phase 0.
+This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md).
-
-## Prerequisites
-
-All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./0_beacon-chain.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout.
 
 ## Configuration
 
@@ -34,76 +37,139 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph
 | - | - | :-: | :-: |
 | `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds |
 
-## Beacon chain processing
+## Fork choice
 
-Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met:
+The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running:
 
-* The parent block with root `block.parent_root` has been processed and accepted.
+* `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time
-* An Ethereum 1.0 block pointed to by the `state.eth1_data.block_hash` has been processed and accepted.
+* `on_block(block)` whenever a block `block` is received
-* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`.
+* `on_attestation(attestation)` whenever an attestation `attestation` is received
 
-*Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year.
+*Notes*:
 
-*Note*: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes.
+1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time).
+2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other.
+3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required.
+4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`.
+5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost).
 
-### Beacon chain fork choice rule
+### Helpers
 
-The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time, a validator `v` subjectively calculates the beacon chain head as follows.
+#### `Target`
-
-* Abstractly define `Store` as the type of storage object for the chain data, and let `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`.
-* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store`, the processing of which sets `B` as finalized.)
-* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists, set `justified_head` to `finalized_head`.
-* Let `get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock` be the ancestor of `block` with slot number `slot`. The `get_ancestor` function can be defined recursively as:
 
 ```python
-def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock:
+@dataclass
-    """
+class Target(object):
-    Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found.
+    epoch: Epoch
-    """
+    root: Hash
-    if block.slot == slot:
-        return block
-    elif block.slot < slot:
-        return None
-    else:
-        return get_ancestor(store, store.get_parent(block), slot)
 ```
 
-* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first.
+#### `Store`
-* Let `get_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`.
-* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`.
-* Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`.
-* The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count.
 
 ```python
-def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock:
+@dataclass
-    """
+class Store(object):
-    Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``.
+    blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict)
-    """
+    states: Dict[Hash, BeaconState] = field(default_factory=dict)
-    validators = start_state.validators
+    time: int = 0
-    active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot))
+    latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict)
-    attestation_targets = [(i, get_attestation_target(store, i)) for i in active_validator_indices]
+    justified_root: Hash = ZERO_HASH
+    finalized_root: Hash = ZERO_HASH
+```
 
-    # Use the rounded-balance-with-hysteresis supplied by the protocol for fork
+#### `get_genesis_store`
-    # choice voting. This reduces the number of recomputations that need to be
+
-    # made for optimized implementations that precompute and save data
+```python
-    def get_vote_count(block: BeaconBlock) -> int:
+def get_genesis_store(genesis_state: BeaconState) -> Store:
-        return sum(
+    genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))
-            start_state.validators[validator_index].effective_balance
+    root = signing_root(genesis_block)
-            for validator_index, target in attestation_targets
+    return Store(
-            if get_ancestor(store, target, block.slot) == block
+        blocks={root: genesis_block},
+        states={root: genesis_state},
+        time=genesis_state.genesis_time,
+        justified_root=root,
+        finalized_root=root,
     )
+```
 
-    head = start_block
+#### `get_ancestor`
-    while 1:
+
-        children = get_children(store, head)
+```python
+def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
+    block = store.blocks[root]
+    assert block.slot >= slot
+    return root if block.slot == slot else get_ancestor(store, block.parent_root, slot)
+```
+
+#### `get_latest_attesting_balance`
+
+```python
+def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei:
+    state = store.states[store.justified_root]
+    active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state))
+    return Gwei(sum(
+        state.validator_registry[i].effective_balance for i in active_indices
+        if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root
+    ))
+```
+
+#### `get_head`
+
+```python
+def get_head(store: Store) -> Hash:
+    # Execute the LMD-GHOST fork choice
+    head = store.justified_root
+    while True:
+        children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head]
         if len(children) == 0:
             return head
-        # Ties broken by favoring block with lexicographically higher root
+        # Sort by latest attesting balance with ties broken lexicographically
-        head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x)))
+        head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root))
 ```
 
-## Implementation notes
+### Handlers
 
-### Justification and finality at genesis
+#### `on_tick`
 
-During genesis, justification and finality root fields within the `BeaconState` reference `ZERO_HASH` rather than a known block. `ZERO_HASH` in `previous_justified_root`, `current_justified_root`, and `finalized_root` should be considered as an alias to the root of the genesis block.
+```python
+def on_tick(store: Store, time: int) -> None:
+    store.time = time
+```
+
+#### `on_block`
+
+```python
+def on_block(store: Store, block: BeaconBlock) -> None:
+    # Make a copy of the state to avoid mutability issues
+    pre_state = store.states[block.parent_root].copy()
+    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
+    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
+    # Add new block to the store
+    store.blocks[signing_root(block)] = block
+    # Check block is a descendant of the finalized block
+    assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root
+    # Check the block is valid and compute the post-state
+    state = state_transition(pre_state, block)
+    # Add new state to the store
+    store.states[signing_root(block)] = state
+    # Update justified block root
+    if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot):
+        store.justified_root = state.current_justified_root
+    elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot):
+        store.justified_root = state.previous_justified_root
+    # Update finalized block root
+    if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot):
+        store.finalized_root = state.finalized_root
+```
+
+#### `on_attestation`
+
+```python
+def on_attestation(store: Store, attestation: Attestation) -> None:
+    state = store.states[get_head(store)]
+    indexed_attestation = convert_to_indexed(state, attestation)
+    validate_indexed_attestation(state, indexed_attestation)
+    for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices:
+        if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch:
+            store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root)
+```

specs/core/1_shard-data-chains.md

@@ -61,7 +61,6 @@ This document describes the shard data layer and the shard fork choice rule in P
 | - | - | :-: | :-: |
 | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs  | 6.2 minutes |
 | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days |
-| `SECONDS_PER_SLOT` | `2**1 * 3**1` (= 6) | 6 seconds |
 
 ### Signature domains
 

test_libs/pyspec/eth2spec/test/test_fork_choice.py

@@ -0,0 +1,60 @@
+from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root
+
+from eth2spec.test.context import with_all_phases, with_state, bls_switch
+
+from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.attestations import get_valid_attestation
+from eth2spec.test.helpers.state import next_slot
+
+
+@with_all_phases
+@with_state
+@bls_switch
+def test_basic(spec, state):
+    state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody()))
+
+    # Initialization
+    store = spec.get_genesis_store(state)
+    blocks = []
+    time = 100
+    spec.on_tick(store, time)
+    assert store.time == time
+
+    # On receiving a block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    blocks.append(block)
+    spec.on_block(store, block)
+    assert store.blocks[signing_root(block)] == block
+
+    # On receiving a block of next epoch
+    store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+    block = build_empty_block_for_next_slot(spec, state)
+    block.slot += spec.SLOTS_PER_EPOCH
+    blocks.append(block)
+
+    spec.on_block(store, block)
+    assert store.blocks[signing_root(block)] == block
+
+    # TODO: add tests for justified_root and finalized_root
+
+
+@with_all_phases
+@with_state
+@bls_switch
+def test_on_attestation(spec, state):
+    store = spec.get_genesis_store(state)
+    time = 100
+    spec.on_tick(store, time)
+
+    next_slot(spec, state)
+
+    attestation = get_valid_attestation(spec, state, slot=1)
+    indexed_attestation = spec.convert_to_indexed(state, attestation)
+    spec.on_attestation(store, attestation)
+    assert (
+        store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] ==
+        spec.Target(
+            epoch=attestation.data.target_epoch,
+            root=attestation.data.target_root,
+        )
+    )

test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py

@@ -1,4 +1,5 @@
 from typing import Dict, Iterator
+import copy
 from types import GeneratorType
 
 
@@ -180,6 +181,9 @@ class Container(Series, metaclass=SSZType):
     def __hash__(self):
         return hash(self.hash_tree_root())
 
+    def copy(self):
+        return copy.deepcopy(self)
+
     @classmethod
     def get_fields(cls) -> Dict[str, SSZType]:
         if not hasattr(cls, '__annotations__'):  # no container fields

test_libs/pyspec/requirements.txt

@@ -3,4 +3,5 @@ eth-typing>=2.1.0,<3.0.0
 pycryptodome==3.7.3
 py_ecc>=1.6.0
 typing_inspect==0.4.0
+dataclasses==0.6
 ssz==0.1.0a10

test_libs/pyspec/setup.py

@@ -10,6 +10,7 @@ setup(
         "pycryptodome==3.7.3",
         "py_ecc>=1.6.0",
         "typing_inspect==0.4.0",
-        "ssz==0.1.0a10"
+        "ssz==0.1.0a10",
+        "dataclasses==0.6",
     ]
 )