diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e9a77c49..a43430f77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,26 +35,26 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v4-deposit-contract + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v4-deposit-contract + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: diff --git a/Makefile b/Makefile index 45c263b56..9764b3528 100644 --- a/Makefile +++ b/Makefile @@ -79,10 +79,10 @@ test_deposit_contract: pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3a1b22efd..a16fa79ac 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -20,6 +20,11 @@ PHASE0_IMPORTS = '''from typing import ( Tuple, ) +from dataclasses import ( + dataclass, + field, +) + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -48,6 +53,11 @@ PHASE1_IMPORTS = '''from typing import ( Tuple, ) +from dataclasses import ( + dataclass, + field, +) + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -223,9 +233,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: return functions, custom_types, constants, ssz_objects, inserts -def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: - functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS) +def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: + phase0_spec = get_spec(phase0_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) + spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec) + spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -235,12 +247,14 @@ def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: def build_phase1_spec(phase0_sourcefile: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, + fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_custody, phase1_shard_data]: + for value in [phase1_custody, phase1_shard_data, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: @@ -254,13 +268,15 @@ if __name__ == '__main__': Build the specs from the md docs. If building phase 0: 1st argument is input spec.md - 2nd argument is output spec.py + 2nd argument is input fork_choice.md + 3rd argument is output spec.py If building phase 1: 1st argument is input spec_phase0.md 2nd argument is input spec_phase1_custody.md 3rd argument is input spec_phase1_shard_data.md - 4th argument is output spec.py + 4th argument is input fork_choice.md + 5th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -268,14 +284,14 @@ If building phase 1: args = parser.parse_args() if args.phase == 0: - if len(args.files) == 2: + if len(args.files) == 3: build_phase0_spec(*args.files) else: - print(" Phase 0 requires an output as well as an input file.") + print(" Phase 0 requires an output as well as spec and forkchoice files.") elif args.phase == 1: - if len(args.files) == 4: + if len(args.files) == 5: build_phase1_spec(*args.files) else: - print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)") + print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)") else: print("Invalid phase: {0}".format(args.phase)) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 0fd2fa476..36dd5b81f 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject: inserts = {} function_matcher = re.compile(FUNCTION_REGEX) inserts_matcher = re.compile(BEGIN_INSERT_REGEX) + is_ssz = False custom_types = {} for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 1910b6d04..25b33ab30 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,23 +8,26 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Prerequisites](#prerequisites) - - [Configuration](#configuration) + - [Constants](#constants) - [Time parameters](#time-parameters) - - [Beacon chain processing](#beacon-chain-processing) - - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) - - [Implementation notes](#implementation-notes) - - [Justification and finality at genesis](#justification-and-finality-at-genesis) + - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [`Target`](#target) + - [`Store`](#store) + - [`get_genesis_store`](#get_genesis_store) + - [`get_ancestor`](#get_ancestor) + - [`get_latest_attesting_balance`](#get_latest_attesting_balance) + - [`get_head`](#get_head) + - [Handlers](#handlers) + - [`on_tick`](#on_tick) + - [`on_block`](#on_block) + - [`on_attestation`](#on_attestation) ## Introduction -This document represents the specification for the beacon chain fork choice rule, part of Ethereum 2.0 Phase 0. - -## Prerequisites - -All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./0_beacon-chain.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md). ## Configuration @@ -34,76 +37,139 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | - | - | :-: | :-: | | `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds | -## Beacon chain processing +## Fork choice -Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: +The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running: -* The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. +* `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time +* `on_block(block)` whenever a block `block` is received +* `on_attestation(attestation)` whenever an attestation `attestation` is received -*Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. +*Notes*: -*Note*: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes. +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). +2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). -### Beacon chain fork choice rule +### Helpers -The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time, a validator `v` subjectively calculates the beacon chain head as follows. - -* Abstractly define `Store` as the type of storage object for the chain data, and let `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`. -* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store`, the processing of which sets `B` as finalized.) -* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists, set `justified_head` to `finalized_head`. -* Let `get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock` be the ancestor of `block` with slot number `slot`. The `get_ancestor` function can be defined recursively as: +#### `Target` ```python -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) +@dataclass +class Target(object): + epoch: Epoch + root: Hash ``` -* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. -* Let `get_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. -* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. -* Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. -* The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. +#### `Store` ```python -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validators - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [(i, get_attestation_target(store, i)) for i in active_validator_indices] +@dataclass +class Store(object): + blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) + states: Dict[Hash, BeaconState] = field(default_factory=dict) + time: int = 0 + latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) + justified_root: Hash = ZERO_HASH + finalized_root: Hash = ZERO_HASH +``` - # Use the rounded-balance-with-hysteresis supplied by the protocol for fork - # choice voting. This reduces the number of recomputations that need to be - # made for optimized implementations that precompute and save data - def get_vote_count(block: BeaconBlock) -> int: - return sum( - start_state.validators[validator_index].effective_balance - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) +#### `get_genesis_store` - head = start_block - while 1: - children = get_children(store, head) +```python +def get_genesis_store(genesis_state: BeaconState) -> Store: + genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) + root = signing_root(genesis_block) + return Store( + blocks={root: genesis_block}, + states={root: genesis_state}, + time=genesis_state.genesis_time, + justified_root=root, + finalized_root=root, + ) +``` + +#### `get_ancestor` + +```python +def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: + block = store.blocks[root] + assert block.slot >= slot + return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) +``` + +#### `get_latest_attesting_balance` + +```python +def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: + state = store.states[store.justified_root] + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) + return Gwei(sum( + state.validator_registry[i].effective_balance for i in active_indices + if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + )) +``` + +#### `get_head` + +```python +def get_head(store: Store) -> Hash: + # Execute the LMD-GHOST fork choice + head = store.justified_root + while True: + children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] if len(children) == 0: return head - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) ``` -## Implementation notes +### Handlers -### Justification and finality at genesis +#### `on_tick` -During genesis, justification and finality root fields within the `BeaconState` reference `ZERO_HASH` rather than a known block. `ZERO_HASH` in `previous_justified_root`, `current_justified_root`, and `finalized_root` should be considered as an alias to the root of the genesis block. +```python +def on_tick(store: Store, time: int) -> None: + store.time = time +``` + +#### `on_block` + +```python +def on_block(store: Store, block: BeaconBlock) -> None: + # Make a copy of the state to avoid mutability issues + pre_state = store.states[block.parent_root].copy() + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT + # Add new block to the store + store.blocks[signing_root(block)] = block + # Check block is a descendant of the finalized block + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check the block is valid and compute the post-state + state = state_transition(pre_state, block) + # Add new state to the store + store.states[signing_root(block)] = state + # Update justified block root + if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.current_justified_root + elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.previous_justified_root + # Update finalized block root + if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + store.finalized_root = state.finalized_root +``` + +#### `on_attestation` + +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + state = store.states[get_head(store)] + indexed_attestation = convert_to_indexed(state, attestation) + validate_indexed_attestation(state, indexed_attestation) + for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: + if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) +``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b83cd54f7..7b2bc9188 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -61,7 +61,6 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | -| `SECONDS_PER_SLOT` | `2**1 * 3**1` (= 6) | 6 seconds | ### Signature domains diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py new file mode 100644 index 000000000..4bc7e8b0a --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -0,0 +1,62 @@ +from typing import List + +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root + +from eth2spec.test.context import with_all_phases, with_state, bls_switch + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +@with_all_phases +@with_state +@bls_switch +def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + + # Initialization + store = spec.get_genesis_store(state) + blocks = [] + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + blocks.append(block) + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@with_state +@bls_switch +def test_on_attestation(spec, state): + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=1) + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f870336e8..077f94b49 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,13 @@ +import copy from types import GeneratorType -from typing import List, Iterable, TypeVar, Type, NewType -from typing import Union +from typing import ( + List, + Iterable, + TypeVar, + Type, + NewType, + Union, +) from typing_inspect import get_origin # SSZ integers @@ -145,6 +152,9 @@ class Container(object): def __hash__(self): return hash(self.hash_tree_root()) + def copy(self): + return copy.deepcopy(self) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -343,6 +353,8 @@ class BytesNMeta(type): return _is_bytes_n_instance_of(self, other.__class__) def __eq__(self, other): + if other == (): + return False return _is_equal_bytes_n_type(self, other) def __ne__(self, other): diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 2de2aa84b..b96b0a80f 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -3,4 +3,5 @@ eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 typing_inspect==0.4.0 +dataclasses==0.6 ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 3856640ab..fe14c498c 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -10,6 +10,7 @@ setup( "pycryptodome==3.7.3", "py_ecc>=1.6.0", "typing_inspect==0.4.0", - "ssz==0.1.0a10" + "ssz==0.1.0a10", + "dataclasses==0.6", ] )