From 11edda64ed91dafa801cb58f831e27b28a153a64 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 15 Jun 2019 18:42:03 -0400 Subject: [PATCH 01/29] Adds fork-choice to spec builder --- Makefile | 4 +- scripts/build_spec.py | 28 +++-- specs/core/0_fork-choice.md | 184 ++++++++++++++++++++---------- specs/core/1_shard-data-chains.md | 1 - 4 files changed, 140 insertions(+), 77 deletions(-) diff --git a/Makefile b/Makefile index f79b89dad..7e2025418 100644 --- a/Makefile +++ b/Makefile @@ -66,10 +66,10 @@ test_deposit_contract: pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7a51970e3..4213c38b6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -177,7 +177,7 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: items = list(objects.items()) for key, value in items: dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Dict|uint\d+|Bytes\d+|bytes', '', x), dependencies) for dep in dependencies: if dep in NEW_TYPES or len(dep) == 0: continue @@ -219,9 +219,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: return functions, constants, ssz_objects, inserts -def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: - functions, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) +def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: + phase0_spec = get_spec(phase0_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) + spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec) + spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -231,12 +233,14 @@ def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: def build_phase1_spec(phase0_sourcefile: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, + fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) + fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_custody, phase1_shard_data]: + for value in [phase1_custody, phase1_shard_data, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: @@ -250,13 +254,15 @@ if __name__ == '__main__': Build the specs from the md docs. If building phase 0: 1st argument is input spec.md - 2nd argument is output spec.py + 2nd argument is input fork_choice.md + 3rd argument is output spec.py If building phase 1: 1st argument is input spec_phase0.md 2nd argument is input spec_phase1_custody.md 3rd argument is input spec_phase1_shard_data.md - 4th argument is output spec.py + 4th argument is input fork_choice.md + 5th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -264,14 +270,14 @@ If building phase 1: args = parser.parse_args() if args.phase == 0: - if len(args.files) == 2: + if len(args.files) == 3: build_phase0_spec(*args.files) else: - print(" Phase 0 requires an output as well as an input file.") + print(" Phase 0 requires an output as well as spec and forkchoice files.") elif args.phase == 1: - if len(args.files) == 4: + if len(args.files) == 5: build_phase1_spec(*args.files) else: - print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)") + print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)") else: print("Invalid phase: {0}".format(args.phase)) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 91c3e27ee..c840179b9 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,23 +8,27 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Prerequisites](#prerequisites) - [Constants](#constants) - [Time parameters](#time-parameters) - - [Beacon chain processing](#beacon-chain-processing) - - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) - - [Implementation notes](#implementation-notes) - - [Justification and finality at genesis](#justification-and-finality-at-genesis) + - [Fork choice](#fork-choice) + - [Containers](#containers) + - [`Target`](#target) + - [`Store`](#store) + - [Helpers](#helpers) + - [`get_genesis_store`](#get_genesis_store) + - [`get_ancestor`](#get_ancestor) + - [`get_attesting_balance_from_store`](#get_attesting_balance_from_store) + - [`get_head`](#get_head) + - [Handlers](#handlers) + - [`on_tick`](#on_tick) + - [`on_block`](#on_block) + - [`on_attestation`](#on_attestation) ## Introduction -This document represents the specification for the beacon chain fork choice rule, part of Ethereum 2.0 Phase 0. - -## Prerequisites - -All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./0_beacon-chain.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md). ## Constants @@ -34,76 +38,130 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | - | - | :-: | :-: | | `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds | -## Beacon chain processing +## Fork choice -Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: +The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running: -* The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. +* `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time +* `on_block(block)` whenever a block `block` is received +* `on_attestation(attestation)` whenever an attestation `attestation` is received -*Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. +*Notes*: -*Note*: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes. +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. +2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -### Beacon chain fork choice rule +### Containers -The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time, a validator `v` subjectively calculates the beacon chain head as follows. - -* Abstractly define `Store` as the type of storage object for the chain data, and let `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`. -* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store`, the processing of which sets `B` as finalized.) -* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists, set `justified_head` to `finalized_head`. -* Let `get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock` be the ancestor of `block` with slot number `slot`. The `get_ancestor` function can be defined recursively as: +#### `Target` ```python -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) +class Target(Container): + epoch: Epoch + root: Bytes32 ``` -* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. -* Let `get_latest_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. -* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. -* Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. -* The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. +#### `Store` ```python -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validator_registry - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [(i, get_latest_attestation_target(store, i)) for i in active_validator_indices] +class Store(Container): + blocks: Dict[Bytes32, BeaconBlock] + states: Dict[Bytes32, BeaconState] + time: uint64 + latest_targets: Dict[ValidatorIndex, Target] + justified_root: Bytes32 + finalized_root: Bytes32 +``` - # Use the rounded-balance-with-hysteresis supplied by the protocol for fork - # choice voting. This reduces the number of recomputations that need to be - # made for optimized implementations that precompute and save data - def get_vote_count(block: BeaconBlock) -> int: - return sum( - start_state.validator_registry[validator_index].effective_balance - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) +### Helpers - head = start_block - while 1: - children = get_children(store, head) +#### `get_genesis_store` + +```python +def get_genesis_store(genesis_state: BeaconState) -> Store: + genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) + root = signing_root(genesis_block) + return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) +``` + +#### `get_ancestor` + +```python +def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: + block = store.blocks[root] + assert block.slot >= slot + return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) +``` + +#### `get_attesting_balance_from_store` + +```python +def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: + state = store.states[store.justified_root] + active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + return sum( + state.validator_registry[i].effective_balance for i in active_indices + if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root + ) +``` + +#### `get_head` + +```python +def get_head(store: Store) -> Bytes32: + # Execute the LMD-GHOST fork choice + head = store.justified_root + while True: + children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] if len(children) == 0: return head - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) + # Sort by attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_attesting_balance_from_store(store, root), root)) ``` -## Implementation notes +### Handlers -### Justification and finality at genesis +#### `on_tick` -During genesis, justification and finality root fields within the `BeaconState` reference `ZERO_HASH` rather than a known block. `ZERO_HASH` in `previous_justified_root`, `current_justified_root`, and `finalized_root` should be considered as an alias to the root of the genesis block. +```python +def on_tick(store: Store, time: int) -> None: + store.time = time +``` + +#### `on_block` + +```python +def on_block(store: Store, block: BeaconBlock) -> None: + # Add new block to the store + store.blocks[signing_root(block)] = block + # Check block is a descendant of the finalized block + assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root + # Check block slot against Unix time + pre_state = store.states[block.parent_root].copy() + assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT + # Check the block is valid and compute the post-state + state = state_transition(pre_state, block) + # Add new state to the store + store.states[signing_root(block)] = state + # Update justified and finalized blocks + if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + store.finalized_root = state.finalized_root + if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.current_justified_root + if state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + store.justified_root = state.previous_justified_root +``` + +#### `on_attestation` + +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + state = store.states[get_head(store)] + indexed_attestation = convert_to_indexed(state, attestation) + validate_indexed_attestation(state, indexed_attestation) + for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: + if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: + store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) +``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 21e08e7c9..84af66ffb 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -56,7 +56,6 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | -| `SECONDS_PER_SLOT` | `2**1 * 3**1` (= 6) | 6 seconds | ### Signature domains From 061ecf7d0a870bf5336cffa1a3567054e5978c27 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 15 Jun 2019 19:13:56 -0400 Subject: [PATCH 02/29] Adds fork-choice tests --- .../pyspec/eth2spec/test/test_fork_choice.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/test_fork_choice.py diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py new file mode 100644 index 000000000..46b60b93c --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -0,0 +1,50 @@ +from eth2spec.utils.ssz.ssz_impl import signing_root + +from eth2spec.test.context import with_all_phases, spec_state_test + +from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.helpers.state import next_slot + + +@with_all_phases +@spec_state_test +def test_basic(spec, state): + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + assert store.time == time + + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(state) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(state) + block.slot += spec.SLOTS_PER_EPOCH + + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + + # TODO: add tests for justified_root and finalized_root + + +@with_all_phases +@spec_state_test +def test_on_attestation(spec, state): + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_slot(state) + + attestation = get_valid_attestation(state, slot=1) + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target(attestation.data.target_epoch, attestation.data.target_root) + ) \ No newline at end of file From 5d10cd63c7301e68ef13dbc008e7a34eccdc4763 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 09:52:52 -0400 Subject: [PATCH 03/29] SSZ (partially) handles Dicts --- specs/core/0_fork-choice.md | 5 +- .../pyspec/eth2spec/test/test_fork_choice.py | 61 ++++++++++++------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 37 ++++++++++- 3 files changed, 78 insertions(+), 25 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index c840179b9..3ed794584 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -163,5 +163,8 @@ def on_attestation(store: Store, attestation: Attestation) -> None: validate_indexed_attestation(state, indexed_attestation) for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) + store.latest_targets[i] = Target( + epoch = attestation.data.target_epoch, + root = attestation.data.target_root, + ) ``` diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 46b60b93c..374466032 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -10,41 +10,56 @@ from eth2spec.test.helpers.state import next_slot @with_all_phases @spec_state_test def test_basic(spec, state): + yield 'pre', state + # Initialization - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) - assert store.time == time + store = spec.get_genesis_store(state) + blocks = [] + time = 100 + spec.on_tick(store, time) + assert store.time == time # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(state) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + yield 'blocks', blocks, List[spec.BeaconBlock] # TODO: add tests for justified_root and finalized_root + yield 'post', state @with_all_phases @spec_state_test def test_on_attestation(spec, state): - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) + yield 'pre', state - next_slot(state) + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) - attestation = get_valid_attestation(state, slot=1) - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target(attestation.data.target_epoch, attestation.data.target_root) - ) \ No newline at end of file + next_slot(spec, state) + + attestation = get_valid_attestation(spec, state, slot=1) + yield 'attestation', attestation + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target( + epoch = attestation.data.target_epoch, + root = attestation.data.target_root + ) + ) + + yield 'post', state \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 55ced4ee2..7119f2e5b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import List, Iterable, TypeVar, Type, NewType +from typing import List, Iterable, TypeVar, Type, NewType, Dict from typing import Union from typing_inspect import get_origin @@ -280,6 +280,32 @@ class Vector(metaclass=VectorMeta): return self.hash_tree_root() == other.hash_tree_root() +# # Super Secret Un-documented SSZ Dict (for forkchoice) +# # ----------------------------- +class Dict(dict): + def __init__(self,*args,**kwargs) : dict.__init__(self,*args,**kwargs) + + def serialize(self): + raise NotImplementedError + + def hash_tree_root(self): + raise NotImplementedError + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + def __eq__(self, other): + raise NotImplementedError + # SSZ BytesN # ----------------------------- @@ -407,6 +433,8 @@ def get_zero_value(typ): return b'' elif is_container_type(typ): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + elif is_dict_type(typ): + return dict() else: raise Exception("Type not supported: {}".format(typ)) @@ -498,6 +526,13 @@ def is_container_type(typ): return isinstance(typ, type) and issubclass(typ, Container) +def is_dict_type(typ): + """ + Check of the given type is a Dict. (Which are a part of the super-secret undocumented SSZ spec) + """ + return get_origin(typ) is Dict or get_origin(typ) is dict + + T = TypeVar('T') L = TypeVar('L') From 4deb311b71ea6e1dabe956dc2de374d517d9dc35 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 12:17:31 -0400 Subject: [PATCH 04/29] Debugging 1st test --- specs/core/0_fork-choice.md | 7 +- .../pyspec/eth2spec/test/test_finality.py | 334 +++++++++--------- .../pyspec/eth2spec/test/test_fork_choice.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 41 +-- 4 files changed, 187 insertions(+), 197 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3ed794584..3d1e70ca4 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -83,6 +83,7 @@ class Store(Container): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) + print('groot', root) return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) ``` @@ -90,6 +91,7 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: ```python def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: + print('ruut', root) block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) @@ -135,14 +137,17 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Add new block to the store + print('setting', signing_root(block)) store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() + print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) + print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) # Add new state to the store store.states[signing_root(block)] = state # Update justified and finalized blocks @@ -150,7 +155,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: store.finalized_root = state.finalized_root if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.current_justified_root - if state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): + elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.previous_justified_root ``` diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 801e8b4fd..9556abbd1 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,203 +1,203 @@ -from copy import deepcopy -from typing import List +# from copy import deepcopy +# from typing import List -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block -from eth2spec.test.helpers.attestations import get_valid_attestation +# from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +# from eth2spec.test.helpers.state import next_epoch +# from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block +# from eth2spec.test.helpers.attestations import get_valid_attestation -def check_finality(spec, - state, - prev_state, - current_justified_changed, - previous_justified_changed, - finalized_changed): - if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root - else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root +# def check_finality(spec, +# state, +# prev_state, +# current_justified_changed, +# previous_justified_changed, +# finalized_changed): +# if current_justified_changed: +# assert state.current_justified_epoch > prev_state.current_justified_epoch +# assert state.current_justified_root != prev_state.current_justified_root +# else: +# assert state.current_justified_epoch == prev_state.current_justified_epoch +# assert state.current_justified_root == prev_state.current_justified_root - if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root - else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root +# if previous_justified_changed: +# assert state.previous_justified_epoch > prev_state.previous_justified_epoch +# assert state.previous_justified_root != prev_state.previous_justified_root +# else: +# assert state.previous_justified_epoch == prev_state.previous_justified_epoch +# assert state.previous_justified_root == prev_state.previous_justified_root - if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root - else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root +# if finalized_changed: +# assert state.finalized_epoch > prev_state.finalized_epoch +# assert state.finalized_root != prev_state.finalized_root +# else: +# assert state.finalized_epoch == prev_state.finalized_epoch +# assert state.finalized_root == prev_state.finalized_root -def next_epoch_with_attestations(spec, - state, - fill_cur_epoch, - fill_prev_epoch): - post_state = deepcopy(state) - blocks = [] - for _ in range(spec.SLOTS_PER_EPOCH): - block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch: - slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(cur_attestation) +# def next_epoch_with_attestations(spec, +# state, +# fill_cur_epoch, +# fill_prev_epoch): +# post_state = deepcopy(state) +# blocks = [] +# for _ in range(spec.SLOTS_PER_EPOCH): +# block = build_empty_block_for_next_slot(spec, post_state) +# if fill_cur_epoch: +# slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 +# if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): +# cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) +# block.body.attestations.append(cur_attestation) - if fill_prev_epoch: - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) - block.body.attestations.append(prev_attestation) +# if fill_prev_epoch: +# slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 +# prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) +# block.body.attestations.append(prev_attestation) - spec.state_transition(post_state, block) - blocks.append(block) +# spec.state_transition(post_state, block) +# blocks.append(block) - return state, blocks, post_state +# return state, blocks, post_state -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_4(spec, state): - yield 'pre', state +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_4(spec, state): +# yield 'pre', state - blocks = [] - for epoch in range(4): - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks +# blocks = [] +# for epoch in range(4): +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks - # justification/finalization skipped at GENESIS_EPOCH - if epoch == 0: - check_finality(spec, state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - elif epoch == 1: - check_finality(spec, state, prev_state, False, False, False) - elif epoch == 2: - check_finality(spec, state, prev_state, True, False, False) - elif epoch >= 3: - # rule 4 of finality - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root +# # justification/finalization skipped at GENESIS_EPOCH +# if epoch == 0: +# check_finality(spec, state, prev_state, False, False, False) +# # justification/finalization skipped at GENESIS_EPOCH + 1 +# elif epoch == 1: +# check_finality(spec, state, prev_state, False, False, False) +# elif epoch == 2: +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch >= 3: +# # rule 4 of finality +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.current_justified_epoch +# assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_1(spec, state): - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_1(spec, state): +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) - yield 'pre', state +# yield 'pre', state - blocks = [] - for epoch in range(3): - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - blocks += new_blocks +# blocks = [] +# for epoch in range(3): +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# blocks += new_blocks - if epoch == 0: - check_finality(spec, state, prev_state, True, False, False) - elif epoch == 1: - check_finality(spec, state, prev_state, True, True, False) - elif epoch == 2: - # finalized by rule 1 - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root +# if epoch == 0: +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch == 1: +# check_finality(spec, state, prev_state, True, True, False) +# elif epoch == 2: +# # finalized by rule 1 +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.previous_justified_epoch +# assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_2(spec, state): - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_2(spec, state): +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) - yield 'pre', state +# yield 'pre', state - blocks = [] - for epoch in range(3): - if epoch == 0: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - check_finality(spec, state, prev_state, True, False, False) - elif epoch == 1: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) - check_finality(spec, state, prev_state, False, True, False) - elif epoch == 2: - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - # finalized by rule 2 - check_finality(spec, state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root +# blocks = [] +# for epoch in range(3): +# if epoch == 0: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# check_finality(spec, state, prev_state, True, False, False) +# elif epoch == 1: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) +# check_finality(spec, state, prev_state, False, True, False) +# elif epoch == 2: +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# # finalized by rule 2 +# check_finality(spec, state, prev_state, True, False, True) +# assert state.finalized_epoch == prev_state.previous_justified_epoch +# assert state.finalized_root == prev_state.previous_justified_root - blocks += new_blocks +# blocks += new_blocks - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state -@with_all_phases -@never_bls -@spec_state_test -def test_finality_rule_3(spec, state): - """ - Test scenario described here - https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 - """ - # get past first two epochs that finality does not run on - next_epoch(spec, state) - apply_empty_block(spec, state) - next_epoch(spec, state) - apply_empty_block(spec, state) +# @with_all_phases +# @never_bls +# @spec_state_test +# def test_finality_rule_3(spec, state): +# """ +# Test scenario described here +# https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 +# """ +# # get past first two epochs that finality does not run on +# next_epoch(spec, state) +# apply_empty_block(spec, state) +# next_epoch(spec, state) +# apply_empty_block(spec, state) - yield 'pre', state +# yield 'pre', state - blocks = [] - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks - check_finality(spec, state, prev_state, True, False, False) +# blocks = [] +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, True, False, False) - # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) - blocks += new_blocks - check_finality(spec, state, prev_state, True, True, True) +# # In epoch N, JE is set to N, prev JE is set to N-1 +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, True, True, True) - # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) - blocks += new_blocks - check_finality(spec, state, prev_state, False, True, False) +# # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) +# blocks += new_blocks +# check_finality(spec, state, prev_state, False, True, False) - # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. - # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) - blocks += new_blocks - # rule 2 - check_finality(spec, state, prev_state, True, False, True) +# # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. +# # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) +# blocks += new_blocks +# # rule 2 +# check_finality(spec, state, prev_state, True, False, True) - # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) - blocks += new_blocks - # rule 3 - check_finality(spec, state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root +# # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. +# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) +# blocks += new_blocks +# # rule 3 +# check_finality(spec, state, prev_state, True, True, True) +# assert state.finalized_epoch == prev_state.current_justified_epoch +# assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] - yield 'post', state +# yield 'blocks', blocks, List[spec.BeaconBlock] +# yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 374466032..c1df605fe 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -58,7 +58,7 @@ def test_on_attestation(spec, state): store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == spec.Target( epoch = attestation.data.target_epoch, - root = attestation.data.target_root + root = attestation.data.target_root, ) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7119f2e5b..5b3500d3e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,5 +1,13 @@ -from typing import List, Iterable, TypeVar, Type, NewType, Dict -from typing import Union +import copy +from typing import ( + List, + Iterable, + TypeVar, + Type, + NewType, + Dict, + Union, +) from typing_inspect import get_origin # SSZ integers @@ -123,6 +131,9 @@ class Container(object): def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] + + def copy(self): + return self.deepcopy(self) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -280,32 +291,6 @@ class Vector(metaclass=VectorMeta): return self.hash_tree_root() == other.hash_tree_root() -# # Super Secret Un-documented SSZ Dict (for forkchoice) -# # ----------------------------- -class Dict(dict): - def __init__(self,*args,**kwargs) : dict.__init__(self,*args,**kwargs) - - def serialize(self): - raise NotImplementedError - - def hash_tree_root(self): - raise NotImplementedError - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return len(self.items) - - def __eq__(self, other): - raise NotImplementedError - # SSZ BytesN # ----------------------------- From f421850dc91db05144b5792912f1c4296ba25cce Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 15:53:42 -0400 Subject: [PATCH 05/29] Fixes tests! Thanks @hwwhww! --- specs/core/0_fork-choice.md | 5 - .../pyspec/eth2spec/test/test_finality.py | 334 +++++++++--------- .../pyspec/eth2spec/test/test_fork_choice.py | 5 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 4 files changed, 172 insertions(+), 174 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3d1e70ca4..9a0657081 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -83,7 +83,6 @@ class Store(Container): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - print('groot', root) return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) ``` @@ -91,7 +90,6 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: ```python def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: - print('ruut', root) block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) @@ -137,17 +135,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: # Add new block to the store - print('setting', signing_root(block)) store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() - print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) - print('store.states[block.parent_root]', hash_tree_root(store.states[block.parent_root])) # Add new state to the store store.states[signing_root(block)] = state # Update justified and finalized blocks diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 9556abbd1..801e8b4fd 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,203 +1,203 @@ -# from copy import deepcopy -# from typing import List +from copy import deepcopy +from typing import List -# from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -# from eth2spec.test.helpers.state import next_epoch -# from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block -# from eth2spec.test.helpers.attestations import get_valid_attestation +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block +from eth2spec.test.helpers.attestations import get_valid_attestation -# def check_finality(spec, -# state, -# prev_state, -# current_justified_changed, -# previous_justified_changed, -# finalized_changed): -# if current_justified_changed: -# assert state.current_justified_epoch > prev_state.current_justified_epoch -# assert state.current_justified_root != prev_state.current_justified_root -# else: -# assert state.current_justified_epoch == prev_state.current_justified_epoch -# assert state.current_justified_root == prev_state.current_justified_root +def check_finality(spec, + state, + prev_state, + current_justified_changed, + previous_justified_changed, + finalized_changed): + if current_justified_changed: + assert state.current_justified_epoch > prev_state.current_justified_epoch + assert state.current_justified_root != prev_state.current_justified_root + else: + assert state.current_justified_epoch == prev_state.current_justified_epoch + assert state.current_justified_root == prev_state.current_justified_root -# if previous_justified_changed: -# assert state.previous_justified_epoch > prev_state.previous_justified_epoch -# assert state.previous_justified_root != prev_state.previous_justified_root -# else: -# assert state.previous_justified_epoch == prev_state.previous_justified_epoch -# assert state.previous_justified_root == prev_state.previous_justified_root + if previous_justified_changed: + assert state.previous_justified_epoch > prev_state.previous_justified_epoch + assert state.previous_justified_root != prev_state.previous_justified_root + else: + assert state.previous_justified_epoch == prev_state.previous_justified_epoch + assert state.previous_justified_root == prev_state.previous_justified_root -# if finalized_changed: -# assert state.finalized_epoch > prev_state.finalized_epoch -# assert state.finalized_root != prev_state.finalized_root -# else: -# assert state.finalized_epoch == prev_state.finalized_epoch -# assert state.finalized_root == prev_state.finalized_root + if finalized_changed: + assert state.finalized_epoch > prev_state.finalized_epoch + assert state.finalized_root != prev_state.finalized_root + else: + assert state.finalized_epoch == prev_state.finalized_epoch + assert state.finalized_root == prev_state.finalized_root -# def next_epoch_with_attestations(spec, -# state, -# fill_cur_epoch, -# fill_prev_epoch): -# post_state = deepcopy(state) -# blocks = [] -# for _ in range(spec.SLOTS_PER_EPOCH): -# block = build_empty_block_for_next_slot(spec, post_state) -# if fill_cur_epoch: -# slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 -# if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): -# cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) -# block.body.attestations.append(cur_attestation) +def next_epoch_with_attestations(spec, + state, + fill_cur_epoch, + fill_prev_epoch): + post_state = deepcopy(state) + blocks = [] + for _ in range(spec.SLOTS_PER_EPOCH): + block = build_empty_block_for_next_slot(spec, post_state) + if fill_cur_epoch: + slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 + if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): + cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(cur_attestation) -# if fill_prev_epoch: -# slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 -# prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) -# block.body.attestations.append(prev_attestation) + if fill_prev_epoch: + slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 + prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest) + block.body.attestations.append(prev_attestation) -# spec.state_transition(post_state, block) -# blocks.append(block) + spec.state_transition(post_state, block) + blocks.append(block) -# return state, blocks, post_state + return state, blocks, post_state -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_4(spec, state): -# yield 'pre', state +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_4(spec, state): + yield 'pre', state -# blocks = [] -# for epoch in range(4): -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks + blocks = [] + for epoch in range(4): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks -# # justification/finalization skipped at GENESIS_EPOCH -# if epoch == 0: -# check_finality(spec, state, prev_state, False, False, False) -# # justification/finalization skipped at GENESIS_EPOCH + 1 -# elif epoch == 1: -# check_finality(spec, state, prev_state, False, False, False) -# elif epoch == 2: -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch >= 3: -# # rule 4 of finality -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.current_justified_epoch -# assert state.finalized_root == prev_state.current_justified_root + # justification/finalization skipped at GENESIS_EPOCH + if epoch == 0: + check_finality(spec, state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 + elif epoch == 1: + check_finality(spec, state, prev_state, False, False, False) + elif epoch == 2: + check_finality(spec, state, prev_state, True, False, False) + elif epoch >= 3: + # rule 4 of finality + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_1(spec, state): -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_1(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) -# yield 'pre', state + yield 'pre', state -# blocks = [] -# for epoch in range(3): -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# blocks += new_blocks + blocks = [] + for epoch in range(3): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + blocks += new_blocks -# if epoch == 0: -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch == 1: -# check_finality(spec, state, prev_state, True, True, False) -# elif epoch == 2: -# # finalized by rule 1 -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.previous_justified_epoch -# assert state.finalized_root == prev_state.previous_justified_root + if epoch == 0: + check_finality(spec, state, prev_state, True, False, False) + elif epoch == 1: + check_finality(spec, state, prev_state, True, True, False) + elif epoch == 2: + # finalized by rule 1 + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_2(spec, state): -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_2(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) -# yield 'pre', state + yield 'pre', state -# blocks = [] -# for epoch in range(3): -# if epoch == 0: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# check_finality(spec, state, prev_state, True, False, False) -# elif epoch == 1: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) -# check_finality(spec, state, prev_state, False, True, False) -# elif epoch == 2: -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# # finalized by rule 2 -# check_finality(spec, state, prev_state, True, False, True) -# assert state.finalized_epoch == prev_state.previous_justified_epoch -# assert state.finalized_root == prev_state.previous_justified_root + blocks = [] + for epoch in range(3): + if epoch == 0: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + check_finality(spec, state, prev_state, True, False, False) + elif epoch == 1: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) + check_finality(spec, state, prev_state, False, True, False) + elif epoch == 2: + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + # finalized by rule 2 + check_finality(spec, state, prev_state, True, False, True) + assert state.finalized_epoch == prev_state.previous_justified_epoch + assert state.finalized_root == prev_state.previous_justified_root -# blocks += new_blocks + blocks += new_blocks -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state -# @with_all_phases -# @never_bls -# @spec_state_test -# def test_finality_rule_3(spec, state): -# """ -# Test scenario described here -# https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 -# """ -# # get past first two epochs that finality does not run on -# next_epoch(spec, state) -# apply_empty_block(spec, state) -# next_epoch(spec, state) -# apply_empty_block(spec, state) +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_3(spec, state): + """ + Test scenario described here + https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 + """ + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) -# yield 'pre', state + yield 'pre', state -# blocks = [] -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, True, False, False) + blocks = [] + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + check_finality(spec, state, prev_state, True, False, False) -# # In epoch N, JE is set to N, prev JE is set to N-1 -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, True, True, True) + # In epoch N, JE is set to N, prev JE is set to N-1 + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + check_finality(spec, state, prev_state, True, True, True) -# # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) -# blocks += new_blocks -# check_finality(spec, state, prev_state, False, True, False) + # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, False) + blocks += new_blocks + check_finality(spec, state, prev_state, False, True, False) -# # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. -# # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) -# blocks += new_blocks -# # rule 2 -# check_finality(spec, state, prev_state, True, False, True) + # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. + # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, False, True) + blocks += new_blocks + # rule 2 + check_finality(spec, state, prev_state, True, False, True) -# # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. -# prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) -# blocks += new_blocks -# # rule 3 -# check_finality(spec, state, prev_state, True, True, True) -# assert state.finalized_epoch == prev_state.current_justified_epoch -# assert state.finalized_root == prev_state.current_justified_root + # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, True) + blocks += new_blocks + # rule 3 + check_finality(spec, state, prev_state, True, True, True) + assert state.finalized_epoch == prev_state.current_justified_epoch + assert state.finalized_root == prev_state.current_justified_root -# yield 'blocks', blocks, List[spec.BeaconBlock] -# yield 'post', state + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index c1df605fe..6f054fc52 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -1,4 +1,6 @@ -from eth2spec.utils.ssz.ssz_impl import signing_root +from typing import List + +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root from eth2spec.test.context import with_all_phases, spec_state_test @@ -10,6 +12,7 @@ from eth2spec.test.helpers.state import next_slot @with_all_phases @spec_state_test def test_basic(spec, state): + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) yield 'pre', state # Initialization diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5b3500d3e..7df95d119 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -133,7 +133,7 @@ class Container(object): return [getattr(self, field) for field in cls.get_field_names()] def copy(self): - return self.deepcopy(self) + return copy.deepcopy(self) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) From 9f2d06b2e7e74f977f20c35dbda7617d44511bda Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 16 Jun 2019 16:02:56 -0400 Subject: [PATCH 06/29] Somehow I had indented with 5 spaces everywhere. --- specs/core/0_fork-choice.md | 4 +- .../pyspec/eth2spec/test/test_fork_choice.py | 78 +++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 9a0657081..f3d57feaa 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -164,7 +164,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: store.latest_targets[i] = Target( - epoch = attestation.data.target_epoch, - root = attestation.data.target_root, + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, ) ``` diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index 6f054fc52..ab1728251 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -12,57 +12,57 @@ from eth2spec.test.helpers.state import next_slot @with_all_phases @spec_state_test def test_basic(spec, state): - state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - yield 'pre', state + state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) + yield 'pre', state # Initialization - store = spec.get_genesis_store(state) - blocks = [] - time = 100 - spec.on_tick(store, time) - assert store.time == time + store = spec.get_genesis_store(state) + blocks = [] + time = 100 + spec.on_tick(store, time) + assert store.time == time - # On receiving a block of `GENESIS_SLOT + 1` slot - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block + # On receiving a block of `GENESIS_SLOT + 1` slot + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block - # On receiving a block of next epoch - store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH - blocks.append(block) + # On receiving a block of next epoch + store.time = time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH + blocks.append(block) - spec.on_block(store, block) - assert store.blocks[signing_root(block)] == block - yield 'blocks', blocks, List[spec.BeaconBlock] + spec.on_block(store, block) + assert store.blocks[signing_root(block)] == block + yield 'blocks', blocks, List[spec.BeaconBlock] - # TODO: add tests for justified_root and finalized_root - yield 'post', state + # TODO: add tests for justified_root and finalized_root + yield 'post', state @with_all_phases @spec_state_test def test_on_attestation(spec, state): - yield 'pre', state + yield 'pre', state - store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) - next_slot(spec, state) + next_slot(spec, state) - attestation = get_valid_attestation(spec, state, slot=1) - yield 'attestation', attestation - indexed_attestation = spec.convert_to_indexed(state, attestation) - spec.on_attestation(store, attestation) - assert ( - store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == - spec.Target( - epoch = attestation.data.target_epoch, - root = attestation.data.target_root, - ) - ) + attestation = get_valid_attestation(spec, state, slot=1) + yield 'attestation', attestation + indexed_attestation = spec.convert_to_indexed(state, attestation) + spec.on_attestation(store, attestation) + assert ( + store.latest_targets[indexed_attestation.custody_bit_0_indices[0]] == + spec.Target( + epoch=attestation.data.target_epoch, + root=attestation.data.target_root, + ) + ) - yield 'post', state \ No newline at end of file + yield 'post', state diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7df95d119..3bd7dd062 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -131,7 +131,7 @@ class Container(object): def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] - + def copy(self): return copy.deepcopy(self) From 2ea6cede3edad9df50088378fd82f2037558f513 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 10:48:33 -0400 Subject: [PATCH 07/29] Moves fork-choice objects away from SSZ --- scripts/build_spec.py | 14 +++++++++++++ scripts/function_puller.py | 1 + specs/core/0_fork-choice.md | 20 ++++++++++--------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 14 ------------- test_libs/pyspec/requirements.txt | 1 + 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 4213c38b6..1091e7bed 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -19,6 +19,13 @@ PHASE0_IMPORTS = '''from typing import ( Tuple, ) +from dataclasses import ( + dataclass, + field, +) + +from copy import deepcopy + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -44,6 +51,13 @@ PHASE1_IMPORTS = '''from typing import ( Tuple, ) +from dataclasses import ( + dataclass, + field, +) + +from copy import deepcopy + from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 303d4ec2f..776fcfa82 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject: inserts = {} function_matcher = re.compile(FUNCTION_REGEX) inserts_matcher = re.compile(BEGIN_INSERT_REGEX) + is_ssz = False for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f3d57feaa..1a1465d25 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -58,7 +58,8 @@ The head block root associated with a `store` is defined as `get_head(store)`. A #### `Target` ```python -class Target(Container): +@dataclass +class Target(object): epoch: Epoch root: Bytes32 ``` @@ -66,13 +67,14 @@ class Target(Container): #### `Store` ```python -class Store(Container): - blocks: Dict[Bytes32, BeaconBlock] - states: Dict[Bytes32, BeaconState] - time: uint64 - latest_targets: Dict[ValidatorIndex, Target] - justified_root: Bytes32 - finalized_root: Bytes32 +@dataclass +class Store(object): + blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) + states: Dict[Bytes32, BeaconState] = field(default_factory=dict) + time: int = 0 + latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) + justified_root: Bytes32 = ZERO_HASH + finalized_root: Bytes32 = ZERO_HASH ``` ### Helpers @@ -139,7 +141,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time - pre_state = store.states[block.parent_root].copy() + pre_state = deepcopy(store.states[block.parent_root]) assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 3bd7dd062..d4b481ef2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,11 +1,9 @@ -import copy from typing import ( List, Iterable, TypeVar, Type, NewType, - Dict, Union, ) from typing_inspect import get_origin @@ -132,9 +130,6 @@ class Container(object): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] - def copy(self): - return copy.deepcopy(self) - def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -418,8 +413,6 @@ def get_zero_value(typ): return b'' elif is_container_type(typ): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - elif is_dict_type(typ): - return dict() else: raise Exception("Type not supported: {}".format(typ)) @@ -511,13 +504,6 @@ def is_container_type(typ): return isinstance(typ, type) and issubclass(typ, Container) -def is_dict_type(typ): - """ - Check of the given type is a Dict. (Which are a part of the super-secret undocumented SSZ spec) - """ - return get_origin(typ) is Dict or get_origin(typ) is dict - - T = TypeVar('T') L = TypeVar('L') diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 3b38930bd..18e5d2a4c 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -3,3 +3,4 @@ eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 typing_inspect==0.4.0 +dataclasses==0.6 From bb36660d569d98ef295fcc148ab9ef10f35c4403 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 11:08:22 -0400 Subject: [PATCH 08/29] Kick the CI cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e9a77c49..21118e88d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v2-pyspec + venv_name: v3-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: From c31a42632fdb9d41096f2cc6d12496525114c3ce Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 17 Jun 2019 11:11:38 -0400 Subject: [PATCH 09/29] Kick the other CI cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21118e88d..dbecd13b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v4-deposit-contract + venv_name: v5-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v4-deposit-contract + venv_name: v5-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From d5d35b1a3ef521d14124c7f2e924fd932a887c16 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:09:55 +0200 Subject: [PATCH 10/29] Trail upgrade CI to python 3.7 --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbecd13b3..70c51b978 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,7 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: # Restore git repo at point close to target branch/revision, to speed up checkout @@ -80,7 +80,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +92,7 @@ jobs: - save_pyspec_cached_venv test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +105,7 @@ jobs: path: test_libs/pyspec/test-reports lint: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -116,7 +116,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: @@ -128,7 +128,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 working_directory: ~/specs-repo steps: - restore_cache: From 28a3e54db094942ce4b37f78239df7455f3147af Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:13:04 +0200 Subject: [PATCH 11/29] Actually resolve some merge conflicts --- specs/core/0_fork-choice.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 05735e081..e4ea773c8 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,12 +8,7 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) -<<<<<<< HEAD - [Constants](#constants) -======= - - [Prerequisites](#prerequisites) - - [Configuration](#configuration) ->>>>>>> dev - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Containers](#containers) @@ -82,13 +77,7 @@ class Store(object): finalized_root: Bytes32 = ZERO_HASH ``` -<<<<<<< HEAD ### Helpers -======= -* The parent block with root `block.parent_root` has been processed and accepted. -* An Ethereum 1.0 block pointed to by the `state.eth1_data.block_hash` has been processed and accepted. -* The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. ->>>>>>> dev #### `get_genesis_store` From eb9fc7b518944db129ac673180cad039206983f4 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:26:12 +0200 Subject: [PATCH 12/29] Remove circle ci repo caching --- .circleci/config.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 70c51b978..2b5e7a292 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,15 +60,9 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - # Restore git repo at point close to target branch/revision, to speed up checkout - - restore_cache: - keys: - - v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - v1-specs-repo-{{ .Branch }}- - - v1-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size @@ -80,7 +74,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +86,7 @@ jobs: - save_pyspec_cached_venv test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +99,7 @@ jobs: path: test_libs/pyspec/test-reports lint: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -116,7 +110,7 @@ jobs: command: make lint install_deposit_contract_test: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: @@ -128,7 +122,7 @@ jobs: - save_deposit_contract_cached_venv deposit_contract: docker: - - image: circleci/python:3.7 + - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: From dfdd283908b2c74280f0ef3ef04c959fa00d8cc6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:30:01 +0200 Subject: [PATCH 13/29] Python 3.7 -> 3.6 again --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b5e7a292..dbecd13b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,12 @@ jobs: - image: circleci/python:3.6 working_directory: ~/specs-repo steps: + # Restore git repo at point close to target branch/revision, to speed up checkout + - restore_cache: + keys: + - v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - v1-specs-repo-{{ .Branch }}- + - v1-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size From 686273e1d6a37f434dc77259d119a2142d8eaf53 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 15:34:07 +0200 Subject: [PATCH 14/29] Maybe venv cache name collision --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbecd13b3..9ba9934a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,26 +35,26 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v3-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v3-pyspec + venv_name: v3-pyspec-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v5-deposit-contract + venv_name: v5-deposit-contract-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v5-deposit-contract + venv_name: v5-deposit-contract-bump2 reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 3bcddf5e49521192d415e89727fe38dea4a975ac Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 20:59:17 +0200 Subject: [PATCH 15/29] Removes defaults from Store to try get CI to pass --- specs/core/0_fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index e4ea773c8..f57fd84b8 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -73,8 +73,8 @@ class Store(object): states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = ZERO_HASH - finalized_root: Bytes32 = ZERO_HASH + justified_root: Bytes32 + finalized_root: Bytes32 ``` ### Helpers From d804cb307bc4a684abe875bb1d0a402769191f3d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 21:50:40 +0200 Subject: [PATCH 16/29] Removes defaults from Store to try get CI to pass --- specs/core/0_fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index f57fd84b8..11d6453e9 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -73,8 +73,8 @@ class Store(object): states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 - finalized_root: Bytes32 + justified_root: Bytes32 = 0x0 + finalized_root: Bytes32 = 0x0 ``` ### Helpers From 7a71919c534fa291209bd4e476b69970c0a3f27e Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 21:55:41 +0200 Subject: [PATCH 17/29] Test removing all the Bytes objects --- specs/core/0_fork-choice.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 11d6453e9..a5d202ffa 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -61,7 +61,6 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch - root: Bytes32 ``` #### `Store` @@ -69,12 +68,8 @@ class Target(object): ```python @dataclass class Store(object): - blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) - states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = 0x0 - finalized_root: Bytes32 = 0x0 ``` ### Helpers From dc01218182e91c471bf011391b6ca3df77ef1835 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 18 Jun 2019 14:23:31 -0600 Subject: [PATCH 18/29] Add `dataclasses==0.6` to setup.py --- test_libs/pyspec/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 3856640ab..fe14c498c 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -10,6 +10,7 @@ setup( "pycryptodome==3.7.3", "py_ecc>=1.6.0", "typing_inspect==0.4.0", - "ssz==0.1.0a10" + "ssz==0.1.0a10", + "dataclasses==0.6", ] ) From 04c7c438745580660d538ab68747deb7955821b7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 18 Jun 2019 14:36:49 -0600 Subject: [PATCH 19/29] kick deposit contract cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ba9934a6..a43430f77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v5-deposit-contract-bump2 + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v5-deposit-contract-bump2 + venv_name: v6-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 61a51417ed4f58091e0555c82ff18d098df2dd93 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 18 Jun 2019 23:00:30 +0200 Subject: [PATCH 20/29] Unbreak things again --- specs/core/0_fork-choice.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index a5d202ffa..e4ea773c8 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -61,6 +61,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch + root: Bytes32 ``` #### `Store` @@ -68,8 +69,12 @@ class Target(object): ```python @dataclass class Store(object): + blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) + states: Dict[Bytes32, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) + justified_root: Bytes32 = ZERO_HASH + finalized_root: Bytes32 = ZERO_HASH ``` ### Helpers From 241fe34da77d35ee48f9848ce41c684e51055206 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 19 Jun 2019 22:37:51 +0200 Subject: [PATCH 21/29] Workaround for python3.6 --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ed303833f..64d50b579 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -349,6 +349,8 @@ class BytesNMeta(type): return _is_bytes_n_instance_of(self, other.__class__) def __eq__(self, other): + if other == (): + return False return _is_equal_bytes_n_type(self, other) def __ne__(self, other): From d73aa31ef6d893598b942e6c67373da278bafc5c Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 20 Jun 2019 11:58:05 +0100 Subject: [PATCH 22/29] Cleanups --- specs/core/0_fork-choice.md | 47 +++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index fead7d55b..6b7e4bdbe 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -11,13 +11,12 @@ - [Constants](#constants) - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - - [Containers](#containers) + - [Helpers](#helpers) - [`Target`](#target) - [`Store`](#store) - - [Helpers](#helpers) - [`get_genesis_store`](#get_genesis_store) - [`get_ancestor`](#get_ancestor) - - [`get_attesting_balance_from_store`](#get_attesting_balance_from_store) + - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - [`get_head`](#get_head) - [Handlers](#handlers) - [`on_tick`](#on_tick) @@ -51,9 +50,9 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. -4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -### Containers +### Helpers #### `Target` @@ -61,7 +60,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A @dataclass class Target(object): epoch: Epoch - root: Bytes32 + root: Hash ``` #### `Store` @@ -69,38 +68,36 @@ class Target(object): ```python @dataclass class Store(object): - blocks: Dict[Bytes32, BeaconBlock] = field(default_factory=dict) - states: Dict[Bytes32, BeaconState] = field(default_factory=dict) + blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) + states: Dict[Hash, BeaconState] = field(default_factory=dict) time: int = 0 latest_targets: Dict[ValidatorIndex, Target] = field(default_factory=dict) - justified_root: Bytes32 = ZERO_HASH - finalized_root: Bytes32 = ZERO_HASH + justified_root: Hash = ZERO_HASH + finalized_root: Hash = ZERO_HASH ``` -### Helpers - #### `get_genesis_store` ```python def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - return Store(blocks={root: genesis_block}, states={root: genesis_state}, finalized_root=root, justified_root=root) + return Store(blocks={root: genesis_block}, states={root: genesis_state}, justified_root=root, finalized_root=root) ``` #### `get_ancestor` ```python -def get_ancestor(store: Store, root: Bytes32, slot: Slot) -> Bytes32: +def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: block = store.blocks[root] assert block.slot >= slot return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) ``` -#### `get_attesting_balance_from_store` +#### `get_latest_attesting_balance` ```python -def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: +def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.states[store.justified_root] active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) return Gwei(sum( @@ -112,15 +109,15 @@ def get_attesting_balance_from_store(store: Store, root: Bytes32) -> Gwei: #### `get_head` ```python -def get_head(store: Store) -> Bytes32: +def get_head(store: Store) -> Hash: # Execute the LMD-GHOST fork choice head = store.justified_root while True: children = [root for root in store.blocks.keys() if store.blocks[root].parent_root == head] if len(children) == 0: return head - # Sort by attesting balance with ties broken lexicographically - head = max(children, key=lambda root: (get_attesting_balance_from_store(store, root), root)) + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) ``` ### Handlers @@ -147,13 +144,14 @@ def on_block(store: Store, block: BeaconBlock) -> None: state = state_transition(pre_state, block) # Add new state to the store store.states[signing_root(block)] = state - # Update justified and finalized blocks - if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): - store.finalized_root = state.finalized_root + # Update justified block root if state.current_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.current_justified_root elif state.previous_justified_epoch > slot_to_epoch(store.blocks[store.justified_root].slot): store.justified_root = state.previous_justified_root + # Update finalized block root + if state.finalized_epoch > slot_to_epoch(store.blocks[store.finalized_root].slot): + store.finalized_root = state.finalized_root ``` #### `on_attestation` @@ -165,8 +163,5 @@ def on_attestation(store: Store, attestation: Attestation) -> None: validate_indexed_attestation(state, indexed_attestation) for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: if i not in store.latest_targets or attestation.data.target_epoch > store.latest_targets[i].epoch: - store.latest_targets[i] = Target( - epoch=attestation.data.target_epoch, - root=attestation.data.target_root, - ) + store.latest_targets[i] = Target(attestation.data.target_epoch, attestation.data.target_root) ``` From 5f8edd6b55d13a7eb91a8ea7950896bab7a165fa Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 20 Jun 2019 20:50:17 +0200 Subject: [PATCH 23/29] Genesis block store uses genesis time Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_fork-choice.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6b7e4bdbe..5a4a52db4 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -82,7 +82,12 @@ class Store(object): def get_genesis_store(genesis_state: BeaconState) -> Store: genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state)) root = signing_root(genesis_block) - return Store(blocks={root: genesis_block}, states={root: genesis_state}, justified_root=root, finalized_root=root) + return Store( + blocks={root: genesis_block}, + states={root: genesis_state}, + time=genesis_state.genesis_time, + justified_root=root, finalized_root=root, + ) ``` #### `get_ancestor` From c26fffc15481ca282c75307a122f2012f4090cd1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 20 Jun 2019 20:55:28 +0200 Subject: [PATCH 24/29] Moves copy into SSZ container --- scripts/build_spec.py | 4 ---- specs/core/0_fork-choice.md | 5 +++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 9232cf00b..a16fa79ac 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,8 +25,6 @@ from dataclasses import ( field, ) -from copy import deepcopy - from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -60,8 +58,6 @@ from dataclasses import ( field, ) -from copy import deepcopy - from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 5a4a52db4..ba97398a2 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -86,7 +86,8 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: blocks={root: genesis_block}, states={root: genesis_state}, time=genesis_state.genesis_time, - justified_root=root, finalized_root=root, + justified_root=root, + finalized_root=root, ) ``` @@ -143,7 +144,7 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time - pre_state = deepcopy(store.states[block.parent_root]) + pre_state = store.states[block.parent_root].copy() assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 64d50b579..077f94b49 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,3 +1,4 @@ +import copy from types import GeneratorType from typing import ( List, @@ -151,6 +152,9 @@ class Container(object): def __hash__(self): return hash(self.hash_tree_root()) + def copy(self): + return copy.deepcopy(self) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) From e88a96c45e2cd724f7a3949e78d4c4c5707d1d40 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 12:13:22 +0200 Subject: [PATCH 25/29] Apply suggestions from @drjtwo's code review Co-Authored-By: Danny Ryan --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index ba97398a2..6ae7e11fa 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -47,10 +47,11 @@ The head block root associated with a `store` is defined as `get_head(store)`. A *Notes*: -1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). ### Helpers From f90469ea25c3b28ec20881d1a4e17033a70a129a Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 12:19:08 +0200 Subject: [PATCH 26/29] Move block timing assertion 1st --- specs/core/0_fork-choice.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 6ae7e11fa..eb7c1a579 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -140,13 +140,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root # Check block slot against Unix time pre_state = store.states[block.parent_root].copy() - assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state to the store From 0e59c6676a66bf1f0b40bf0eb76c3040da01656f Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 21 Jun 2019 13:00:42 +0200 Subject: [PATCH 27/29] Stop yielding from fork-choie tests --- specs/core/0_fork-choice.md | 4 ++-- .../pyspec/eth2spec/test/test_fork_choice.py | 16 +++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index eb7c1a579..3d85cf571 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -140,14 +140,14 @@ def on_tick(store: Store, time: int) -> None: ```python def on_block(store: Store, block: BeaconBlock) -> None: + # Make a copy of the state to avoid mutability issues + pre_state = store.states[block.parent_root].copy() # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT # Add new block to the store store.blocks[signing_root(block)] = block # Check block is a descendant of the finalized block assert get_ancestor(store, signing_root(block), store.blocks[store.finalized_root].slot) == store.finalized_root - # Check block slot against Unix time - pre_state = store.states[block.parent_root].copy() # Check the block is valid and compute the post-state state = state_transition(pre_state, block) # Add new state to the store diff --git a/test_libs/pyspec/eth2spec/test/test_fork_choice.py b/test_libs/pyspec/eth2spec/test/test_fork_choice.py index ab1728251..4bc7e8b0a 100644 --- a/test_libs/pyspec/eth2spec/test/test_fork_choice.py +++ b/test_libs/pyspec/eth2spec/test/test_fork_choice.py @@ -2,7 +2,7 @@ from typing import List from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root -from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.context import with_all_phases, with_state, bls_switch from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation @@ -10,10 +10,10 @@ from eth2spec.test.helpers.state import next_slot @with_all_phases -@spec_state_test +@with_state +@bls_switch def test_basic(spec, state): state.latest_block_header = spec.BeaconBlockHeader(body_root=hash_tree_root(spec.BeaconBlockBody())) - yield 'pre', state # Initialization store = spec.get_genesis_store(state) @@ -36,17 +36,14 @@ def test_basic(spec, state): spec.on_block(store, block) assert store.blocks[signing_root(block)] == block - yield 'blocks', blocks, List[spec.BeaconBlock] # TODO: add tests for justified_root and finalized_root - yield 'post', state @with_all_phases -@spec_state_test +@with_state +@bls_switch def test_on_attestation(spec, state): - yield 'pre', state - store = spec.get_genesis_store(state) time = 100 spec.on_tick(store, time) @@ -54,7 +51,6 @@ def test_on_attestation(spec, state): next_slot(spec, state) attestation = get_valid_attestation(spec, state, slot=1) - yield 'attestation', attestation indexed_attestation = spec.convert_to_indexed(state, attestation) spec.on_attestation(store, attestation) assert ( @@ -64,5 +60,3 @@ def test_on_attestation(spec, state): root=attestation.data.target_root, ) ) - - yield 'post', state From b46e047baa4fc06a0980904c25ef2137f51a06a2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 21 Jun 2019 12:57:30 +0100 Subject: [PATCH 28/29] Minor simplification from #1198 --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 3d85cf571..892c592e9 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -106,7 +106,7 @@ def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: ```python def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei: state = store.states[store.justified_root] - active_indices = get_active_validator_indices(state.validator_registry, slot_to_epoch(state.slot)) + active_indices = get_active_validator_indices(state.validator_registry, get_current_epoch(state)) return Gwei(sum( state.validator_registry[i].effective_balance for i in active_indices if get_ancestor(store, store.latest_targets[i].root, store.blocks[root].slot) == root From acbccbc2a8eeed367167e22732c91987cdd06637 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 11:18:24 -0600 Subject: [PATCH 29/29] minor typo --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 892c592e9..25b33ab30 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -51,7 +51,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. 3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). ### Helpers