Merge branch 'dev' into v_guide_checkpoint

Makefile

@@ -29,16 +29,21 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
 all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
-clean:
+# deletes everything except the venvs
+partial_clean:
 	rm -rf $(YAML_TEST_DIR)
 	rm -rf $(GENERATOR_VENVS)
-	rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
 	rm -rf $(PY_SPEC_DIR)/.coverage
 	rm -rf $(PY_SPEC_DIR)/test-reports
 
+clean: partial_clean
+	rm -rf $(PY_SPEC_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
+
 # "make gen_yaml_tests" to run generators
 gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
 
@@ -78,7 +83,7 @@ test_deposit_contract:
 pyspec: $(PY_SPEC_ALL_TARGETS)
 
 $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@
 
 $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
 	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@

scripts/build_spec.py

@@ -11,7 +11,7 @@ from typing import (
 
 
 PHASE0_IMPORTS = '''from typing import (
-    Any, Dict, Set, Sequence, Tuple,
+    Any, Dict, Set, Sequence, Tuple, Optional
 )
 
 from dataclasses import (
@@ -31,6 +31,7 @@ from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
     bls_verify,
     bls_verify_multiple,
+    bls_sign,
 )
 
 from eth2spec.utils.hash_function import hash
@@ -68,6 +69,10 @@ _hash = hash
 hash_cache: Dict[bytes, Hash] = {}
 
 
+def get_eth1_data(distance: uint64) -> Hash:
+    return hash(distance)
+
+
 def hash(x: bytes) -> Hash:
     if x not in hash_cache:
         hash_cache[x] = Hash(_hash(x))
@@ -243,10 +248,14 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     return functions, custom_types, constants, ssz_objects, inserts
 
 
-def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]:
+def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
+                      v_guide_sourcefile: str, outfile: str=None) -> Optional[str]:
     phase0_spec = get_spec(phase0_sourcefile)
     fork_choice_spec = get_spec(fork_choice_sourcefile)
-    spec_objects = combine_spec_objects(phase0_spec, fork_choice_spec)
+    v_guide = get_spec(v_guide_sourcefile)
+    spec_objects = phase0_spec
+    for value in [fork_choice_spec, v_guide]:
+        spec_objects = combine_spec_objects(spec_objects, value)
     spec = objects_to_spec(*spec_objects, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
@@ -279,7 +288,8 @@ Build the specs from the md docs.
 If building phase 0:
     1st argument is input spec.md
     2nd argument is input fork_choice.md
-    3rd argument is output spec.py
+    3rd argument is input validator_guide.md
+    4th argument is output spec.py
 
 If building phase 1:
     1st argument is input spec_phase0.md
@@ -294,14 +304,15 @@ If building phase 1:
 
     args = parser.parse_args()
     if args.phase == 0:
-        if len(args.files) == 3:
+        if len(args.files) == 4:
             build_phase0_spec(*args.files)
         else:
-            print(" Phase 0 requires an output as well as spec and forkchoice files.")
+            print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.")
     elif args.phase == 1:
         if len(args.files) == 5:
             build_phase1_spec(*args.files)
         else:
-            print(" Phase 1 requires an output as well as 4 input files (phase0.md and phase1.md, phase1.md, fork_choice.md)")
+            print(" Phase 1 requires 4 input files as well as an output file: "
+                  + "(phase0.md and phase1.md, phase1.md, fork_choice.md, output.py)")
     else:
         print("Invalid phase: {0}".format(args.phase))

specs/core/0_fork-choice.md

@@ -171,8 +171,6 @@ def on_block(store: Store, block: BeaconBlock) -> None:
     # Update justified checkpoint
     if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
         store.justified_checkpoint = state.current_justified_checkpoint
-    elif state.previous_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
-        store.justified_checkpoint = state.previous_justified_checkpoint
 
     # Update finalized checkpoint
     if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:

specs/validator/0_beacon-chain-validator.md

@@ -119,8 +119,9 @@ In normal operation, the validator is quickly activated, at which point the vali
 The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows:
 
 ```python
-validator = state.validators[validator_index]
+def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool:
-is_active = is_active_validator(validator, get_current_epoch(state))
+    validator = state.validators[validator_index]
+    return is_active_validator(validator, get_current_epoch(state))
 ```
 
 Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited.
@@ -134,13 +135,14 @@ A validator can get committee assignments for a given epoch using the following
 ```python
 def get_committee_assignment(state: BeaconState,
                              epoch: Epoch,
-                             validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]:
+                             validator_index: ValidatorIndex) -> Optional[Tuple[Sequence[ValidatorIndex], Shard, Slot]]:
     """
     Return the committee assignment in the ``epoch`` for ``validator_index``.
     ``assignment`` returned is a tuple of the following form:
         * ``assignment[0]`` is the list of validators in the committee
         * ``assignment[1]`` is the shard to which the committee is assigned
         * ``assignment[2]`` is the slot at which the committee is assigned
+    Return None if no assignment.
     """
     next_epoch = get_current_epoch(state) + 1
     assert epoch <= next_epoch
@@ -151,10 +153,11 @@ def get_committee_assignment(state: BeaconState,
         offset = committees_per_slot * (slot % SLOTS_PER_EPOCH)
         slot_start_shard = (get_start_shard(state, epoch) + offset) % SHARD_COUNT
         for i in range(committees_per_slot):
-            shard = (slot_start_shard + i) % SHARD_COUNT
+            shard = Shard((slot_start_shard + i) % SHARD_COUNT)
             committee = get_crosslink_committee(state, epoch, shard)
             if validator_index in committee:
-                return committee, shard, slot
+                return committee, shard, Slot(slot)
+    return None
 ```
 
 A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch.
@@ -205,18 +208,12 @@ Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the `
 
 ##### Randao reveal
 
-Set `block.randao_reveal = epoch_signature` where `epoch_signature` is defined as:
+Set `block.randao_reveal = epoch_signature` where `epoch_signature` is obtained from:
 
 ```python
-epoch_signature = bls_sign(
+def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature:
-    privkey=validator.privkey,  # privkey stored locally, not in state
+    domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_of_slot(block.slot))
-    message_hash=hash_tree_root(compute_epoch_of_slot(block.slot)),
+    return bls_sign(privkey, hash_tree_root(compute_epoch_of_slot(block.slot)), domain)
-    domain=get_domain(
-        fork=fork,  # `fork` is the fork object at the epoch `compute_epoch_of_slot(block.slot)`
-        epoch=compute_epoch_of_slot(block.slot),
-        domain_type=DOMAIN_RANDAO,
-    )
-)
 ```
 
 ##### Eth1 Data
@@ -232,11 +229,12 @@ def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Dat
 
     valid_votes = []
     for slot, vote in enumerate(state.eth1_data_votes):
-        period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_square_root(SLOTS_PER_ETH1_VOTING_PERIOD)
+        period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_squareroot(SLOTS_PER_ETH1_VOTING_PERIOD)
         if vote in new_eth1_data or (period_tail and vote in all_eth1_data):
             valid_votes.append(vote)
 
-    return max(valid_votes,
+    return max(
+        valid_votes,
         key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)),  # Tiebreak by smallest distance
         default=get_eth1_data(ETH1_FOLLOW_DISTANCE),
     )
@@ -244,18 +242,12 @@ def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Dat
 
 ##### Signature
 
-Set `block.signature = block_signature` where `block_signature` is defined as:
+Set `header.signature = block_signature` where `block_signature` is obtained from:
 
 ```python
-block_signature = bls_sign(
+def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature:
-    privkey=validator.privkey,  # privkey store locally, not in state
+    domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_of_slot(header.slot))
-    message_hash=signing_root(block),
+    return bls_sign(privkey, signing_root(header), domain)
-    domain=get_domain(
-        fork=fork,  # `fork` is the fork object at the epoch `compute_epoch_of_slot(block.slot)`
-        epoch=compute_epoch_of_slot(block.slot),
-        domain_type=DOMAIN_BEACON_BLOCK,
-    )
-)
 ```
 
 #### Block body
@@ -342,24 +334,17 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes
 
 ##### Aggregate signature
 
-Set `attestation.aggregate_signature = signed_attestation_data` where `signed_attestation_data` is defined as:
+Set `attestation.signature = signed_attestation_data` where `signed_attestation_data` is obtained from:
 
 ```python
-attestation_data_and_custody_bit = AttestationDataAndCustodyBit(
+def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature:
-    data=attestation.data,
+    attestation_data_and_custody_bit = AttestationDataAndCustodyBit(
-    custody_bit=0b0,
+        data=attestation.data,
-)
+        custody_bit=0b0,
-attestation_message = hash_tree_root(attestation_data_and_custody_bit)
-
-signed_attestation_data = bls_sign(
-    privkey=validator.privkey,  # privkey stored locally, not in state
-    message_hash=attestation_message,
-    domain=get_domain(
-        fork=fork,  # `fork` is the fork object at epoch `target.epoch`
-        epoch=target.epoch,
-        domain_type=DOMAIN_ATTESTATION,
     )
-)
+
+    domain = get_domain(state, DOMAIN_ATTESTATION, attestation.data.target.epoch)
+    return bls_sign(privkey, hash_tree_root(attestation_data_and_custody_bit), domain)
 ```
 
 ## How to avoid slashing

test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py

@@ -2,6 +2,7 @@ from eth2spec.utils.ssz.ssz_impl import signing_root
 
 from eth2spec.test.context import with_all_phases, with_state, bls_switch
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations
 
 
 def run_on_block(spec, state, store, block, valid=True):
@@ -17,6 +18,17 @@ def run_on_block(spec, state, store, block, valid=True):
     assert store.blocks[signing_root(block)] == block
 
 
+def apply_next_epoch_with_attestations(spec, state, store):
+    _, new_blocks, state = next_epoch_with_attestations(spec, state, True, False)
+    for block in new_blocks:
+        block_root = signing_root(block)
+        store.blocks[block_root] = block
+        store.block_states[block_root] = state
+        last_block = block
+    spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT)
+    return state, store, last_block
+
+
 @with_all_phases
 @with_state
 @bls_switch
@@ -41,6 +53,32 @@ def test_basic(spec, state):
     # TODO: add tests for justified_root and finalized_root
 
 
+@with_all_phases
+@with_state
+@bls_switch
+def test_on_block_checkpoints(spec, state):
+    # Initialization
+    store = spec.get_genesis_store(state)
+    time = 100
+    spec.on_tick(store, time)
+
+    next_epoch(spec, state)
+    spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT)
+    state, store, last_block = apply_next_epoch_with_attestations(spec, state, store)
+    next_epoch(spec, state)
+    spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT)
+    last_block_root = signing_root(last_block)
+
+    # Mock the finalized_checkpoint
+    store.block_states[last_block_root].finalized_checkpoint = (
+        store.block_states[last_block_root].current_justified_checkpoint
+    )
+
+    # On receiving a block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    run_on_block(spec, state, store, block)
+
+
 @with_all_phases
 @with_state
 @bls_switch

test_libs/pyspec/eth2spec/test/helpers/state.py

@@ -1,4 +1,6 @@
-from eth2spec.test.helpers.block import sign_block
+from copy import deepcopy
+from eth2spec.test.helpers.attestations import get_valid_attestation
+from eth2spec.test.helpers.block import sign_block, build_empty_block_for_next_slot
 
 
 def get_balance(state, index):
@@ -36,3 +38,30 @@ def state_transition_and_sign_block(spec, state, block):
     spec.state_transition(state, block)
     block.state_root = state.hash_tree_root()
     sign_block(spec, state, block)
+
+
+def next_epoch_with_attestations(spec,
+                                 state,
+                                 fill_cur_epoch,
+                                 fill_prev_epoch):
+    assert state.slot % spec.SLOTS_PER_EPOCH == 0
+
+    post_state = deepcopy(state)
+    blocks = []
+    for _ in range(spec.SLOTS_PER_EPOCH):
+        block = build_empty_block_for_next_slot(spec, post_state)
+        if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
+            slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
+            if slot_to_attest >= spec.compute_start_slot_of_epoch(spec.get_current_epoch(post_state)):
+                cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest)
+                block.body.attestations.append(cur_attestation)
+
+        if fill_prev_epoch:
+            slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1
+            prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest)
+            block.body.attestations.append(prev_attestation)
+
+        state_transition_and_sign_block(spec, post_state, block)
+        blocks.append(block)
+
+    return state, blocks, post_state

test_libs/pyspec/eth2spec/test/test_finality.py

@@ -1,9 +1,6 @@
-from copy import deepcopy
-
 from eth2spec.test.context import spec_state_test, never_bls, with_all_phases
-from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block
+from eth2spec.test.helpers.state import next_epoch, next_epoch_with_attestations
-from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block
+from eth2spec.test.helpers.block import apply_empty_block
-from eth2spec.test.helpers.attestations import get_valid_attestation
 
 
 def check_finality(spec,
@@ -31,33 +28,6 @@ def check_finality(spec,
         assert state.finalized_checkpoint == prev_state.finalized_checkpoint
 
 
-def next_epoch_with_attestations(spec,
-                                 state,
-                                 fill_cur_epoch,
-                                 fill_prev_epoch):
-    assert state.slot % spec.SLOTS_PER_EPOCH == 0
-
-    post_state = deepcopy(state)
-    blocks = []
-    for _ in range(spec.SLOTS_PER_EPOCH):
-        block = build_empty_block_for_next_slot(spec, post_state)
-        if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
-            slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
-            if slot_to_attest >= spec.compute_start_slot_of_epoch(spec.get_current_epoch(post_state)):
-                cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest)
-                block.body.attestations.append(cur_attestation)
-
-        if fill_prev_epoch:
-            slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1
-            prev_attestation = get_valid_attestation(spec, post_state, slot_to_attest)
-            block.body.attestations.append(prev_attestation)
-
-        state_transition_and_sign_block(spec, post_state, block)
-        blocks.append(block)
-
-    return state, blocks, post_state
-
-
 @with_all_phases
 @never_bls
 @spec_state_test