backport 839 into dev

.gitignore

@@ -4,3 +4,4 @@
 /.pytest_cache
 
 build/
+output/

specs/core/0_beacon-chain.md

@@ -692,7 +692,8 @@ def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader:
         previous_block_root=block.previous_block_root,
         state_root=ZERO_HASH,
         block_body_root=hash_tree_root(block.body),
-        signature=block.signature,
+        # signed_root(block) is used for block id purposes so signature is a stub
+        signature=EMPTY_SIGNATURE,
     )
 ```
 
@@ -1749,7 +1750,7 @@ def cache_state(state: BeaconState) -> None:
         state.latest_block_header.state_root = previous_slot_state_root
 
     # store latest known block for previous slot
-    state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = hash_tree_root(state.latest_block_header)
+    state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signed_root(state.latest_block_header)
 ```
 
 ### Per-epoch processing
@@ -2198,8 +2199,8 @@ def update_registry_and_shuffling_data(state: BeaconState) -> None:
         state.current_shuffling_epoch = next_epoch
         state.current_shuffling_start_shard = (
             state.current_shuffling_start_shard +
-            get_current_epoch_committee_count(state) % SHARD_COUNT
+            get_current_epoch_committee_count(state)
-        )
+        ) % SHARD_COUNT
         state.current_shuffling_seed = generate_seed(state, state.current_shuffling_epoch)
     else:
         # If processing at least one crosslink keeps failing, then reshuffle every power of two,
@@ -2315,7 +2316,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None:
     # Verify that the slots match
     assert block.slot == state.slot
     # Verify that the parent matches
-    assert block.previous_block_root == hash_tree_root(state.latest_block_header)
+    assert block.previous_block_root == signed_root(state.latest_block_header)
     # Save current block as the new latest block
     state.latest_block_header = get_temporary_block_header(block)
     # Verify proposer is not slashed

tests/phase0/block_processing/test_process_deposit.py

@@ -4,12 +4,14 @@ import pytest
 import build.phase0.spec as spec
 
 from build.phase0.spec import (
-    Deposit,
     get_balance,
+    ZERO_HASH,
     process_deposit,
 )
 from tests.phase0.helpers import (
     build_deposit,
+    privkeys,
+    pubkeys,
 )
 
 
@@ -17,8 +19,10 @@ from tests.phase0.helpers import (
 pytestmark = pytest.mark.voluntary_exits
 
 
-def test_success(state, deposit_data_leaves, pubkeys, privkeys):
+def test_success(state):
     pre_state = deepcopy(state)
+    # fill previous deposits with zero-hash
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     index = len(deposit_data_leaves)
     pubkey = pubkeys[index]
@@ -47,8 +51,9 @@ def test_success(state, deposit_data_leaves, pubkeys, privkeys):
     return pre_state, deposit, post_state
 
 
-def test_success_top_up(state, deposit_data_leaves, pubkeys, privkeys):
+def test_success_top_up(state):
     pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     validator_index = 0
     amount = spec.MAX_DEPOSIT_AMOUNT // 4
@@ -78,8 +83,9 @@ def test_success_top_up(state, deposit_data_leaves, pubkeys, privkeys):
     return pre_state, deposit, post_state
 
 
-def test_wrong_index(state, deposit_data_leaves, pubkeys, privkeys):
+def test_wrong_index(state):
     pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     index = len(deposit_data_leaves)
     pubkey = pubkeys[index]
@@ -106,8 +112,9 @@ def test_wrong_index(state, deposit_data_leaves, pubkeys, privkeys):
     return pre_state, deposit, None
 
 
-def test_bad_merkle_proof(state, deposit_data_leaves, pubkeys, privkeys):
+def test_bad_merkle_proof(state):
     pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     index = len(deposit_data_leaves)
     pubkey = pubkeys[index]

tests/phase0/block_processing/test_voluntary_exit.py

@@ -10,6 +10,7 @@ from build.phase0.spec import (
 )
 from tests.phase0.helpers import (
     build_voluntary_exit,
+    pubkey_to_privkey,
 )
 
 
@@ -17,7 +18,7 @@ from tests.phase0.helpers import (
 pytestmark = pytest.mark.voluntary_exits
 
 
-def test_success(state, pub_to_priv):
+def test_success(state):
     pre_state = deepcopy(state)
     #
     # setup pre_state
@@ -30,7 +31,7 @@ def test_success(state, pub_to_priv):
     #
     current_epoch = get_current_epoch(pre_state)
     validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
-    privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
 
     voluntary_exit = build_voluntary_exit(
         pre_state,
@@ -52,11 +53,11 @@ def test_success(state, pub_to_priv):
     return pre_state, voluntary_exit, post_state
 
 
-def test_validator_not_active(state, pub_to_priv):
+def test_validator_not_active(state):
     pre_state = deepcopy(state)
     current_epoch = get_current_epoch(pre_state)
     validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
-    privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
 
     #
     # setup pre_state
@@ -79,7 +80,7 @@ def test_validator_not_active(state, pub_to_priv):
     return pre_state, voluntary_exit, None
 
 
-def test_validator_already_exited(state, pub_to_priv):
+def test_validator_already_exited(state):
     pre_state = deepcopy(state)
     #
     # setup pre_state
@@ -89,7 +90,7 @@ def test_validator_already_exited(state, pub_to_priv):
 
     current_epoch = get_current_epoch(pre_state)
     validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
-    privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
 
     # but validator already has exited
     pre_state.validator_registry[validator_index].exit_epoch = current_epoch + 2
@@ -110,7 +111,7 @@ def test_validator_already_exited(state, pub_to_priv):
     return pre_state, voluntary_exit, None
 
 
-def test_validator_already_initiated_exit(state, pub_to_priv):
+def test_validator_already_initiated_exit(state):
     pre_state = deepcopy(state)
     #
     # setup pre_state
@@ -120,7 +121,7 @@ def test_validator_already_initiated_exit(state, pub_to_priv):
 
     current_epoch = get_current_epoch(pre_state)
     validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
-    privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
 
     # but validator already has initiated exit
     pre_state.validator_registry[validator_index].initiated_exit = True
@@ -141,14 +142,14 @@ def test_validator_already_initiated_exit(state, pub_to_priv):
     return pre_state, voluntary_exit, None
 
 
-def test_validator_not_active_long_enough(state, pub_to_priv):
+def test_validator_not_active_long_enough(state):
     pre_state = deepcopy(state)
     #
     # setup pre_state
     #
     current_epoch = get_current_epoch(pre_state)
     validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
-    privkey = pub_to_priv[pre_state.validator_registry[validator_index].pubkey]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
 
     # but validator already has initiated exit
     pre_state.validator_registry[validator_index].initiated_exit = True

tests/phase0/conftest.py

@@ -3,9 +3,6 @@ import pytest
 from build.phase0 import spec
 
 from tests.phase0.helpers import (
-    privkeys_list,
-    pubkeys_list,
-    pubkey_to_privkey,
     create_genesis_state,
 )
 
@@ -25,21 +22,6 @@ MINIMAL_CONFIG = {
 }
 
 
-@pytest.fixture
-def privkeys():
-    return privkeys_list
-
-
-@pytest.fixture
-def pubkeys():
-    return pubkeys_list
-
-
-@pytest.fixture
-def pub_to_priv():
-    return pubkey_to_privkey
-
-
 def overwrite_spec_config(config):
     for field in config:
         setattr(spec, field, config[field])

tests/phase0/helpers.py

@@ -35,18 +35,20 @@ from build.phase0.utils.merkle_minimal import (
 )
 
 
-privkeys_list = [i + 1 for i in range(1000)]
+privkeys = [i + 1 for i in range(1000)]
-pubkeys_list = [bls.privtopub(privkey) for privkey in privkeys_list]
+pubkeys = [bls.privtopub(privkey) for privkey in privkeys]
-pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys_list, pubkeys_list)}
+pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)}
 
 
-def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves):
+def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None):
+    if not deposit_data_leaves:
+        deposit_data_leaves = []
     deposit_timestamp = 0
     proof_of_possession = b'\x33' * 96
 
     deposit_data_list = []
     for i in range(num_validators):
-        pubkey = pubkeys_list[i]
+        pubkey = pubkeys[i]
         deposit_data = DepositData(
             amount=spec.MAX_DEPOSIT_AMOUNT,
             timestamp=deposit_timestamp,
@@ -75,7 +77,7 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves):
     return genesis_validator_deposits, root
 
 
-def create_genesis_state(num_validators, deposit_data_leaves):
+def create_genesis_state(num_validators, deposit_data_leaves=None):
     initial_deposits, deposit_root = create_mock_genesis_validator_deposits(
         num_validators,
         deposit_data_leaves,
@@ -105,7 +107,7 @@ def build_empty_block_for_next_slot(state):
     previous_block_header = deepcopy(state.latest_block_header)
     if previous_block_header.state_root == spec.ZERO_HASH:
         previous_block_header.state_root = state.hash_tree_root()
-    empty_block.previous_block_root = previous_block_header.hash_tree_root()
+    empty_block.previous_block_root = signed_root(previous_block_header)
     return empty_block
 
 

tests/phase0/test_sanity.py

@@ -47,6 +47,8 @@ from tests.phase0.helpers import (
     build_empty_block_for_next_slot,
     force_registry_change_at_next_epoch,
     get_valid_proposer_slashing,
+    privkeys,
+    pubkeys,
 )
 
 
@@ -116,7 +118,7 @@ def test_empty_epoch_transition_not_finalizing(state):
     return state, [block], test_state
 
 
-def test_proposer_slashing(state, pubkeys, privkeys):
+def test_proposer_slashing(state):
     test_state = deepcopy(state)
     proposer_slashing = get_valid_proposer_slashing(state)
     validator_index = proposer_slashing.proposer_index
@@ -142,9 +144,9 @@ def test_proposer_slashing(state, pubkeys, privkeys):
     return state, [block], test_state
 
 
-def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys):
+def test_deposit_in_block(state):
     pre_state = deepcopy(state)
-    test_deposit_data_leaves = deepcopy(deposit_data_leaves)
+    test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     index = len(test_deposit_data_leaves)
     pubkey = pubkeys[index]
@@ -179,9 +181,9 @@ def test_deposit_in_block(state, deposit_data_leaves, pubkeys, privkeys):
     return pre_state, [block], post_state
 
 
-def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves):
+def test_deposit_top_up(state):
     pre_state = deepcopy(state)
-    test_deposit_data_leaves = deepcopy(deposit_data_leaves)
+    test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
 
     validator_index = 0
     amount = spec.MAX_DEPOSIT_AMOUNT // 4
@@ -218,7 +220,7 @@ def test_deposit_top_up(state, pubkeys, privkeys, deposit_data_leaves):
     return pre_state, [block], post_state
 
 
-def test_attestation(state, pubkeys, privkeys):
+def test_attestation(state):
     test_state = deepcopy(state)
     slot = state.slot
     shard = state.current_shuffling_start_shard
@@ -287,7 +289,7 @@ def test_attestation(state, pubkeys, privkeys):
     return state, [attestation_block, epoch_block], test_state
 
 
-def test_voluntary_exit(state, pubkeys, privkeys):
+def test_voluntary_exit(state):
     pre_state = deepcopy(state)
     validator_index = get_active_validator_indices(
         pre_state.validator_registry,
@@ -375,7 +377,7 @@ def test_no_exit_churn_too_long_since_change(state):
     return pre_state, [block], post_state
 
 
-def test_transfer(state, pubkeys, privkeys):
+def test_transfer(state):
     pre_state = deepcopy(state)
     current_epoch = get_current_epoch(pre_state)
     sender_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[-1]

utils/phase0/jsonize.py

@@ -0,0 +1,52 @@
+from .minimal_ssz import hash_tree_root
+
+
+def jsonize(value, typ, include_hash_tree_roots=False):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return value
+    elif typ == 'bool':
+        assert value in (True, False)
+        return value
+    elif isinstance(typ, list):
+        return [jsonize(element, typ[0], include_hash_tree_roots) for element in value]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return '0x' + value.hex()
+    elif hasattr(typ, 'fields'):
+        ret = {}
+        for field, subtype in typ.fields.items():
+            ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots)
+            if include_hash_tree_roots:
+                ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex()
+        if include_hash_tree_roots:
+            ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex()
+        return ret
+    else:
+        print(value, typ)
+        raise Exception("Type not recognized")
+
+
+def dejsonize(json, typ):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return json
+    elif typ == 'bool':
+        assert json in (True, False)
+        return json
+    elif isinstance(typ, list):
+        return [dejsonize(element, typ[0]) for element in json]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return bytes.fromhex(json[2:])
+    elif hasattr(typ, 'fields'):
+        temp = {}
+        for field, subtype in typ.fields.items():
+            temp[field] = dejsonize(json[field], subtype)
+            if field + "_hash_tree_root" in json:
+                assert(json[field + "_hash_tree_root"][2:] == 
+                       hash_tree_root(temp[field], subtype).hex())
+        ret = typ(**temp)
+        if "hash_tree_root" in json:
+            assert(json["hash_tree_root"][2:] == 
+                   hash_tree_root(ret, typ).hex())
+        return ret
+    else:
+        print(json, typ)
+        raise Exception("Type not recognized")