Merge branch 'master' into dev-master-conflicts

test_libs/pyspec/tests/test_sanity.py

@@ -380,7 +380,7 @@ def test_voluntary_exit(state):
     return pre_state, [initiate_exit_block, exit_block], post_state
 
 
-def test_transfer(state, config):
+def test_transfer(state):
     # overwrite default 0 to test
     spec.MAX_TRANSFERS = 1
 

tests/phase0/block_processing/test_process_block_header.py

@@ -0,0 +1,60 @@
+from copy import deepcopy
+import pytest
+
+
+from build.phase0.spec import (
+    get_beacon_proposer_index,
+    cache_state,
+    advance_slot,
+    process_block_header,
+)
+from tests.phase0.helpers import (
+    build_empty_block_for_next_slot,
+)
+
+# mark entire file as 'header'
+pytestmark = pytest.mark.header
+
+
+def prepare_state_for_header_processing(state):
+    cache_state(state)
+    advance_slot(state)
+
+
+def run_block_header_processing(state, block, valid=True):
+    """
+    Run ``process_block_header`` returning the pre and post state.
+    If ``valid == False``, run expecting ``AssertionError``
+    """
+    prepare_state_for_header_processing(state)
+    post_state = deepcopy(state)
+
+    if not valid:
+        with pytest.raises(AssertionError):
+            process_block_header(post_state, block)
+        return state, None
+
+    process_block_header(post_state, block)
+    return state, post_state
+
+
+def test_success(state):
+    block = build_empty_block_for_next_slot(state)
+    pre_state, post_state = run_block_header_processing(state, block)
+    return state, block, post_state
+
+
+def test_invalid_slot(state):
+    block = build_empty_block_for_next_slot(state)
+    block.slot = state.slot + 2  # invalid slot
+
+    pre_state, post_state = run_block_header_processing(state, block, valid=False)
+    return pre_state, block, None
+
+
+def test_invalid_previous_block_root(state):
+    block = build_empty_block_for_next_slot(state)
+    block.previous_block_root = b'\12'*32  # invalid prev root
+
+    pre_state, post_state = run_block_header_processing(state, block, valid=False)
+    return pre_state, block, None

tests/phase0/block_processing/test_process_deposit.py

@@ -0,0 +1,140 @@
+from copy import deepcopy
+import pytest
+
+import build.phase0.spec as spec
+
+from build.phase0.spec import (
+    ZERO_HASH,
+    process_deposit,
+)
+from tests.phase0.helpers import (
+    build_deposit,
+    privkeys,
+    pubkeys,
+)
+
+
+# mark entire file as 'voluntary_exits'
+pytestmark = pytest.mark.voluntary_exits
+
+
+def test_success(state):
+    pre_state = deepcopy(state)
+    # fill previous deposits with zero-hash
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
+
+    index = len(deposit_data_leaves)
+    pubkey = pubkeys[index]
+    privkey = privkeys[index]
+    deposit, root, deposit_data_leaves = build_deposit(
+        pre_state,
+        deposit_data_leaves,
+        pubkey,
+        privkey,
+        spec.MAX_DEPOSIT_AMOUNT,
+    )
+
+    pre_state.latest_eth1_data.deposit_root = root
+    pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
+
+    post_state = deepcopy(pre_state)
+
+    process_deposit(post_state, deposit)
+
+    assert len(post_state.validator_registry) == len(state.validator_registry) + 1
+    assert len(post_state.validator_balances) == len(state.validator_balances) + 1
+    assert post_state.validator_registry[index].pubkey == pubkeys[index]
+    assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count
+
+    return pre_state, deposit, post_state
+
+
+def test_success_top_up(state):
+    pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
+
+    validator_index = 0
+    amount = spec.MAX_DEPOSIT_AMOUNT // 4
+    pubkey = pubkeys[validator_index]
+    privkey = privkeys[validator_index]
+    deposit, root, deposit_data_leaves = build_deposit(
+        pre_state,
+        deposit_data_leaves,
+        pubkey,
+        privkey,
+        amount,
+    )
+
+    pre_state.latest_eth1_data.deposit_root = root
+    pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
+    pre_balance = pre_state.validator_balances[validator_index]
+
+    post_state = deepcopy(pre_state)
+
+    process_deposit(post_state, deposit)
+
+    assert len(post_state.validator_registry) == len(state.validator_registry)
+    assert len(post_state.validator_balances) == len(state.validator_balances)
+    assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count
+    assert post_state.validator_balances[validator_index] == pre_balance + amount
+
+    return pre_state, deposit, post_state
+
+
+def test_wrong_index(state):
+    pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
+
+
+    index = len(deposit_data_leaves)
+    pubkey = pubkeys[index]
+    privkey = privkeys[index]
+    deposit, root, deposit_data_leaves = build_deposit(
+        pre_state,
+        deposit_data_leaves,
+        pubkey,
+        privkey,
+        spec.MAX_DEPOSIT_AMOUNT,
+    )
+
+    # mess up deposit_index
+    deposit.index = pre_state.deposit_index + 1
+
+    pre_state.latest_eth1_data.deposit_root = root
+    pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
+
+    post_state = deepcopy(pre_state)
+
+    with pytest.raises(AssertionError):
+        process_deposit(post_state, deposit)
+
+    return pre_state, deposit, None
+
+
+def test_bad_merkle_proof(state):
+    pre_state = deepcopy(state)
+    deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry)
+
+    index = len(deposit_data_leaves)
+    pubkey = pubkeys[index]
+    privkey = privkeys[index]
+    deposit, root, deposit_data_leaves = build_deposit(
+        pre_state,
+        deposit_data_leaves,
+        pubkey,
+        privkey,
+        spec.MAX_DEPOSIT_AMOUNT,
+    )
+
+    # mess up merkle branch
+    deposit.proof[-1] = spec.ZERO_HASH
+
+    pre_state.latest_eth1_data.deposit_root = root
+    pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
+
+    post_state = deepcopy(pre_state)
+
+    with pytest.raises(AssertionError):
+        process_deposit(post_state, deposit)
+
+    return pre_state, deposit, None

tests/phase0/block_processing/test_voluntary_exit.py

@@ -0,0 +1,175 @@
+from copy import deepcopy
+import pytest
+
+import build.phase0.spec as spec
+
+from build.phase0.spec import (
+    get_active_validator_indices,
+    get_current_epoch,
+    process_voluntary_exit,
+)
+from tests.phase0.helpers import (
+    build_voluntary_exit,
+    pubkey_to_privkey,
+)
+
+
+# mark entire file as 'voluntary_exits'
+pytestmark = pytest.mark.voluntary_exits
+
+
+def test_success(state):
+    pre_state = deepcopy(state)
+    #
+    # setup pre_state
+    #
+    # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit
+    pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+
+    #
+    # build voluntary exit
+    #
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
+
+    voluntary_exit = build_voluntary_exit(
+        pre_state,
+        current_epoch,
+        validator_index,
+        privkey,
+    )
+
+    post_state = deepcopy(pre_state)
+
+    #
+    # test valid exit
+    #
+    process_voluntary_exit(post_state, voluntary_exit)
+
+    assert not pre_state.validator_registry[validator_index].initiated_exit
+    assert post_state.validator_registry[validator_index].initiated_exit
+
+    return pre_state, voluntary_exit, post_state
+
+
+def test_validator_not_active(state):
+    pre_state = deepcopy(state)
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
+
+    #
+    # setup pre_state
+    #
+    pre_state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH
+
+    #
+    # build and test voluntary exit
+    #
+    voluntary_exit = build_voluntary_exit(
+        pre_state,
+        current_epoch,
+        validator_index,
+        privkey,
+    )
+
+    with pytest.raises(AssertionError):
+        process_voluntary_exit(pre_state, voluntary_exit)
+
+    return pre_state, voluntary_exit, None
+
+
+def test_validator_already_exited(state):
+    pre_state = deepcopy(state)
+    #
+    # setup pre_state
+    #
+    # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit
+    pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
+
+    # but validator already has exited
+    pre_state.validator_registry[validator_index].exit_epoch = current_epoch + 2
+
+    #
+    # build voluntary exit
+    #
+    voluntary_exit = build_voluntary_exit(
+        pre_state,
+        current_epoch,
+        validator_index,
+        privkey,
+    )
+
+    with pytest.raises(AssertionError):
+        process_voluntary_exit(pre_state, voluntary_exit)
+
+    return pre_state, voluntary_exit, None
+
+
+def test_validator_already_initiated_exit(state):
+    pre_state = deepcopy(state)
+    #
+    # setup pre_state
+    #
+    # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit
+    pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH
+
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
+
+    # but validator already has initiated exit
+    pre_state.validator_registry[validator_index].initiated_exit = True
+
+    #
+    # build voluntary exit
+    #
+    voluntary_exit = build_voluntary_exit(
+        pre_state,
+        current_epoch,
+        validator_index,
+        privkey,
+    )
+
+    with pytest.raises(AssertionError):
+        process_voluntary_exit(pre_state, voluntary_exit)
+
+    return pre_state, voluntary_exit, None
+
+
+def test_validator_not_active_long_enough(state):
+    pre_state = deepcopy(state)
+    #
+    # setup pre_state
+    #
+    current_epoch = get_current_epoch(pre_state)
+    validator_index = get_active_validator_indices(pre_state.validator_registry, current_epoch)[0]
+    privkey = pubkey_to_privkey[pre_state.validator_registry[validator_index].pubkey]
+
+    # but validator already has initiated exit
+    pre_state.validator_registry[validator_index].initiated_exit = True
+
+    #
+    # build voluntary exit
+    #
+    voluntary_exit = build_voluntary_exit(
+        pre_state,
+        current_epoch,
+        validator_index,
+        privkey,
+    )
+
+    assert (
+        current_epoch - pre_state.validator_registry[validator_index].activation_epoch <
+        spec.PERSISTENT_COMMITTEE_PERIOD
+    )
+
+    with pytest.raises(AssertionError):
+        process_voluntary_exit(pre_state, voluntary_exit)
+
+    return pre_state, voluntary_exit, None

tests/phase0/helpers.py

@@ -0,0 +1,203 @@
+from copy import deepcopy
+
+from py_ecc import bls
+
+import build.phase0.spec as spec
+from build.phase0.utils.minimal_ssz import signed_root
+from build.phase0.spec import (
+    # constants
+    EMPTY_SIGNATURE,
+    # SSZ
+    AttestationData,
+    Deposit,
+    DepositInput,
+    DepositData,
+    Eth1Data,
+    VoluntaryExit,
+    # functions
+    get_block_root,
+    get_current_epoch,
+    get_domain,
+    get_empty_block,
+    get_epoch_start_slot,
+    get_genesis_beacon_state,
+    verify_merkle_branch,
+    hash,
+)
+from build.phase0.utils.merkle_minimal import (
+    calc_merkle_tree_from_leaves,
+    get_merkle_proof,
+    get_merkle_root,
+)
+
+
+privkeys = [i + 1 for i in range(1000)]
+pubkeys = [bls.privtopub(privkey) for privkey in privkeys]
+pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)}
+
+
+def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None):
+    if not deposit_data_leaves:
+        deposit_data_leaves = []
+    deposit_timestamp = 0
+    proof_of_possession = b'\x33' * 96
+
+    deposit_data_list = []
+    for i in range(num_validators):
+        pubkey = pubkeys[i]
+        deposit_data = DepositData(
+            amount=spec.MAX_DEPOSIT_AMOUNT,
+            timestamp=deposit_timestamp,
+            deposit_input=DepositInput(
+                pubkey=pubkey,
+                # insecurely use pubkey as withdrawal key as well
+                withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:],
+                proof_of_possession=proof_of_possession,
+            ),
+        )
+        item = hash(deposit_data.serialize())
+        deposit_data_leaves.append(item)
+        tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
+        root = get_merkle_root((tuple(deposit_data_leaves)))
+        proof = list(get_merkle_proof(tree, item_index=i))
+        assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root)
+        deposit_data_list.append(deposit_data)
+
+    genesis_validator_deposits = []
+    for i in range(num_validators):
+        genesis_validator_deposits.append(Deposit(
+            proof=list(get_merkle_proof(tree, item_index=i)),
+            index=i,
+            deposit_data=deposit_data_list[i]
+        ))
+    return genesis_validator_deposits, root
+
+
+def create_genesis_state(num_validators, deposit_data_leaves=None):
+    initial_deposits, deposit_root = create_mock_genesis_validator_deposits(
+        num_validators,
+        deposit_data_leaves,
+    )
+    return get_genesis_beacon_state(
+        initial_deposits,
+        genesis_time=0,
+        genesis_eth1_data=Eth1Data(
+            deposit_root=deposit_root,
+            deposit_count=len(initial_deposits),
+            block_hash=spec.ZERO_HASH,
+        ),
+    )
+
+
+def force_registry_change_at_next_epoch(state):
+    # artificially trigger registry update at next epoch transition
+    state.finalized_epoch = get_current_epoch(state) - 1
+    for crosslink in state.latest_crosslinks:
+        crosslink.epoch = state.finalized_epoch
+    state.validator_registry_update_epoch = state.finalized_epoch - 1
+
+
+def build_empty_block_for_next_slot(state):
+    empty_block = get_empty_block()
+    empty_block.slot = state.slot + 1
+    previous_block_header = deepcopy(state.latest_block_header)
+    if previous_block_header.state_root == spec.ZERO_HASH:
+        previous_block_header.state_root = state.hash_tree_root()
+    empty_block.previous_block_root = signed_root(previous_block_header)
+    return empty_block
+
+
+def build_deposit_data(state, pubkey, privkey, amount):
+    deposit_input = DepositInput(
+        pubkey=pubkey,
+        # insecurely use pubkey as withdrawal key as well
+        withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:],
+        proof_of_possession=EMPTY_SIGNATURE,
+    )
+    proof_of_possession = bls.sign(
+        message_hash=signed_root(deposit_input),
+        privkey=privkey,
+        domain=get_domain(
+            state.fork,
+            get_current_epoch(state),
+            spec.DOMAIN_DEPOSIT,
+        )
+    )
+    deposit_input.proof_of_possession = proof_of_possession
+    deposit_data = DepositData(
+        amount=amount,
+        timestamp=0,
+        deposit_input=deposit_input,
+    )
+    return deposit_data
+
+
+def build_attestation_data(state, slot, shard):
+    assert state.slot >= slot
+
+    block_root = build_empty_block_for_next_slot(state).previous_block_root
+
+    epoch_start_slot = get_epoch_start_slot(get_current_epoch(state))
+    if epoch_start_slot == slot:
+        epoch_boundary_root = block_root
+    else:
+        get_block_root(state, epoch_start_slot)
+
+    if slot < epoch_start_slot:
+        justified_block_root = state.previous_justified_root
+    else:
+        justified_block_root = state.current_justified_root
+
+    return AttestationData(
+        slot=slot,
+        shard=shard,
+        beacon_block_root=block_root,
+        source_epoch=state.current_justified_epoch,
+        source_root=justified_block_root,
+        target_root=epoch_boundary_root,
+        crosslink_data_root=spec.ZERO_HASH,
+        previous_crosslink=deepcopy(state.latest_crosslinks[shard]),
+    )
+
+
+def build_voluntary_exit(state, epoch, validator_index, privkey):
+    voluntary_exit = VoluntaryExit(
+        epoch=epoch,
+        validator_index=validator_index,
+        signature=EMPTY_SIGNATURE,
+    )
+    voluntary_exit.signature = bls.sign(
+        message_hash=signed_root(voluntary_exit),
+        privkey=privkey,
+        domain=get_domain(
+            fork=state.fork,
+            epoch=epoch,
+            domain_type=spec.DOMAIN_VOLUNTARY_EXIT,
+        )
+    )
+
+    return voluntary_exit
+
+
+def build_deposit(state,
+                  deposit_data_leaves,
+                  pubkey,
+                  privkey,
+                  amount):
+    deposit_data = build_deposit_data(state, pubkey, privkey, amount)
+
+    item = hash(deposit_data.serialize())
+    index = len(deposit_data_leaves)
+    deposit_data_leaves.append(item)
+    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
+    root = get_merkle_root((tuple(deposit_data_leaves)))
+    proof = list(get_merkle_proof(tree, item_index=index))
+    assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root)
+
+    deposit = Deposit(
+        proof=list(proof),
+        index=index,
+        deposit_data=deposit_data,
+    )
+
+    return deposit, root, deposit_data_leaves

utils/phase0/jsonize.py

@@ -0,0 +1,52 @@
+from .minimal_ssz import hash_tree_root
+
+
+def jsonize(value, typ, include_hash_tree_roots=False):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return value
+    elif typ == 'bool':
+        assert value in (True, False)
+        return value
+    elif isinstance(typ, list):
+        return [jsonize(element, typ[0], include_hash_tree_roots) for element in value]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return '0x' + value.hex()
+    elif hasattr(typ, 'fields'):
+        ret = {}
+        for field, subtype in typ.fields.items():
+            ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots)
+            if include_hash_tree_roots:
+                ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex()
+        if include_hash_tree_roots:
+            ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex()
+        return ret
+    else:
+        print(value, typ)
+        raise Exception("Type not recognized")
+
+
+def dejsonize(json, typ):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return json
+    elif typ == 'bool':
+        assert json in (True, False)
+        return json
+    elif isinstance(typ, list):
+        return [dejsonize(element, typ[0]) for element in json]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return bytes.fromhex(json[2:])
+    elif hasattr(typ, 'fields'):
+        temp = {}
+        for field, subtype in typ.fields.items():
+            temp[field] = dejsonize(json[field], subtype)
+            if field + "_hash_tree_root" in json:
+                assert(json[field + "_hash_tree_root"][2:] == 
+                       hash_tree_root(temp[field], subtype).hex())
+        ret = typ(**temp)
+        if "hash_tree_root" in json:
+            assert(json["hash_tree_root"][2:] == 
+                   hash_tree_root(ret, typ).hex())
+        return ret
+    else:
+        print(json, typ)
+        raise Exception("Type not recognized")