move tests, update operations test vector generation

test_generators/operations/deposits.py

@@ -1,180 +0,0 @@
-from eth2spec.phase0 import spec
-from eth_utils import (
-    to_dict, to_tuple
-)
-from gen_base import gen_suite, gen_typing
-from preset_loader import loader
-from eth2spec.debug.encode import encode
-from eth2spec.utils.minimal_ssz import signing_root
-from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof
-
-from typing import List, Tuple
-
-import genesis
-import keys
-from py_ecc import bls
-
-
-def build_deposit_data(state,
-                       pubkey: spec.BLSPubkey,
-                       withdrawal_cred: spec.Bytes32,
-                       privkey: int,
-                       amount: int):
-    deposit_data = spec.DepositData(
-        pubkey=pubkey,
-        withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:],
-        amount=amount,
-    )
-    deposit_data.proof_of_possession = bls.sign(
-        message_hash=signing_root(deposit_data),
-        privkey=privkey,
-        domain=spec.get_domain(
-            state,
-            spec.get_current_epoch(state),
-            spec.DOMAIN_DEPOSIT,
-        )
-    )
-    return deposit_data
-
-
-def build_deposit(state,
-                  deposit_data_leaves: List[spec.Bytes32],
-                  pubkey: spec.BLSPubkey,
-                  withdrawal_cred: spec.Bytes32,
-                  privkey: int,
-                  amount: int) -> spec.Deposit:
-
-    deposit_data = build_deposit_data(state, pubkey, withdrawal_cred, privkey, amount)
-
-    item = deposit_data.hash_tree_root()
-    index = len(deposit_data_leaves)
-    deposit_data_leaves.append(item)
-    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
-    proof = list(get_merkle_proof(tree, item_index=index))
-
-    deposit = spec.Deposit(
-        proof=list(proof),
-        index=index,
-        data=deposit_data,
-    )
-    assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, get_merkle_root(tuple(deposit_data_leaves)))
-
-    return deposit
-
-
-def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState]:
-    genesis_deposits = genesis.create_deposits(
-        keys.pubkeys[:initial_validator_count],
-        keys.withdrawal_creds[:initial_validator_count]
-    )
-    state = genesis.create_genesis_state(genesis_deposits)
-
-    deposit_data_leaves = [dep.data.hash_tree_root() for dep in genesis_deposits]
-
-    deposit = build_deposit(
-        state,
-        deposit_data_leaves,
-        keys.pubkeys[index],
-        keys.withdrawal_creds[index],
-        keys.privkeys[index],
-        spec.MAX_EFFECTIVE_BALANCE,
-    )
-
-    state.latest_eth1_data.deposit_root = get_merkle_root(tuple(deposit_data_leaves))
-    state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
-
-    return deposit, state
-
-
-@to_dict
-def valid_deposit():
-    new_dep, state = build_deposit_for_index(10, 10)
-    yield 'description', 'valid deposit to add new validator'
-    yield 'pre', encode(state, spec.BeaconState)
-    yield 'deposit', encode(new_dep, spec.Deposit)
-    spec.process_deposit(state, new_dep)
-    yield 'post', encode(state, spec.BeaconState)
-
-
-@to_dict
-def valid_topup():
-    new_dep, state = build_deposit_for_index(10, 3)
-    yield 'description', 'valid deposit to top-up existing validator'
-    yield 'pre', encode(state, spec.BeaconState)
-    yield 'deposit', encode(new_dep, spec.Deposit)
-    spec.process_deposit(state, new_dep)
-    yield 'post', encode(state, spec.BeaconState)
-
-
-@to_dict
-def invalid_deposit_index():
-    new_dep, state = build_deposit_for_index(10, 10)
-    # Mess up deposit index, 1 too small
-    state.deposit_index = 9
-
-    yield 'description', 'invalid deposit index'
-    yield 'pre', encode(state, spec.BeaconState)
-    yield 'deposit', encode(new_dep, spec.Deposit)
-    try:
-        spec.process_deposit(state, new_dep)
-    except AssertionError:
-        # expected
-        yield 'post', None
-        return
-    raise Exception('invalid_deposit_index has unexpectedly allowed deposit')
-
-
-@to_dict
-def invalid_deposit_proof():
-    new_dep, state = build_deposit_for_index(10, 10)
-    # Make deposit proof invalid (at bottom of proof)
-    new_dep.proof[-1] = spec.ZERO_HASH
-
-    yield 'description', 'invalid deposit proof'
-    yield 'pre', encode(state, spec.BeaconState)
-    yield 'deposit', encode(new_dep, spec.Deposit)
-    try:
-        spec.process_deposit(state, new_dep)
-    except AssertionError:
-        # expected
-        yield 'post', None
-        return
-    raise Exception('invalid_deposit_index has unexpectedly allowed deposit')
-
-
-@to_tuple
-def deposit_cases():
-    yield valid_deposit()
-    yield valid_topup()
-    yield invalid_deposit_index()
-    yield invalid_deposit_proof()
-
-
-def mini_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
-    presets = loader.load_presets(configs_path, 'minimal')
-    spec.apply_constants_preset(presets)
-
-    return ("deposit_minimal", "deposits", gen_suite.render_suite(
-        title="deposit operation",
-        summary="Test suite for deposit type operation processing",
-        forks_timeline="testing",
-        forks=["phase0"],
-        config="minimal",
-        runner="operations",
-        handler="deposits",
-        test_cases=deposit_cases()))
-
-
-def full_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
-    presets = loader.load_presets(configs_path, 'mainnet')
-    spec.apply_constants_preset(presets)
-
-    return ("deposit_full", "deposits", gen_suite.render_suite(
-        title="deposit operation",
-        summary="Test suite for deposit type operation processing",
-        forks_timeline="mainnet",
-        forks=["phase0"],
-        config="mainnet",
-        runner="operations",
-        handler="deposits",
-        test_cases=deposit_cases()))

test_generators/operations/genesis.py

@@ -1,44 +0,0 @@
-from eth2spec.phase0 import spec
-from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof
-from typing import List
-
-
-def create_genesis_state(deposits: List[spec.Deposit]) -> spec.BeaconState:
-    deposit_root = get_merkle_root((tuple([(dep.data.hash_tree_root()) for dep in deposits])))
-
-    return spec.get_genesis_beacon_state(
-        deposits,
-        genesis_time=0,
-        genesis_eth1_data=spec.Eth1Data(
-            deposit_root=deposit_root,
-            deposit_count=len(deposits),
-            block_hash=spec.ZERO_HASH,
-        ),
-    )
-
-
-def create_deposits(pubkeys: List[spec.BLSPubkey], withdrawal_cred: List[spec.Bytes32]) -> List[spec.Deposit]:
-
-    # Mock proof of possession
-    proof_of_possession = b'\x33' * 96
-
-    deposit_data = [
-        spec.DepositData(
-            pubkey=pubkeys[i],
-            withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[i][1:],
-            amount=spec.MAX_EFFECTIVE_BALANCE,
-            proof_of_possession=proof_of_possession,
-        ) for i in range(len(pubkeys))
-    ]
-
-    # Fill tree with existing deposits
-    deposit_data_leaves = [data.hash_tree_root() for data in deposit_data]
-    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
-
-    return [
-        spec.Deposit(
-            proof=list(get_merkle_proof(tree, item_index=i)),
-            index=i,
-            data=deposit_data[i]
-        ) for i in range(len(deposit_data))
-    ]

test_generators/operations/keys.py

@@ -1,7 +0,0 @@
-from py_ecc import bls
-from eth2spec.phase0.spec import hash
-
-privkeys = list(range(1, 101))
-pubkeys = [bls.privtopub(k) for k in privkeys]
-# Insecure, but easier to follow
-withdrawal_creds = [hash(bls.privtopub(k)) for k in privkeys]

test_generators/operations/main.py

@@ -1,9 +1,31 @@
+from eth2spec.testing.block_processing import (
+    test_process_attestation,
+    test_process_attester_slashing,
+    test_process_block_header,
+    test_process_deposit,
+    test_process_proposer_slashing,
+    test_process_transfer,
+    test_process_voluntary_exit
+)
+
 from gen_base import gen_runner
 
-from deposits import mini_deposits_suite, full_deposits_suite
+from suite_creator import generate_from_tests, create_suite
 
 if __name__ == "__main__":
     gen_runner.run_generator("operations", [
-        mini_deposits_suite,
+        create_suite('attestation',       'minimal', lambda: generate_from_tests(test_process_attestation)),
-        full_deposits_suite
+        create_suite('attestation',       'mainnet', lambda: generate_from_tests(test_process_attestation)),
+        create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)),
+        create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)),
+        create_suite('block_header',      'minimal', lambda: generate_from_tests(test_process_block_header)),
+        create_suite('block_header',      'mainnet', lambda: generate_from_tests(test_process_block_header)),
+        create_suite('deposits',          'minimal', lambda: generate_from_tests(test_process_deposit)),
+        create_suite('deposits',          'mainnet', lambda: generate_from_tests(test_process_deposit)),
+        create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)),
+        create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing)),
+        create_suite('transfer',          'minimal', lambda: generate_from_tests(test_process_transfer)),
+        create_suite('transfer',          'mainnet', lambda: generate_from_tests(test_process_transfer)),
+        create_suite('voluntary_exit',    'minimal', lambda: generate_from_tests(test_process_voluntary_exit)),
+        create_suite('voluntary_exit',    'mainnet', lambda: generate_from_tests(test_process_voluntary_exit)),
     ])

test_generators/operations/requirements.txt

@@ -1,5 +1,4 @@
 eth-utils==1.4.1
 ../../test_libs/gen_helpers
 ../../test_libs/config_helpers
-../../test_libs/pyspec
+../../test_libs/pyspec
-py_ecc

test_generators/operations/suite_creator.py

@@ -0,0 +1,39 @@
+from typing import Callable, Iterable
+from inspect import getmembers, isfunction
+from gen_base import gen_suite, gen_typing
+from preset_loader import loader
+from eth2spec.phase0 import spec
+
+
+def generate_from_tests(pkg):
+    fn_names = [
+        name for (name, _) in getmembers(pkg, isfunction)
+        if name.startswith('test_')
+    ]
+    out = []
+    print("generating test vectors from tests package: %s" % pkg.__name__)
+    for name in fn_names:
+        tfn = getattr(pkg, name)
+        try:
+            out.append(tfn(generator_mode=True))
+        except AssertionError:
+            print("ERROR: failed to generate vector from test: %s (pkg: %s)" % (name, pkg.__name__))
+    return out
+
+
+def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]])\
+        -> Callable[[str], gen_typing.TestSuiteOutput]:
+    def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput:
+        presets = loader.load_presets(configs_path, config_name)
+        spec.apply_constants_preset(presets)
+
+        return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite(
+            title="%s operation" % operation_name,
+            summary="Test suite for deposit type operation processing",
+            forks_timeline="testing",
+            forks=["phase0"],
+            config=config_name,
+            runner="operations",
+            handler=config_name,
+            test_cases=get_cases()))
+    return suite_definition

test_libs/pyspec/eth2spec/testing/block_processing/test_process_attestation.py

@@ -1,5 +1,4 @@
 from copy import deepcopy
-import pytest
 
 import eth2spec.phase0.spec as spec
 
@@ -10,14 +9,14 @@ from eth2spec.phase0.spec import (
     get_current_epoch,
     process_attestation
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     build_empty_block_for_next_slot,
     get_valid_attestation,
     next_epoch,
     next_slot,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_attestation_processing(state, attestation, valid=True):
@@ -35,8 +34,7 @@ def run_attestation_processing(state, attestation, valid=True):
 
     # If the attestation is invalid, processing is aborted, and there is no post-state.
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_attestation(state, attestation))
-            process_attestation(state, attestation)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/block_processing/test_process_attester_slashing.py

@@ -1,17 +1,15 @@
-import pytest
-
 import eth2spec.phase0.spec as spec
 from eth2spec.phase0.spec import (
     get_beacon_proposer_index,
     process_attester_slashing,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     get_balance,
     get_valid_attester_slashing,
     next_epoch,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_attester_slashing_processing(state, attester_slashing, valid=True):
@@ -27,8 +25,7 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True):
     yield 'attester_slashing', attester_slashing
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_attester_slashing(state, attester_slashing))
-            process_attester_slashing(state, attester_slashing)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/block_processing/test_process_block_header.py

@@ -1,6 +1,4 @@
 from copy import deepcopy
-import pytest
-
 
 from eth2spec.phase0.spec import (
     get_beacon_proposer_index,
@@ -8,12 +6,12 @@ from eth2spec.phase0.spec import (
     advance_slot,
     process_block_header,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     build_empty_block_for_next_slot,
     next_slot,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def prepare_state_for_header_processing(state):
@@ -35,8 +33,7 @@ def run_block_header_processing(state, block, valid=True):
     yield 'block', block
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_block_header(state, block))
-            process_block_header(state, block)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/block_processing/test_process_deposit.py

@@ -1,12 +1,10 @@
-import pytest
-
 import eth2spec.phase0.spec as spec
 
 from eth2spec.phase0.spec import (
     ZERO_HASH,
     process_deposit,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     get_balance,
     build_deposit,
     prepare_state_and_deposit,
@@ -14,7 +12,7 @@ from tests.helpers import (
     pubkeys,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_deposit_processing(state, deposit, validator_index, valid=True):
@@ -37,8 +35,7 @@ def run_deposit_processing(state, deposit, validator_index, valid=True):
     yield 'deposit', deposit
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_deposit(state, deposit))
-            process_deposit(state, deposit)
         yield 'post', None
         return
 
@@ -90,6 +87,9 @@ def test_wrong_index(state):
     yield from run_deposit_processing(state, deposit, validator_index, valid=False)
 
 
+# TODO: test invalid signature
+
+
 @spec_state_test
 def test_bad_merkle_proof(state):
     validator_index = len(state.validator_registry)

test_libs/pyspec/eth2spec/testing/block_processing/test_process_proposer_slashing.py

@@ -1,16 +1,14 @@
-import pytest
-
 import eth2spec.phase0.spec as spec
 from eth2spec.phase0.spec import (
     get_current_epoch,
     process_proposer_slashing,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     get_balance,
     get_valid_proposer_slashing,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_proposer_slashing_processing(state, proposer_slashing, valid=True):
@@ -27,8 +25,7 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True):
     yield 'proposer_slashing', proposer_slashing
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_proposer_slashing(state, proposer_slashing))
-            process_proposer_slashing(state, proposer_slashing)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/block_processing/test_process_transfer.py

@@ -1,5 +1,3 @@
-import pytest
-
 import eth2spec.phase0.spec as spec
 
 from eth2spec.phase0.spec import (
@@ -8,12 +6,12 @@ from eth2spec.phase0.spec import (
     get_current_epoch,
     process_transfer,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     get_valid_transfer,
     next_epoch,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_transfer_processing(state, transfer, valid=True):
@@ -34,8 +32,7 @@ def run_transfer_processing(state, transfer, valid=True):
     yield 'transfer', transfer
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_transfer(state, transfer))
-            process_transfer(state, transfer)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/block_processing/test_process_voluntary_exit.py

@@ -1,5 +1,3 @@
-import pytest
-
 import eth2spec.phase0.spec as spec
 
 from eth2spec.phase0.spec import (
@@ -8,12 +6,12 @@ from eth2spec.phase0.spec import (
     get_current_epoch,
     process_voluntary_exit,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     build_voluntary_exit,
     pubkey_to_privkey,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test, expect_assertion_error
 
 
 def run_voluntary_exit_processing(state, voluntary_exit, valid=True):
@@ -31,8 +29,7 @@ def run_voluntary_exit_processing(state, voluntary_exit, valid=True):
     yield 'voluntary_exit', voluntary_exit
 
     if not valid:
-        with pytest.raises(AssertionError):
+        expect_assertion_error(lambda: process_voluntary_exit(state, voluntary_exit))
-            process_voluntary_exit(state, voluntary_exit)
         yield 'post', None
         return
 

test_libs/pyspec/eth2spec/testing/context.py

@@ -1,12 +1,8 @@
-
 from eth2spec.phase0 import spec
-from tests.utils import with_args
 
-from .helpers import (
+from .helpers import create_genesis_state
-    create_genesis_state,
-)
 
-from tests.utils import spectest
+from .utils import spectest, with_args
 
 # Provides a genesis state as first argument to the function decorated with this
 with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, list())])
@@ -15,3 +11,11 @@ with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8, l
 # shorthand for decorating @with_state @spectest()
 def spec_state_test(fn):
     return with_state(spectest()(fn))
+
+
+def expect_assertion_error(fn):
+    try:
+        fn()
+        raise AssertionError('expected an assertion error, but got none.')
+    except AssertionError:
+        pass

test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_crosslinks.py

@@ -10,7 +10,7 @@ from eth2spec.phase0.spec import (
     get_crosslink_deltas,
     process_crosslinks,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     add_attestation_to_state,
     build_empty_block_for_next_slot,
     fill_aggregate_attestation,
@@ -19,8 +19,7 @@ from tests.helpers import (
     next_epoch,
     next_slot,
 )
-
+from eth2spec.testing.context import spec_state_test
-from tests.context import spec_state_test
 
 
 def run_process_crosslinks(state, valid=True):

test_libs/pyspec/eth2spec/testing/epoch_processing/test_process_registry_updates.py

@@ -4,11 +4,11 @@ from eth2spec.phase0.spec import (
     get_current_epoch,
     is_active_validator,
 )
-from tests.helpers import (
+from eth2spec.testing.helpers import (
     next_epoch,
 )
 
-from tests.context import spec_state_test
+from eth2spec.testing.context import spec_state_test
 
 
 @spec_state_test