Merge pull request #979 from ethereum/pytest-use-config

pytests use configuration system

.circleci/config.yml

@@ -60,15 +60,15 @@ jobs:
       - restore_cache:
           key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_cached_venv:
-          venv_name: v1-pyspec
-          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
+          venv_name: v1-test_libs
+          reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}'
       - run:
           name: Install pyspec requirements
           command: make install_test
       - save_cached_venv:
-          venv_name: v1-pyspec
-          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
-          venv_path: ./test_libs/pyspec/venv
+          venv_name: v1-test_libs
+          reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}'
+          venv_path: ./test_libs/venv
   test:
     docker:
       - image: circleci/python:3.6
@@ -77,8 +77,8 @@ jobs:
       - restore_cache:
           key: v1-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_cached_venv:
-          venv_name: v1-pyspec
-          reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}'
+          venv_name: v1-test_libs
+          reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}'
       - run:
           name: Run py-tests
           command: make citest

.gitignore

@@ -4,6 +4,10 @@ venv
 .venvs
 .venv
 /.pytest_cache
+*.egg
+*.egg-info
+eggs
+.eggs
 
 build/
 output/
@@ -13,3 +17,6 @@ eth2.0-spec-tests/
 
 # Dynamically built from Markdown spec
 test_libs/pyspec/eth2spec/phase0/spec.py
+
+# vscode
+.vscode/**

Makefile

@@ -2,6 +2,7 @@ SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
 TEST_LIBS_DIR = ./test_libs
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+CONFIG_HELPERS_DIR = $(TEST_LIBS_DIR)/config_helpers
 YAML_TEST_DIR = ./eth2.0-spec-tests/tests
 GENERATOR_DIR = ./test_generators
 CONFIGS_DIR = ./configs
@@ -23,7 +24,8 @@ all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 clean:
 	rm -rf $(YAML_TEST_DIR)
 	rm -rf $(GENERATOR_VENVS)
-	rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache
+	rm -rf $(TEST_LIBS_DIR)/venv
+	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
 
 # "make gen_yaml_tests" to run generators
@@ -31,13 +33,17 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
 
 # installs the packages to run pyspec tests
 install_test:
-	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt;
+	cd $(TEST_LIBS_DIR); python3 -m venv venv; . venv/bin/activate; \
+	cd ..; cd $(CONFIG_HELPERS_DIR); pip3 install -e .; \
+	cd ../..; cd $(PY_SPEC_DIR); pip3 install -e .[dev];
 
 test: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest -m minimal_config .
+	cd $(TEST_LIBS_DIR); . venv/bin/activate; \
+	cd ..; cd $(PY_SPEC_DIR); python -m pytest .;
 
 citest: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml -m minimal_config .
+	cd $(TEST_LIBS_DIR); . venv/bin/activate; \
+	cd ..; cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml .
 
 # "make pyspec" to create the pyspec for all phases.
 pyspec: $(PY_SPEC_ALL_TARGETS)

test_generators/README.md

@@ -72,7 +72,7 @@ Note: make sure to run `make pyspec` from the root of the specs repository, to b
 
 Install all the necessary requirements (re-run when you add more):
 ```bash
-pip3 install -r requirements.txt
+pip3 install -e .[pyspec]
 ```
 
 And write your initial test generator, extending the base generator:

test_libs/config_helpers/requirements.txt

@@ -1 +0,0 @@
-ruamel.yaml==0.15.87

test_libs/config_helpers/setup.py

@@ -1,9 +1,20 @@
-from distutils.core import setup
+from setuptools import setup, find_packages
+
+
+deps = {
+    'preset_loader': [
+        "ruamel.yaml==0.15.87",
+    ],
+}
+
+deps['dev'] = (
+    deps['preset_loader']
+)
+
+install_requires = deps['preset_loader']
 
 setup(
     name='config_helpers',
-    packages=['preset_loader'],
-    install_requires=[
-        "ruamel.yaml==0.15.87"
-    ]
+    packages=find_packages(exclude=["tests", "tests.*"]),
+    install_requires=install_requires,
 )

test_libs/gen_helpers/requirements.txt

@@ -1,2 +0,0 @@
-ruamel.yaml==0.15.87
-eth-utils==1.4.1

test_libs/gen_helpers/setup.py

@@ -1,10 +1,21 @@
-from distutils.core import setup
+from setuptools import setup, find_packages
+
+
+deps = {
+    'gen_base': [
+        "ruamel.yaml==0.15.87",
+        "eth-utils==1.4.1",
+    ],
+}
+
+deps['dev'] = (
+    deps['gen_base']
+)
+
+install_requires = deps['gen_base']
 
 setup(
     name='gen_helpers',
-    packages=['gen_base'],
-    install_requires=[
-        "ruamel.yaml==0.15.87",
-        "eth-utils==1.4.1"
-    ]
+    packages=find_packages(exclude=["tests", "tests.*"]),
+    install_requires=install_requires,
 )

test_libs/pyspec/README.md

@@ -38,7 +38,7 @@ Install dependencies:
 ```bash
 python3 -m venv venv
 . venv/bin/activate
-pip3 install -r requirements.txt
+pip3 install -e .[dev]
 ```
 Note: make sure to run `make -B pyspec` from the root of the specs repository,
  to build the parts of the pyspec module derived from the markdown specs.
@@ -46,7 +46,7 @@ The `-B` flag may be helpful to force-overwrite the `pyspec` output after you ma
 
 Run the tests:
 ```
-pytest -m minimal_config .
+pytest --config=minimal
 ```
 
 

test_libs/pyspec/requirements.txt

@@ -1,5 +0,0 @@
-eth-utils>=1.3.0,<2
-eth-typing>=2.1.0,<3.0.0
-pycryptodome==3.7.3
-py_ecc>=1.6.0
-pytest>=3.6,<3.7

test_libs/pyspec/setup.py

@@ -1,13 +1,28 @@
 from setuptools import setup, find_packages
 
-setup(
-    name='pyspec',
-    packages=find_packages(),
-    tests_require=["pytest"],
-    install_requires=[
+
+deps = {
+    'pyspec': [
         "eth-utils>=1.3.0,<2",
         "eth-typing>=2.1.0,<3.0.0",
         "pycryptodome==3.7.3",
         "py_ecc>=1.6.0",
-    ]
+    ],
+    'test': [
+        "pytest>=3.6,<3.7",
+    ],
+}
+
+deps['dev'] = (
+    deps['pyspec'] +
+    deps['test']
+)
+
+install_requires = deps['pyspec']
+
+setup(
+    name='pyspec',
+    packages=find_packages(exclude=["tests", "tests.*"]),
+    install_requires=install_requires,
+    extras_require=deps,
 )

test_libs/pyspec/tests/conftest.py

@@ -1,63 +1,29 @@
 import pytest
 
 from eth2spec.phase0 import spec
+from preset_loader import loader
 
 from .helpers import (
     create_genesis_state,
 )
 
 
-DEFAULT_CONFIG = {}  # no change
-
-MINIMAL_CONFIG = {
-    "SHARD_COUNT": 8,
-    "MIN_ATTESTATION_INCLUSION_DELAY": 2,
-    "TARGET_COMMITTEE_SIZE": 4,
-    "SLOTS_PER_EPOCH": 8,
-    "SLOTS_PER_HISTORICAL_ROOT": 64,
-    "LATEST_RANDAO_MIXES_LENGTH": 64,
-    "LATEST_ACTIVE_INDEX_ROOTS_LENGTH": 64,
-    "LATEST_SLASHED_EXIT_LENGTH": 64,
-}
-
-
-def overwrite_spec_config(config):
-    for field in config:
-        setattr(spec, field, config[field])
-        if field == "LATEST_RANDAO_MIXES_LENGTH":
-            spec.BeaconState.fields['latest_randao_mixes'][1] = config[field]
-        elif field == "SHARD_COUNT":
-            spec.BeaconState.fields['current_crosslinks'][1] = config[field]
-            spec.BeaconState.fields['previous_crosslinks'][1] = config[field]
-        elif field == "SLOTS_PER_HISTORICAL_ROOT":
-            spec.BeaconState.fields['latest_block_roots'][1] = config[field]
-            spec.BeaconState.fields['latest_state_roots'][1] = config[field]
-            spec.HistoricalBatch.fields['block_roots'][1] = config[field]
-            spec.HistoricalBatch.fields['state_roots'][1] = config[field]
-        elif field == "LATEST_ACTIVE_INDEX_ROOTS_LENGTH":
-            spec.BeaconState.fields['latest_active_index_roots'][1] = config[field]
-        elif field == "LATEST_SLASHED_EXIT_LENGTH":
-            spec.BeaconState.fields['latest_slashed_balances'][1] = config[field]
-
-
-@pytest.fixture(
-    params=[
-        pytest.param(MINIMAL_CONFIG, marks=pytest.mark.minimal_config),
-        DEFAULT_CONFIG,
-    ]
+def pytest_addoption(parser):
+    parser.addoption(
+        "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration"
     )
-def config(request):
-    return request.param
 
 
 @pytest.fixture(autouse=True)
-def overwrite_config(config):
-    overwrite_spec_config(config)
+def config(request):
+    config_name = request.config.getoption("--config")
+    presets = loader.load_presets('../../configs/', config_name)
+    spec.apply_constants_preset(presets)
 
 
 @pytest.fixture
-def num_validators():
-    return 100
+def num_validators(config):
+    return spec.SLOTS_PER_EPOCH * 8
 
 
 @pytest.fixture

test_libs/pyspec/tests/helpers.py

@@ -48,7 +48,7 @@ from eth2spec.utils.merkle_minimal import (
 )
 
 
-privkeys = [i + 1 for i in range(1000)]
+privkeys = [i + 1 for i in range(1024)]
 pubkeys = [bls.privtopub(privkey) for privkey in privkeys]
 pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)}
 
@@ -122,6 +122,7 @@ def create_genesis_state(num_validators, deposit_data_leaves=None):
 def build_empty_block_for_next_slot(state):
     empty_block = BeaconBlock()
     empty_block.slot = state.slot + 1
+    empty_block.body.eth1_data.deposit_count = state.deposit_index
     previous_block_header = deepcopy(state.latest_block_header)
     if previous_block_header.state_root == spec.ZERO_HASH:
         previous_block_header.state_root = state.hash_tree_root()

test_libs/setup.py

@@ -0,0 +1,29 @@
+from setuptools import setup, find_packages
+
+
+deps = {
+    'pyspec': [
+        "eth-utils>=1.3.0,<2",
+        "eth-typing>=2.1.0,<3.0.0",
+        "pycryptodome==3.7.3",
+        "py_ecc>=1.6.0",
+    ],
+    'test': [
+        "pytest>=3.6,<3.7",
+    ],
+}
+
+deps['dev'] = (
+    deps['pyspec'] +
+    deps['test']
+)
+
+install_requires = deps['pyspec']
+
+
+setup(
+    name='pyspec',
+    packages=find_packages(exclude=["tests", "tests.*"]),
+    install_requires=install_requires,
+    extras_require=deps,
+)