diff --git a/.gitignore b/.gitignore index c6b39955f..16d39a434 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py test_libs/pyspec/eth2spec/phase1/spec.py + +# coverage reports +.htmlcov +.coverage + +# local CI testing output +test_libs/pyspec/test-reports diff --git a/Makefile b/Makefile index f79b89dad..4ee757f88 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,11 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) +COV_HTML_OUT=.htmlcov +COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html -.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test install_deposit_contract_test test_deposit_contract compile_deposit_contract +.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \ + install_deposit_contract_test test_deposit_contract compile_deposit_contract all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) @@ -32,6 +35,9 @@ clean: rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache + rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) + rm -rf $(PY_SPEC_DIR)/.coverage + rm -rf $(PY_SPEC_DIR)/test-reports # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -41,11 +47,15 @@ install_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest eth2spec + cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ + python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml eth2spec + cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ + python -m pytest --junitxml=test-reports/eth2spec/test_results.xml eth2spec + +open_cov: + ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 2c2226ee7..53750517d 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -28,7 +28,7 @@ These tests are sanity tests, to verify if the spec itself is consistent. #### Automated -Run `make test` from the root of the specs repository. +Run `make test` from the root of the specs repository (after running `make install_test` if have not before). #### Manual @@ -50,6 +50,10 @@ pytest --config=minimal eth2spec ``` Note the package-name, this is to locate the tests. +### How to view code coverage report + +Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report. + ## Contributing diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index 331d0fa28..edd141650 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -2,3 +2,4 @@ pytest>=3.6,<3.7 ../config_helpers flake8==3.7.7 +pytest-cov