From 1c51982c6b5de8fd9dfb851cb85086fa83205d52 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 23:12:04 +0200 Subject: [PATCH 1/3] generate coverage reports in make test, open as html site --- .gitignore | 7 +++++++ Makefile | 14 ++++++++++---- test_libs/pyspec/requirements-testing.txt | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c6b39955f..16d39a434 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py test_libs/pyspec/eth2spec/phase1/spec.py + +# coverage reports +.htmlcov +.coverage + +# local CI testing output +test_libs/pyspec/test-reports diff --git a/Makefile b/Makefile index f79b89dad..9e658f35d 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,10 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) +COV_INDEX_FILE=$(PY_SPEC_DIR)/.htmlcov/index.html -.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test install_deposit_contract_test test_deposit_contract compile_deposit_contract +.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \ + install_deposit_contract_test test_deposit_contract compile_deposit_contract all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) @@ -41,11 +43,15 @@ install_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest eth2spec + cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ + python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:.htmlcov" --cov-branch eth2spec citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml eth2spec + cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ + python -m pytest --junitxml=test-reports/eth2spec/test_results.xml eth2spec + +open_cov: + ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index 331d0fa28..edd141650 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -2,3 +2,4 @@ pytest>=3.6,<3.7 ../config_helpers flake8==3.7.7 +pytest-cov From 20aa539f4dc3903a7f73fac2a4d26e257b34ee26 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 01:13:29 +0200 Subject: [PATCH 2/3] update clean command --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9e658f35d..1b7155625 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) -COV_INDEX_FILE=$(PY_SPEC_DIR)/.htmlcov/index.html +COV_HTML_OUT=.htmlcov +COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html .PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \ install_deposit_contract_test test_deposit_contract compile_deposit_contract @@ -34,6 +35,9 @@ clean: rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache + rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT) + rm -rf $(PY_SPEC_DIR)/.coverage + rm -rf $(PY_SPEC_DIR)/test-reports # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -44,7 +48,7 @@ install_test: test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ - python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:.htmlcov" --cov-branch eth2spec + python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec -k transfer citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ From 2f9c554bf7370319bfc0d61feaaf2a34338583b9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 16 Jun 2019 17:38:48 -0600 Subject: [PATCH 3/3] minor fix to makefile, add codecov instructiosn to readme --- Makefile | 2 +- test_libs/pyspec/README.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1b7155625..4ee757f88 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ install_test: test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; export PYTHONPATH="./"; \ - python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec -k transfer + python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 2c2226ee7..53750517d 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -28,7 +28,7 @@ These tests are sanity tests, to verify if the spec itself is consistent. #### Automated -Run `make test` from the root of the specs repository. +Run `make test` from the root of the specs repository (after running `make install_test` if have not before). #### Manual @@ -50,6 +50,10 @@ pytest --config=minimal eth2spec ``` Note the package-name, this is to locate the tests. +### How to view code coverage report + +Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report. + ## Contributing