From 6d146e2bc115c3172341d03e3aef1626952e6ca1 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 16 Jul 2024 14:16:18 -0500 Subject: [PATCH] For pytest, replace NUMBER_OF_CORES with auto --- Makefile | 7 +++---- docker/README.md | 8 ++++---- scripts/build_run_docker_tests.sh | 10 +++------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index bdf4bdde7..04b25267e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ SOLIDITY_FILE_NAME = deposit_contract.json DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester CONFIGS_DIR = ./configs TEST_PRESET_TYPE ?= minimal -NUMBER_OF_CORES=16 # Collect a list of generator names GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.))) # Map this list of generator paths to "gen_{generator name}" entries @@ -118,7 +117,7 @@ install_test: preinstallation # Testing against `minimal` or `mainnet` config by default test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n auto --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec # Testing against `minimal` or `mainnet` config by default find_test: pyspec @@ -129,10 +128,10 @@ citest: pyspec mkdir -p $(TEST_REPORT_DIR); ifdef fork . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec else . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec endif diff --git a/docker/README.md b/docker/README.md index 6d5b21e59..4f0c1ecca 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,15 +6,15 @@ This dockerfile sets up the dependencies required to run consensus-spec tests. T Handy commands: - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests -- `docker run $IMAGE_NAME make citest` will run the make citest command inside the docker container +- `docker run $IMAGE_NAME make citest` will run the make citest command inside the docker container Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users. -The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), number of cores, preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies. +The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies. E.g: -- `./build_run_test.sh --p mainnet --n 16` will run the mainnet preset tests with 16 threads +- `./build_run_test.sh --p mainnet` will run the mainnet preset tests - `./build_run_test.sh --a` will run all the tests across all the forks -- `./build_run_test.sh --f deneb --n 16` will only run deneb tests on 16 threads +- `./build_run_test.sh --f deneb` will only run deneb tests Results are always placed in a folder called `./testResults`. The results are `.xml` files and contain the fork they represent and the date/time they were run at. \ No newline at end of file diff --git a/scripts/build_run_docker_tests.sh b/scripts/build_run_docker_tests.sh index 8b20cfae6..9d2740ca2 100755 --- a/scripts/build_run_docker_tests.sh +++ b/scripts/build_run_docker_tests.sh @@ -13,7 +13,6 @@ ALL_EXECUTABLE_SPECS=("phase0" "altair" "bellatrix" "capella" "deneb" "electra" "whisk") TEST_PRESET_TYPE=minimal FORK_TO_TEST=phase0 -NUMBER_OF_CORES=4 WORKDIR="//consensus-specs//tests//core//pyspec" ETH2SPEC_FOLDER_NAME="eth2spec" CONTAINER_NAME="consensus-specs-tests" @@ -21,17 +20,15 @@ DATE=$(date +"%Y%m%d-%H-%M") # Default flag values version=$(git log --pretty=format:'%h' -n 1) IMAGE_NAME="consensus-specs:$version" -number_of_core=4 # displays the available options display_help() { echo "Run 'consensus-specs' tests from a container instance." echo "Be sure to launch Docker before running this script." echo - echo "Syntax: build_run_test.sh [--v TAG | --n NUMBER_OF_CORE | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]" + echo "Syntax: build_run_test.sh [--v TAG | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]" echo " --f Specify the fork to test" echo " --i Specify the docker image to use" - echo " --n Specify the number of cores" echo " --p Specify the test preset type" echo " --a Test all forks" echo " --h Display this help and exit" @@ -63,7 +60,6 @@ while [[ "$#" -gt 0 ]]; do case $1 in --f) FORK_TO_TEST="$2"; shift ;; --v) IMAGE_NAME="$2"; shift ;; - --n) NUMBER_OF_CORES="$2"; shift ;; --p) TEST_PRESET_TYPE="$2"; shift ;; --a) FORK_TO_TEST="all" ;; --h) display_help; exit 0 ;; @@ -90,12 +86,12 @@ fi if [ "$FORK_TO_TEST" == "all" ]; then for fork in "${ALL_EXECUTABLE_SPECS[@]}"; do docker run --name $CONTAINER_NAME $IMAGE_NAME \ - make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES + make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE copy_test_results $fork done else docker run --name $CONTAINER_NAME $IMAGE_NAME \ - make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES + make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE copy_test_results $FORK_TO_TEST fi