diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..792d2420d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/venv +**/.venv \ No newline at end of file diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f32311fa5..c7c43eef5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -66,6 +66,37 @@ jobs: - name: Run linter for test generators run: make lint_generators + dockerfile-test: + runs-on: self-hosted + needs: preclear + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Checkout this repo + uses: actions/checkout@v3.2.0 + - name: get git commit hash + id: git_commit_hash + shell: bash + run: | + echo "git_commit_hash=$(echo $(git log --pretty=format:'%h' -n 1))" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - name: Build and push to local registry + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/Dockerfile + push: true + tags: localhost:5000/consensus-specs-dockerfile-test:${{ steps.git_commit_hash.outputs.git_commit_hash }} + - name: Test the image by running the linter + run: | + docker run localhost:5000/consensus-specs-dockerfile-test:${{ steps.git_commit_hash.outputs.git_commit_hash }} make lint + pyspec-tests: runs-on: self-hosted needs: [preclear,lint,codespell,table_of_contents] diff --git a/.gitignore b/.gitignore index cdfddfb0c..e88207dd7 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ docs/ssz docs/fork_choice docs/README.md site + +# docker test results +testResults diff --git a/Makefile b/Makefile index fc3ae085b..d0d750e89 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ SOLIDITY_FILE_NAME = deposit_contract.json DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester CONFIGS_DIR = ./configs TEST_PRESET_TYPE ?= minimal +NUMBER_OF_CORES=16 # Collect a list of generator names GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.))) # Map this list of generator paths to "gen_{generator name}" entries @@ -128,10 +129,10 @@ citest: pyspec mkdir -p $(TEST_REPORT_DIR); ifdef fork . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 16 --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec else . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 16 --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec endif diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..771309931 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,22 @@ +# Rename the build stage from 'base' to 'builder' for clarification and code readability +FROM python:3.11.0-slim-bullseye as builder + +ENV DEBIAN_FRONTEND=noninteractive \ + WORKDIR=/consensus-specs \ + PIP_UPGRADE_CMD="python -m pip install --upgrade pip" \ + INSTALL_CMD="apt install -y git build-essential" + +RUN mkdir ${WORKDIR} +WORKDIR ${WORKDIR} + +# Chain the commands together +RUN apt update && ${INSTALL_CMD} && ${PIP_UPGRADE_CMD} && rm -rf /var/lib/apt/lists/* + +# Copy the current directory contents into the builder +COPY . . + +# Inline installation commands +RUN make install_test && \ + make preinstallation && \ + make pyspec + diff --git a/requirements_preinstallation.txt b/requirements_preinstallation.txt index 69d9a6660..02f6d2f0b 100644 --- a/requirements_preinstallation.txt +++ b/requirements_preinstallation.txt @@ -1,3 +1,4 @@ pip>=23.1.2 wheel>=0.40.0 setuptools>=68.0.0 +pylint>=3.0.0 \ No newline at end of file diff --git a/scripts/build_run_docker_tests.sh b/scripts/build_run_docker_tests.sh new file mode 100755 index 000000000..368fab94c --- /dev/null +++ b/scripts/build_run_docker_tests.sh @@ -0,0 +1,103 @@ +#! /bin/sh + +# Run 'consensus-specs' tests from a docker container instance. +# *Be sure to launch Docker before running this script.* +# +# It does the below: +# 1. Run pytest for consensus-specs in a container. +# 2. Copy and paste the coverage report. +# 3. Remove all exited containers that use the consensus-specs: images. + + +# Set variables +ALL_EXECUTABLE_SPECS=("phase0" "altair" "bellatrix" "capella" "deneb" "eip6110" "whisk") +TEST_PRESET_TYPE=minimal +FORK_TO_TEST=phase0 +NUMBER_OF_CORES=4 +WORKDIR="//consensus-specs//tests//core//pyspec" +ETH2SPEC_FOLDER_NAME="eth2spec" +CONTAINER_NAME="consensus-specs-tests" +DATE=$(date +"%Y%m%d-%H-%M") +# Default flag values +version=$(git log --pretty=format:'%h' -n 1) +IMAGE_NAME="consensus-specs:$version" +number_of_core=4 + +# displays the available options +display_help() { + echo "Run 'consensus-specs' tests from a container instance." + echo "Be sure to launch Docker before running this script." + echo + echo "Syntax: build_run_test.sh [--v TAG | --n NUMBER_OF_CORE | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]" + echo " --f Specify the fork to test" + echo " --i Specify the docker image to use" + echo " --n Specify the number of cores" + echo " --p Specify the test preset type" + echo " --a Test all forks" + echo " --h Display this help and exit" +} + +# Stop and remove the 'consensus-specs-dockerfile-test' container. +# If this container doesn't exist, then a error message is printed +# (but the process is not stopped). +cleanup() { + echo "Stop and remove the 'consensus-specs-tests' container." + docker stop $CONTAINER_NAME || true && docker rm $CONTAINER_NAME || true + +} + +# Copy the results from the container to a local folder +copy_test_results() { + local fork_name="$1" # Storing the first argument in a variable + + docker cp $CONTAINER_NAME:$WORKDIR/test-reports/test_results.xml ./testResults/test-results-$fork_name-$DATE.xml +} + +# Function to check if the Docker image already exists +image_exists() { + docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "$1" +} + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + --f) FORK_TO_TEST="$2"; shift ;; + --v) IMAGE_NAME="$2"; shift ;; + --n) NUMBER_OF_CORES="$2"; shift ;; + --p) TEST_PRESET_TYPE="$2"; shift ;; + --a) FORK_TO_TEST="all" ;; + --h) display_help; exit 0 ;; + *) echo "Unknown parameter: $1"; display_help; exit 1 ;; + esac + shift +done + +# initialize a test result directory +mkdir -p ./testResults + +# Only clean container after user exit console +trap cleanup SIGINT + +# Build Docker container if it doesn't exist +if ! image_exists "$IMAGE_NAME"; then + echo "Image $IMAGE_NAME does not exist. Building Docker image..." + docker build ../ -t $IMAGE_NAME -f ../docker/Dockerfile +else + echo "Image $IMAGE_NAME already exists. Skipping build..." +fi + +# Equivalent to `make citest with the subsequent flags` +if [ "$FORK_TO_TEST" == "all" ]; then + for fork in "${ALL_EXECUTABLE_SPECS[@]}"; do + docker run --name $CONTAINER_NAME $IMAGE_NAME \ + make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES + copy_test_results $fork + done +else + docker run --name $CONTAINER_NAME $IMAGE_NAME \ + make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES + copy_test_results $FORK_TO_TEST +fi + +# Stop and remove the container +cleanup