diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 000000000..3e23d1910 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,133 @@ +name: Run spec tests and linter + +defaults: + run: + shell: zsh {0} + +env: + TEST_PRESET_TYPE: "minimal" + DEFAULT_BRANCH: "dev" + +# Run tests on workflow_Dispatch +on: + push: + branches: + - dev + - master + pull_request: + workflow_dispatch: + inputs: + test_preset_type: + default: minimal + description: Type of test to run, either mainnet or minimal + type: string + required: true + commitRef: + description: The branch, tag or SHA to checkout and build from + default: dev + required: true + schedule: + - cron: '0 0 * * *' + +jobs: + precleanup: + runs-on: self-hosted + if: always() + steps: + - name: 'Cleanup build folder' + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + setup-env: + runs-on: self-hosted + needs: precleanup + steps: + - name: Checkout this repo + uses: actions/checkout@v3.2.0 + with: + ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }} + - uses: actions/cache@v3.2.2 + id: cache-git + with: + path: ./* + key: ${{ github.sha }} + + table_of_contents: + runs-on: self-hosted + needs: setup-env + steps: + - uses: actions/cache@v3.2.2 + id: restore-build + with: + path: ./* + key: ${{ github.sha }} + - name: Check table of contents + run: sudo npm install -g doctoc@2 && make check_toc + + codespell: + runs-on: self-hosted + needs: setup-env + steps: + - name: Check codespell + run: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell + + lint: + runs-on: self-hosted + needs: setup-env + steps: + - name: Run linter for pyspec + run: make lint + - name: Run linter for test generators + run: make lint_generators + + pyspec-tests: + runs-on: self-hosted + needs: [setup-env,lint,codespell,table_of_contents] + strategy: + matrix: + version: ["phase0", "altair", "bellatrix", "capella", "eip4844"] + steps: + - uses: actions/cache@v3.2.2 + id: restore-build + with: + path: ./* + key: ${{ github.sha }} + - name: set TEST_PRESET_TYPE + if: github.event.inputs.test_preset_type != '' + run: | + echo "spec_test_preset_type=${{ github.event.inputs.test_preset_type || env.TEST_PRESET_TYPE }}" >> $GITHUB_ENV + - name: set TEST_PRESET_TYPE + if: ${{ (github.event_name == 'push' && github.ref_name != 'master') || github.event_name == 'pull_request' }} + run: | + echo "spec_test_preset_type=${{ env.TEST_PRESET_TYPE}}" >> $GITHUB_ENV + - name: set TEST_PRESET_TYPE + if: ${{ github.event_name == 'push' && github.ref_name == 'master' }} + run: | + echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV + - name: set TEST_PRESET_TYPE + if: github.event.schedule=='0 0 * * *' + run: | + echo "spec_test_preset_type=mainnet" >> $GITHUB_ENV + - name: Install pyspec requirements + run: make install_test + - name: test-${{ matrix.version }} + run: make citest fork=${{ matrix.version }} TEST_PRESET_TYPE=${{env.spec_test_preset_type}} + - uses: actions/upload-artifact@v3 + if: always() + with: + name: test-${{ matrix.version }} + path: tests/core/pyspec/test-reports + + cleanup: + runs-on: self-hosted + needs: [setup-env,pyspec-tests,codespell,lint,table_of_contents] + if: always() + steps: + - name: 'Cleanup build folder' + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ \ No newline at end of file diff --git a/Makefile b/Makefile index 645ed7006..ac769b200 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_cont SOLIDITY_FILE_NAME = deposit_contract.json DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester CONFIGS_DIR = ./configs - +TEST_PRESET_TYPE ?= minimal # Collect a list of generator names GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.))) # Map this list of generator paths to "gen_{generator name}" entries @@ -96,30 +96,30 @@ generate_tests: $(GENERATOR_TARGETS) # "make pyspec" to create the pyspec for all phases. pyspec: - . venv/bin/activate; python3 setup.py pyspecdev + python3 -m venv venv; . venv/bin/activate; python3 setup.py pyspecdev # installs the packages to run pyspec tests install_test: python3 -m venv venv; . venv/bin/activate; python3 -m pip install -e .[lint]; python3 -m pip install -e .[test] -# Testing against `minimal` config by default +# Testing against `minimal` or `mainnet` config by default test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec -# Testing against `minimal` config by default +# Testing against `minimal` or `mainnet` config by default find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.minimal --cov=eth2spec.altair.minimal --cov=eth2spec.bellatrix.minimal --cov=eth2spec.capella.minimal --cov=eth2spec.eip4844.minimal --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.$(TEST_PRESET_TYPE) --cov=eth2spec.altair.$(TEST_PRESET_TYPE) --cov=eth2spec.bellatrix.$(TEST_PRESET_TYPE) --cov=eth2spec.capella.$(TEST_PRESET_TYPE) --cov=eth2spec.eip4844.$(TEST_PRESET_TYPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p $(TEST_REPORT_DIR); ifdef fork . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --bls-type=milagro --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec else . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python3 -m pytest -n 4 --bls-type=milagro --junitxml=test-reports/test_results.xml eth2spec + python3 -m pytest -n 16 --bls-type=milagro --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec endif @@ -135,7 +135,7 @@ check_toc: $(MARKDOWN_FILES:=.toc) rm $*.tmp codespell: - codespell . --skip ./.git -I .codespell-whitelist + codespell . --skip "./.git,./venv,$(PY_SPEC_DIR)/.mypy_cache" -I .codespell-whitelist # TODO: add future protocol upgrade patch packages to linting. # NOTE: we use `pylint` just for catching unused arguments in spec code