separate deposit contract compiler and tester, pin compiler to python 3.7. Upgrade others to 3.8

.circleci/config.yml

@@ -35,32 +35,45 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v7-pyspec
+          venv_name: v9-pyspec
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v7-pyspec
+          venv_name: v9-pyspec
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
           venv_path: ./tests/core/pyspec/venv
-  restore_deposit_contract_cached_venv:
-    description: "Restore the cache with deposit_contract keys"
+  restore_deposit_contract_compiler_cached_venv:
+    description: "Restore the venv from cache for the deposit contract compiler"
     steps:
       - restore_cached_venv:
-          venv_name: v9-deposit-contract
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
-  save_deposit_contract_cached_venv:
-    description: Save a venv into a cache with deposit_contract keys"
+          venv_name: v15-deposit-contract-compiler
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
+  save_deposit_contract_compiler_cached_venv:
+    description: "Save the venv to cache for later use of the deposit contract compiler"
     steps:
       - save_cached_venv:
-          venv_name: v9-deposit-contract
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
-          venv_path: ./deposit_contract/venv
+          venv_name: v15-deposit-contract-compiler
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
+          venv_path: ./deposit_contract/compiler/venv
+  restore_deposit_contract_tester_cached_venv:
+    description: "Restore the venv from cache for the deposit contract tester"
+    steps:
+      - restore_cached_venv:
+          venv_name: v15-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+  save_deposit_contract_tester_cached_venv:
+    description: "Save the venv to cache for later use of the deposit contract tester"
+    steps:
+      - save_cached_venv:
+          venv_name: v15-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+          venv_path: ./deposit_contract/tester/venv
 jobs:
   checkout_specs:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       # Restore git repo at point close to target branch/revision, to speed up checkout
@@ -80,7 +93,7 @@ jobs:
             - ~/specs-repo
   install_pyspec_test:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -92,7 +105,7 @@ jobs:
       - save_pyspec_cached_venv
   test:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -114,7 +127,7 @@ jobs:
           command: sudo npm install -g doctoc && make check_toc
   codespell:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - checkout
@@ -123,7 +136,7 @@ jobs:
           command: pip install codespell --user &&  make codespell
   lint:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -132,29 +145,54 @@ jobs:
       - run:
           name: Run linter
           command: make lint
-  install_deposit_contract_test:
+  install_deposit_contract_compiler:
     docker:
-      - image: circleci/python:3.6
+        # The deposit contract compiler is pinned to python 3.7 because of the vyper version pin.
+        - image: circleci/python:3.7
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_compiler_cached_venv
       - run:
-          name: Install deposit contract requirements
-          command: make install_deposit_contract_test
-      - save_deposit_contract_cached_venv
-  deposit_contract:
+          name: Install deposit contract compiler requirements
+          command: make install_deposit_contract_compiler
+      - save_deposit_contract_compiler_cached_venv
+  install_deposit_contract_tester:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_tester_cached_venv
+      - run:
+          name: Install deposit contract tester requirements
+          command: make install_deposit_contract_tester
+      - save_deposit_contract_tester_cached_venv
+  test_compile_deposit_contract:
+    docker:
+      - image: circleci/python:3.7
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_compiler_cached_venv
+      - run:
+          name: Run deposit contract compile test
+          command: make test_compile_deposit_contract
+  test_deposit_contract:
+    docker:
+      - image: circleci/python:3.8
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_tester_cached_venv
       - run:
           name: Run deposit contract test
           command: make test_deposit_contract
+
 workflows:
   version: 2.1
   test_spec:
@@ -171,9 +209,15 @@ workflows:
       - lint:
           requires:
             - test
-      - install_deposit_contract_test:
+      - install_deposit_contract_compiler:
           requires:
             - checkout_specs
-      - deposit_contract:
+      - test_compile_deposit_contract:
           requires:
-            - install_deposit_contract_test
+            - install_deposit_contract_compiler
+      - install_deposit_contract_tester:
+          requires:
+            - checkout_specs
+      - test_deposit_contract:
+          requires:
+            - install_deposit_contract_tester

Makefile

@@ -5,7 +5,8 @@ TEST_LIBS_DIR = ./tests/core
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
 TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
 GENERATOR_DIR = ./tests/generators
-DEPOSIT_CONTRACT_DIR = ./deposit_contract
+DEPOSIT_CONTRACT_COMPILER_DIR = ./deposit_contract/compiler
+DEPOSIT_CONTRACT_TESTER_DIR = ./deposit_contract/tester
 CONFIGS_DIR = ./configs
 
 # Collect a list of generator names
@@ -35,7 +36,8 @@ COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
 .PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
-        install_deposit_contract_test test_deposit_contract compile_deposit_contract check_toc
+        install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \
+        compile_deposit_contract test_compile_deposit_contract check_toc
 
 all: $(PY_SPEC_ALL_TARGETS)
 
@@ -45,14 +47,16 @@ partial_clean:
 	rm -rf $(GENERATOR_VENVS)
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
 	rm -rf $(PY_SPEC_DIR)/.coverage
 	rm -rf $(PY_SPEC_DIR)/test-reports
 
 clean: partial_clean
 	rm -rf $(PY_SPEC_DIR)/venv
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/venv
 
 # "make generate_tests" to run all generators
 generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)
@@ -66,7 +70,7 @@ test: $(PY_SPEC_ALL_TARGETS)
 	python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \
+	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; export PYTHONPATH="./"; \
 	python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec
 
 open_cov:
@@ -89,17 +93,24 @@ lint: $(PY_SPEC_ALL_TARGETS)
 	&& cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \
 	&& mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1;
 
-install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS)
-	cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt
-
-compile_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
-	python tool/compile_deposit_contract.py contracts/validator_registration.vy;
+install_deposit_contract_tester: $(PY_SPEC_ALL_TARGETS)
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt
 
 test_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \
 	python -m pytest .
 
+install_deposit_contract_compiler:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); python3.7 -m venv venv; . venv/bin/activate; pip3.7 install -r requirements.txt
+
+compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 deposit_contract/compile.py contracts/validator_registration.vy
+
+test_compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 -m pytest .
+
 # "make pyspec" to create the pyspec for all phases.
 pyspec: $(PY_SPEC_ALL_TARGETS)
 

deposit_contract/README.md

@@ -5,7 +5,7 @@
 Under the `eth2.0-specs` directory, execute:
 
 ```sh
-make install_deposit_contract_test
+make install_deposit_contract_tester
 ```
 
 ## How to compile the contract?
@@ -14,11 +14,25 @@ make install_deposit_contract_test
 make compile_deposit_contract
 ```
 
+The compiler dependencies can be installed with:
+
+```sh
+make install_deposit_contract_compiler
+```
+
+Note that this requires python 3.7 to be installed. The pinned vyper version will not work on 3.8.
+
 The ABI and bytecode will be updated at [`contracts/validator_registration.json`](./contracts/validator_registration.json).
 
 
 ## How to run tests?
 
+For running the contract tests:
 ```sh
 make test_deposit_contract
 ```
+
+For testing the compiler output against the expected formally-verified bytecode:
+```sh
+make test_compile_deposit_contract
+```

deposit_contract/compiler/deposit_contract/compile.py

@@ -2,9 +2,7 @@ import argparse
 import json
 import os
 
-from vyper import (
-    compiler,
-)
+from vyper import compiler
 
 DIR = os.path.dirname(__file__)
 

deposit_contract/compiler/deposit_contract/test_compile.py

@@ -0,0 +1,29 @@
+from vyper import compiler
+
+import json
+import os
+
+DIR = os.path.dirname(__file__)
+
+
+def get_deposit_contract_code():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.vy')
+    deposit_contract_code = open(file_path).read()
+    return deposit_contract_code
+
+
+def get_deposit_contract_json():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.json')
+    deposit_contract_json = open(file_path).read()
+    return json.loads(deposit_contract_json)
+
+
+def test_compile_deposit_contract():
+    compiled_deposit_contract_json = get_deposit_contract_json()
+
+    deposit_contract_code = get_deposit_contract_code()
+    abi = compiler.mk_full_signature(deposit_contract_code)
+    bytecode = compiler.compile_code(deposit_contract_code)['bytecode']
+
+    assert abi == compiled_deposit_contract_json["abi"]
+    assert bytecode == compiled_deposit_contract_json["bytecode"]

deposit_contract/compiler/requirements.txt

@@ -0,0 +1,7 @@
+# Vyper beta version used to generate the bytecode that was then formally verified.
+# On top of this beta version, a later change was backported, and included in the formal verification:
+#   https://github.com/vyperlang/vyper/issues/1761
+# The resulting vyper version is pinned and maintained as protected branch.
+git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
+
+pytest==3.6.1

deposit_contract/compiler/setup.py

@@ -0,0 +1,10 @@
+from distutils.core import setup
+
+setup(
+    name='deposit_contract_compiler',
+    packages=['deposit_contract'],
+    package_dir={"": "."},
+    python_requires="3.7",  # pinned vyper compiler stops working after 3.7. See vyper issue 1835.
+    tests_requires=["pytest==3.6.1"],
+    install_requires=[],  # see requirements.txt file
+)

deposit_contract/requirements-testing.txt

@@ -1,5 +0,0 @@
-eth-tester[py-evm]==0.1.0b39
-git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
-web3==5.0.0b2
-pytest==3.6.1
-../tests/core/pyspec

deposit_contract/tester/deposit_contract/conftest.py

@@ -1,8 +1,3 @@
-from random import (
-    randint,
-)
-import re
-
 import pytest
 
 import eth_tester
@@ -10,17 +5,19 @@ from eth_tester import (
     EthereumTester,
     PyEVMBackend,
 )
-from vyper import (
-    compiler,
-)
 from web3 import Web3
-from web3.providers.eth_tester import (
-    EthereumTesterProvider,
-)
-from .utils import (
-    get_deposit_contract_code,
-    get_deposit_contract_json,
-)
+from web3.providers.eth_tester import EthereumTesterProvider
+
+import json
+import os
+
+DIR = os.path.dirname(__file__)
+
+
+def get_deposit_contract_json():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.json')
+    deposit_contract_json = open(file_path).read()
+    return json.loads(deposit_contract_json)
 
 
 # Constants

deposit_contract/tester/deposit_contract/test_deposit.py

@@ -1,23 +1,16 @@
-from random import (
-    randint,
-)
-
+from random import randint
 import pytest
-
 import eth_utils
-from tests.contracts.conftest import (
+
+from eth2spec.phase0.spec import DepositData
+from eth2spec.utils.ssz.ssz_typing import List
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+from deposit_contract.conftest import (
     FULL_DEPOSIT_AMOUNT,
     MIN_DEPOSIT_AMOUNT,
 )
 
-from eth2spec.phase0.spec import (
-    DepositData,
-)
-from eth2spec.utils.ssz.ssz_typing import List
-from eth2spec.utils.ssz.ssz_impl import (
-    hash_tree_root,
-)
-
 
 SAMPLE_PUBKEY = b'\x11' * 48
 SAMPLE_WITHDRAWAL_CREDENTIALS = b'\x22' * 32

deposit_contract/tester/requirements.txt

@@ -0,0 +1,4 @@
+eth-tester[py-evm]>=0.3.0b1,<0.4
+web3==5.4.0
+pytest==3.6.1
+../../tests/core/pyspec

deposit_contract/tester/setup.py

@@ -0,0 +1,9 @@
+from distutils.core import setup
+
+setup(
+    name='deposit_contract_tester',
+    packages=['deposit_contract'],
+    package_dir={"": "."},
+    tests_requires=[],
+    install_requires=[]  # see requirements.txt file
+)

deposit_contract/tests/contracts/test_compile.py

@@ -1,19 +0,0 @@
-from vyper import (
-    compiler,
-)
-
-from .utils import (
-    get_deposit_contract_code,
-    get_deposit_contract_json,
-)
-
-
-def test_compile_deposit_contract():
-    compiled_deposit_contract_json = get_deposit_contract_json()
-
-    deposit_contract_code = get_deposit_contract_code()
-    abi = compiler.mk_full_signature(deposit_contract_code)
-    bytecode = compiler.compile_code(deposit_contract_code)['bytecode']
-
-    assert abi == compiled_deposit_contract_json["abi"]
-    assert bytecode == compiled_deposit_contract_json["bytecode"]

deposit_contract/tests/contracts/utils.py

@@ -1,16 +0,0 @@
-import json
-import os
-
-DIR = os.path.dirname(__file__)
-
-
-def get_deposit_contract_code():
-    file_path = os.path.join(DIR, './../../contracts/validator_registration.vy')
-    deposit_contract_code = open(file_path).read()
-    return deposit_contract_code
-
-
-def get_deposit_contract_json():
-    file_path = os.path.join(DIR, './../../contracts/validator_registration.json')
-    deposit_contract_json = open(file_path).read()
-    return json.loads(deposit_contract_json)

tests/core/pyspec/setup.py

@@ -11,5 +11,6 @@ setup(
         "py_ecc==2.0.0",
         "ssz==0.1.3",
         "dataclasses==0.6",
+        "pytest"
     ]
 )