Merge branch 'dev' into phase1rebase

.circleci/config.yml

@@ -44,19 +44,32 @@ commands:
           venv_name: v9-pyspec
           reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }}
           venv_path: ./tests/core/pyspec/venv
-  restore_deposit_contract_cached_venv:
+  restore_deposit_contract_compiler_cached_venv:
-    description: "Restore the cache with deposit_contract keys"
+    description: "Restore the venv from cache for the deposit contract compiler"
     steps:
       - restore_cached_venv:
-          venv_name: v11-deposit-contract
+          venv_name: v15-deposit-contract-compiler
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
-  save_deposit_contract_cached_venv:
+  save_deposit_contract_compiler_cached_venv:
-    description: Save a venv into a cache with deposit_contract keys"
+    description: "Save the venv to cache for later use of the deposit contract compiler"
     steps:
       - save_cached_venv:
-          venv_name: v11-deposit-contract
+          venv_name: v15-deposit-contract-compiler
-          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
+          reqs_checksum: cache-{{ checksum "deposit_contract/compiler/requirements.txt" }}
-          venv_path: ./deposit_contract/venv
+          venv_path: ./deposit_contract/compiler/venv
+  restore_deposit_contract_tester_cached_venv:
+    description: "Restore the venv from cache for the deposit contract tester"
+    steps:
+      - restore_cached_venv:
+          venv_name: v15-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+  save_deposit_contract_tester_cached_venv:
+    description: "Save the venv to cache for later use of the deposit contract tester"
+    steps:
+      - save_cached_venv:
+          venv_name: v15-deposit-contract-tester
+          reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/tester/requirements.txt" }}
+          venv_path: ./deposit_contract/tester/venv
 jobs:
   checkout_specs:
     docker:
@@ -132,29 +145,54 @@ jobs:
       - run:
           name: Run linter
           command: make lint
-  install_deposit_contract_test:
+  install_deposit_contract_compiler:
     docker:
-      - image: circleci/python:3.6
+        # The deposit contract compiler is pinned to python 3.7 because of the vyper version pin.
+        - image: circleci/python:3.7
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_compiler_cached_venv
       - run:
-          name: Install deposit contract requirements
+          name: Install deposit contract compiler requirements
-          command: make install_deposit_contract_test
+          command: make install_deposit_contract_compiler
-      - save_deposit_contract_cached_venv
+      - save_deposit_contract_compiler_cached_venv
-  deposit_contract:
+  install_deposit_contract_tester:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.8
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_deposit_contract_cached_venv
+      - restore_deposit_contract_tester_cached_venv
+      - run:
+          name: Install deposit contract tester requirements
+          command: make install_deposit_contract_tester
+      - save_deposit_contract_tester_cached_venv
+  test_compile_deposit_contract:
+    docker:
+      - image: circleci/python:3.7
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_compiler_cached_venv
+      - run:
+          name: Run deposit contract compile test
+          command: make test_compile_deposit_contract
+  test_deposit_contract:
+    docker:
+      - image: circleci/python:3.8
+    working_directory: ~/specs-repo
+    steps:
+      - restore_cache:
+          key: v2-specs-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_deposit_contract_tester_cached_venv
       - run:
           name: Run deposit contract test
           command: make test_deposit_contract
+
 workflows:
   version: 2.1
   test_spec:
@@ -171,9 +209,15 @@ workflows:
       - lint:
           requires:
             - test
-      - install_deposit_contract_test:
+      - install_deposit_contract_compiler:
           requires:
             - checkout_specs
-      - deposit_contract:
+      - test_compile_deposit_contract:
           requires:
-            - install_deposit_contract_test
+            - install_deposit_contract_compiler
+      - install_deposit_contract_tester:
+          requires:
+            - checkout_specs
+      - test_deposit_contract:
+          requires:
+            - install_deposit_contract_tester

Makefile

@@ -5,7 +5,8 @@ TEST_LIBS_DIR = ./tests/core
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
 TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
 GENERATOR_DIR = ./tests/generators
-DEPOSIT_CONTRACT_DIR = ./deposit_contract
+DEPOSIT_CONTRACT_COMPILER_DIR = ./deposit_contract/compiler
+DEPOSIT_CONTRACT_TESTER_DIR = ./deposit_contract/tester
 CONFIGS_DIR = ./configs
 
 # Collect a list of generator names
@@ -35,7 +36,8 @@ COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
 .PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
-        install_deposit_contract_test test_deposit_contract compile_deposit_contract check_toc
+        install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \
+        compile_deposit_contract test_compile_deposit_contract check_toc
 
 all: $(PY_SPEC_ALL_TARGETS)
 
@@ -45,14 +47,16 @@ partial_clean:
 	rm -rf $(GENERATOR_VENVS)
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/.pytest_cache
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
 	rm -rf $(PY_SPEC_DIR)/.coverage
 	rm -rf $(PY_SPEC_DIR)/test-reports
 
 clean: partial_clean
 	rm -rf $(PY_SPEC_DIR)/venv
-	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_COMPILER_DIR)/venv
+	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/venv
 
 # "make generate_tests" to run all generators
 generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)
@@ -66,7 +70,7 @@ test: $(PY_SPEC_ALL_TARGETS)
 	python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \
+	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; export PYTHONPATH="./"; \
 	python -m pytest -n 4 --junitxml=test-reports/eth2spec/test_results.xml eth2spec
 
 open_cov:
@@ -89,17 +93,24 @@ lint: $(PY_SPEC_ALL_TARGETS)
 	&& cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \
 	&& mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1;
 
-install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS)
+install_deposit_contract_tester: $(PY_SPEC_ALL_TARGETS)
-	cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt
-
-compile_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
-	python tool/compile_deposit_contract.py contracts/validator_registration.vy;
 
 test_deposit_contract:
-	cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \
+	cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \
 	python -m pytest .
 
+install_deposit_contract_compiler:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); python3.7 -m venv venv; . venv/bin/activate; pip3.7 install -r requirements.txt
+
+compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 deposit_contract/compile.py contracts/validator_registration.vy
+
+test_compile_deposit_contract:
+	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
+	python3.7 -m pytest .
+
 # "make pyspec" to create the pyspec for all phases.
 pyspec: $(PY_SPEC_ALL_TARGETS)
 

deposit_contract/README.md

@@ -5,7 +5,7 @@
 Under the `eth2.0-specs` directory, execute:
 
 ```sh
-make install_deposit_contract_test
+make install_deposit_contract_tester
 ```
 
 ## How to compile the contract?
@@ -14,11 +14,25 @@ make install_deposit_contract_test
 make compile_deposit_contract
 ```
 
+The compiler dependencies can be installed with:
+
+```sh
+make install_deposit_contract_compiler
+```
+
+Note that this requires python 3.7 to be installed. The pinned vyper version will not work on 3.8.
+
 The ABI and bytecode will be updated at [`contracts/validator_registration.json`](./contracts/validator_registration.json).
 
 
 ## How to run tests?
 
+For running the contract tests:
 ```sh
 make test_deposit_contract
 ```
+
+For testing the compiler output against the expected formally-verified bytecode:
+```sh
+make test_compile_deposit_contract
+```

deposit_contract/compiler/deposit_contract/compile.py

@@ -2,9 +2,7 @@ import argparse
 import json
 import os
 
-from vyper import (
+from vyper import compiler
-    compiler,
-)
 
 DIR = os.path.dirname(__file__)
 

deposit_contract/compiler/deposit_contract/test_compile.py

@@ -0,0 +1,29 @@
+from vyper import compiler
+
+import json
+import os
+
+DIR = os.path.dirname(__file__)
+
+
+def get_deposit_contract_code():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.vy')
+    deposit_contract_code = open(file_path).read()
+    return deposit_contract_code
+
+
+def get_deposit_contract_json():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.json')
+    deposit_contract_json = open(file_path).read()
+    return json.loads(deposit_contract_json)
+
+
+def test_compile_deposit_contract():
+    compiled_deposit_contract_json = get_deposit_contract_json()
+
+    deposit_contract_code = get_deposit_contract_code()
+    abi = compiler.mk_full_signature(deposit_contract_code)
+    bytecode = compiler.compile_code(deposit_contract_code)['bytecode']
+
+    assert abi == compiled_deposit_contract_json["abi"]
+    assert bytecode == compiled_deposit_contract_json["bytecode"]

deposit_contract/compiler/requirements.txt

@@ -0,0 +1,7 @@
+# Vyper beta version used to generate the bytecode that was then formally verified.
+# On top of this beta version, a later change was backported, and included in the formal verification:
+#   https://github.com/vyperlang/vyper/issues/1761
+# The resulting vyper version is pinned and maintained as protected branch.
+git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
+
+pytest==3.6.1

deposit_contract/compiler/setup.py

@@ -0,0 +1,10 @@
+from distutils.core import setup
+
+setup(
+    name='deposit_contract_compiler',
+    packages=['deposit_contract'],
+    package_dir={"": "."},
+    python_requires="3.7",  # pinned vyper compiler stops working after 3.7. See vyper issue 1835.
+    tests_requires=["pytest==3.6.1"],
+    install_requires=[],  # see requirements.txt file
+)

deposit_contract/requirements-testing.txt

@@ -1,6 +0,0 @@
-eth-tester[py-evm]==0.1.0b39
-git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
-web3==5.0.0b2
-pytest==3.6.1
-../tests/core/config_helpers
-../tests/core/pyspec

deposit_contract/tester/deposit_contract/conftest.py

@@ -1,8 +1,3 @@
-from random import (
-    randint,
-)
-import re
-
 import pytest
 
 import eth_tester
@@ -10,17 +5,19 @@ from eth_tester import (
     EthereumTester,
     PyEVMBackend,
 )
-from vyper import (
-    compiler,
-)
 from web3 import Web3
-from web3.providers.eth_tester import (
+from web3.providers.eth_tester import EthereumTesterProvider
-    EthereumTesterProvider,
+
-)
+import json
-from .utils import (
+import os
-    get_deposit_contract_code,
+
-    get_deposit_contract_json,
+DIR = os.path.dirname(__file__)
-)
+
+
+def get_deposit_contract_json():
+    file_path = os.path.join(DIR, '../../contracts/validator_registration.json')
+    deposit_contract_json = open(file_path).read()
+    return json.loads(deposit_contract_json)
 
 
 # Constants

deposit_contract/tester/deposit_contract/test_deposit.py

@@ -1,23 +1,16 @@
-from random import (
+from random import randint
-    randint,
-)
-
 import pytest
-
 import eth_utils
-from tests.contracts.conftest import (
+
+from eth2spec.phase0.spec import DepositData
+from eth2spec.utils.ssz.ssz_typing import List
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+from deposit_contract.conftest import (
     FULL_DEPOSIT_AMOUNT,
     MIN_DEPOSIT_AMOUNT,
 )
 
-from eth2spec.phase0.spec import (
-    DepositData,
-)
-from eth2spec.utils.ssz.ssz_typing import List
-from eth2spec.utils.ssz.ssz_impl import (
-    hash_tree_root,
-)
-
 
 SAMPLE_PUBKEY = b'\x11' * 48
 SAMPLE_WITHDRAWAL_CREDENTIALS = b'\x22' * 32

deposit_contract/tester/requirements.txt

@@ -0,0 +1,4 @@
+eth-tester[py-evm]>=0.3.0b1,<0.4
+web3==5.4.0
+pytest==3.6.1
+../../tests/core/pyspec

deposit_contract/tester/setup.py

@@ -0,0 +1,9 @@
+from distutils.core import setup
+
+setup(
+    name='deposit_contract_tester',
+    packages=['deposit_contract'],
+    package_dir={"": "."},
+    tests_requires=[],
+    install_requires=[]  # see requirements.txt file
+)

deposit_contract/tests/contracts/test_compile.py

@@ -1,19 +0,0 @@
-from vyper import (
-    compiler,
-)
-
-from .utils import (
-    get_deposit_contract_code,
-    get_deposit_contract_json,
-)
-
-
-def test_compile_deposit_contract():
-    compiled_deposit_contract_json = get_deposit_contract_json()
-
-    deposit_contract_code = get_deposit_contract_code()
-    abi = compiler.mk_full_signature(deposit_contract_code)
-    bytecode = compiler.compile_code(deposit_contract_code)['bytecode']
-
-    assert abi == compiled_deposit_contract_json["abi"]
-    assert bytecode == compiled_deposit_contract_json["bytecode"]

deposit_contract/tests/contracts/utils.py

@@ -1,16 +0,0 @@
-import json
-import os
-
-DIR = os.path.dirname(__file__)
-
-
-def get_deposit_contract_code():
-    file_path = os.path.join(DIR, './../../contracts/validator_registration.vy')
-    deposit_contract_code = open(file_path).read()
-    return deposit_contract_code
-
-
-def get_deposit_contract_json():
-    file_path = os.path.join(DIR, './../../contracts/validator_registration.json')
-    deposit_contract_json = open(file_path).read()
-    return json.loads(deposit_contract_json)

specs/phase0/beacon-chain.md

@@ -76,7 +76,7 @@
     - [`compute_start_slot_at_epoch`](#compute_start_slot_at_epoch)
     - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch)
     - [`compute_domain`](#compute_domain)
-  - [`compute_signing_root`](#compute_signing_root)
+    - [`compute_signing_root`](#compute_signing_root)
   - [Beacon state accessors](#beacon-state-accessors)
     - [`get_current_epoch`](#get_current_epoch)
     - [`get_previous_epoch`](#get_previous_epoch)
@@ -588,8 +588,8 @@ Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specificati
 - `def Sign(SK: int, message: Bytes) -> BLSSignature`
 - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool`
 - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature`
-- `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool`
+- `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool`
-- `def AggregateVerify(pairs: Sequence[PK: BLSSignature, message: Bytes], signature: BLSSignature) -> bool`
+- `def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool`
 
 Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used.
 
@@ -795,7 +795,7 @@ def compute_domain(domain_type: DomainType, fork_version: Version=GENESIS_FORK_V
     return Domain(domain_type + fork_version)
 ```
 
-### `compute_signing_root`
+#### `compute_signing_root`
 
 ```python
 def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root:

specs/phase0/p2p-interface.md

@@ -80,7 +80,7 @@ It consists of four main sections:
     - [How do we upgrade gossip channels (e.g. changes in encoding, compression)?](#how-do-we-upgrade-gossip-channels-eg-changes-in-encoding-compression)
     - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair)
     - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes)
-  - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id)
+    - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id)
     - [Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets?](#why-is-there-maximum_gossip_clock_disparity-when-validating-slot-ranges-of-messages-in-gossip-subnets)
     - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets)
     - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots)
@@ -759,7 +759,7 @@ No security or privacy guarantees are lost as a result of choosing plaintext top
 
 Furthermore, the Eth2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily.
 
-## Why are we overriding the default libp2p pubsub `message-id`?
+### Why are we overriding the default libp2p pubsub `message-id`?
 
 For our current purposes, there is no need to address messages based on source peer, and it seems likely we might even override the message `from` to obfuscate the peer. By overriding the default `message-id` to use content-addressing we can filter unnecessary duplicates before hitting the application layer.
 

specs/phase0/validator.md

@@ -128,7 +128,7 @@ To submit a deposit:
 
 ### Process deposit
 
-Deposits cannot be processed into the beacon chain until the Eth1 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth1 blocks (~4 hours) plus `ETH1_DATA_VOTING_PERIOD` epochs (~1.7 hours). Once the requisite Eth1 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validators` within an epoch or two. The validator is then in a queue to be activated.
+Deposits cannot be processed into the beacon chain until the Eth1 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth1 blocks (~4 hours) plus `SLOTS_PER_ETH1_VOTING_PERIOD` slots (~1.7 hours). Once the requisite Eth1 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validators` within an epoch or two. The validator is then in a queue to be activated.
 
 ### Validator index
 

tests/core/pyspec/setup.py

@@ -12,5 +12,6 @@ setup(
         "py_ecc==2.0.0",
         "ssz==0.1.3",
         "dataclasses==0.6",
+        "pytest"
     ]
 )