Merge in dev

.circleci/config.yml

@@ -48,13 +48,13 @@ commands:
     description: "Restore the cache with deposit_contract keys"
     steps:
       - restore_cached_venv:
-          venv_name: v6-deposit-contract
+          venv_name: v7-deposit-contract
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
   save_deposit_contract_cached_venv:
     description: Save a venv into a cache with deposit_contract keys"
     steps:
       - save_cached_venv:
-          venv_name: v6-deposit-contract
+          venv_name: v7-deposit-contract
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }}
           venv_path: ./deposit_contract/venv
 jobs:
@@ -112,6 +112,15 @@ jobs:
       - run:
           name: Check table of contents
           command: sudo npm install -g doctoc && make check_toc
+  codespell:
+    docker:
+      - image: circleci/python:3.6
+    working_directory: ~/specs-repo
+    steps:
+      - checkout
+      - run:
+          name: Check codespell
+          command: pip install codespell --user &&  make codespell
   lint:
     docker:
       - image: circleci/python:3.6
@@ -158,6 +167,7 @@ workflows:
           requires:
             - install_pyspec_test
       - table_of_contents
+      - codespell
       - lint:
           requires:
             - test

Makefile

@@ -77,6 +77,9 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 	diff -q $* $*.tmp && \
 	rm $*.tmp
 
+codespell:
+	codespell . --skip ./.git -I .codespell-whitelist
+
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \
 	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \

deposit_contract/contracts/validator_registration.v.py

@@ -1,4 +1,4 @@
-# Vyper target 0.1.0b13
+# Vyper target 0.1.0b13.hotfix1761
 MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000  # Gwei
 DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32
 MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1

deposit_contract/requirements-testing.txt

@@ -1,5 +1,5 @@
 eth-tester[py-evm]==0.1.0b39
-vyper==0.1.0b13
+git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13
 web3==5.0.0b2
 pytest==3.6.1
 ../test_libs/pyspec

scripts/build_spec.py

@@ -156,8 +156,6 @@ def objects_to_spec(functions: Dict[str, str],
             del functions[k]
     functions_spec = '\n\n'.join(functions.values())
     for k in list(constants.keys()):
-        if k.startswith('DOMAIN_'):
-            constants[k] = f"DomainType(bytes.fromhex('{constants[k]}'[2:]))"
         if k == "BLS12_381_Q":
             constants[k] += "  # noqa: E501"
     constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants))
@@ -245,7 +243,7 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]
     return old_objects
 
 
-# inserts are handeled the same way as functions
+# inserts are handled the same way as functions
 combine_inserts = combine_functions
 
 

specs/core/0_beacon-chain.md

@@ -200,7 +200,7 @@ The following values are (non-configurable) constants used throughout the specif
 | - | - |
 | `GENESIS_SLOT` | `Slot(0)` |
 | `GENESIS_EPOCH` | `Epoch(0)` |
-| `BLS_WITHDRAWAL_PREFIX` | `Bytes1(b'\x00')` |
+| `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` |
 
 ### Time parameters
 
@@ -252,11 +252,11 @@ The following values are (non-configurable) constants used throughout the specif
 
 | Name | Value |
 | - | - |
-| `DOMAIN_BEACON_PROPOSER` | `0x00000000` |
+| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` |
-| `DOMAIN_BEACON_ATTESTER` | `0x01000000` |
+| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` |
-| `DOMAIN_RANDAO` | `0x02000000` |
+| `DOMAIN_RANDAO` | `DomainType('0x02000000')` |
-| `DOMAIN_DEPOSIT` | `0x03000000` |
+| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` |
-| `DOMAIN_VOLUNTARY_EXIT` | `0x04000000` |
+| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` |
 
 ## Containers
 
@@ -567,7 +567,7 @@ def int_to_bytes(n: uint64, length: uint64) -> bytes:
 ```python
 def bytes_to_int(data: bytes) -> uint64:
     """
-    Return the integer deserialization of ``data`` intepreted as ``ENDIANNESS``-endian.
+    Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian.
     """
     return int.from_bytes(data, ENDIANNESS)
 ```

specs/core/0_fork-choice.md

@@ -196,7 +196,7 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB
 ```python
 def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]:
     """
-    Retrieve a filtered block true from ``store``, only returning branches
+    Retrieve a filtered block tree from ``store``, only returning branches
     whose leaf state's justified/finalized info agrees with that in ``store``.
     """
     base = store.justified_checkpoint.root

specs/core/1_custody-game.md

@@ -131,7 +131,7 @@ The following types are defined, mapping into `DomainType` (little endian):
 
 | Name | Value |
 | - | - |
-| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `0x06000000` |
+| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `DomainType('0x06000000')` |
 
 ### TODO PLACEHOLDER
 

specs/core/1_shard-data-chains.md

@@ -101,8 +101,8 @@ This document describes the shard transition function (data layer only) and the
 
 | Name | Value |
 | - | - |
-| `DOMAIN_SHARD_PROPOSER` | `0x80000000` |
+| `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` |
-| `DOMAIN_SHARD_ATTESTER` | `0x81000000` |
+| `DOMAIN_SHARD_ATTESTER` | `DomainType('0x81000000')` |
 
 ## Containers
 

specs/networking/p2p-interface.md

@@ -53,10 +53,10 @@ It consists of four main sections:
   - [The discovery domain: discv5](#the-discovery-domain-discv5)
     - [Integration into libp2p stacks](#integration-into-libp2p-stacks)
     - [ENR structure](#enr-structure)
+      - [Attestation subnet bitfield](#attestation-subnet-bitfield)
       - [Interop](#interop-5)
       - [Mainnet](#mainnet-5)
     - [Topic advertisement](#topic-advertisement)
-      - [Interop](#interop-6)
       - [Mainnet](#mainnet-6)
 - [Design decision rationale](#design-decision-rationale)
   - [Transport](#transport-1)
@@ -81,6 +81,7 @@ It consists of four main sections:
     - [How do we upgrade gossip channels (e.g. changes in encoding, compression)?](#how-do-we-upgrade-gossip-channels-eg-changes-in-encoding-compression)
     - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair)
     - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes)
+  - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id)
     - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets)
     - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots)
     - [Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s?](#why-are-aggregate-attestations-broadcast-to-the-global-topic-as-aggregateandproofs-rather-than-just-as-attestations)
@@ -212,6 +213,13 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot
 
 Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. Clients MUST reject (fail validation) messages that are over this size limit. Likewise, clients MUST NOT emit or propagate messages larger than this limit.
 
+The `message-id` of a gossipsub message MUST be:
+
+```python
+   message-id: base64(SHA256(message.data))
+```
+where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc4648#section-3.2) with padding characters omitted.
+
 The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic:
 
 | Topic                                  | Message Type      |
@@ -437,6 +445,8 @@ Clients SHOULD immediately disconnect from one another following the handshake a
 
 Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) SHOULD request beacon blocks from its counterparty via the `BeaconBlocksByRange` request.
 
+*Note*: Under abnormal network condition or after some rounds of `BeaconBlocksByRange` requests, the client might need to send `Status` request again to learn if the peer has a higher head. Implementers are free to implement such behavior in their own way.
+
 #### Goodbye
 
 **Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/``
@@ -557,6 +567,14 @@ The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the follo
 
 Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778).
 
+#### Attestation subnet bitfield
+
+The ENR MAY contain an entry (`attnets`) signifying the attestation subnet bitfield with the following form to more easily discover peers participating in particular attestation gossip subnets.
+
+| Key          | Value                                            |
+|:-------------|:-------------------------------------------------|
+| `attnets`    | SSZ `Bitvector[ATTESTATION_SUBNET_COUNT]`        |
+
 #### Interop
 
 In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth2 capability information, as it would be superfluous.
@@ -569,13 +587,11 @@ On mainnet, ENRs MUST include a structure enumerating the capabilities offered b
 
 ### Topic advertisement
 
-#### Interop
-
-This feature will not be used in the interoperability testnet.
-
 #### Mainnet
 
-In mainnet, we plan to use discv5’s topic advertisement feature as a rendezvous facility for peers on shards (thus subscribing to the relevant gossipsub topics).
+discv5's topic advertisement feature is not expected to be ready for mainnet launch of Phase 0.
+
+Once this feature is built out and stable, we expect to use topic advertisement as a rendezvous facility for peers on shards. Until then, the ENR [attestation subnet bitfield](#attestation-subnet-bitfield) will be used for discovery of peers on particular subnets.
 
 # Design decision rationale
 
@@ -738,6 +754,16 @@ No security or privacy guarantees are lost as a result of choosing plaintext top
 
 Furthermore, the Eth2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily.
 
+## Why are we overriding the default libp2p pubsub `message-id`?
+
+For our current purposes, there is no need to address messages based on source peer, and it seems likely we might even override the message `from` to obfuscate the peer. By overriding the default `message-id` to use content-addressing we can filter unnecessary duplicates before hitting the application layer.
+
+Some examples of where messages could be duplicated:
+
+* A validator client connected to multiple beacon nodes publishing duplicate gossip messages
+* Attestation aggregation strategies where clients partially aggregate attestations and propagate them. Partial aggregates could be duplicated
+* Clients re-publishing seen messages
+
 ### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?
 
 Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). The value is currently set to to be equal `MAX_COMMITTEES_PER_SLOT` until network tests indicate otherwise.
@@ -764,9 +790,9 @@ The prohibition of unverified-block-gossiping extends to nodes that cannot verif
 
 ### How are we going to discover peers in a gossipsub topic?
 
-Via discv5 topics. ENRs should not be used for this purpose, as they store identity, location, and capability information, not volatile [advertisements](#topic-advertisement).
+In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR.
 
-In the interoperability testnet, all peers will be subscribed to all global beacon chain topics, so discovering peers in specific shard topics will be unnecessary.
+Although this method will be sufficient for early phases of Eth2, we aim to use the more appropriate discv5 topics for this and other similar tasks in the future. ENRs should ultimately not be used for this purpose. They are best suited to store identity, location, and capability information, rather than more volatile advertisements.
 
 ## Req/Resp
 

specs/test_formats/ssz_generic/README.md

@@ -150,7 +150,7 @@ Template:
 
 Data:
 
-{container name}: Any of the container names listed below (exluding the `(Container)` python super type)
+{container name}: Any of the container names listed below (excluding the `(Container)` python super type)
 ```
 
 ```python

specs/validator/0_beacon-chain-validator.md

@@ -197,8 +197,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe
 Specifically a validator should:
 * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments.
 * Join the pubsub topic -- `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`.
-    * If any current peers are subscribed to the topic, the validator simply sends `subscribe` messages for the new topic.
+    * For any current peer subscribed to the topic, the validator simply sends a `subscribe` message for the new topic.
-    * If no current peers are subscribed to the topic, the validator must discover new peers on this topic. If "topic discovery" is available, use topic discovery to find peers that advertise subscription to the topic. If not, "guess and check" by connecting with a number of random new peers, persisting connections with peers subscribed to the topic and (potentially) dropping the new peers otherwise.
+    * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`.
 
 ## Beacon chain responsibilities
 
@@ -447,7 +447,11 @@ Where
 
 ## Phase 0 attestation subnet stability
 
-Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets. The lifetime of each random subscription should be a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`.
+Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must:
+
+* Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets
+* Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets
+* Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR
 
 ## How to avoid slashing
 

test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py

@@ -262,7 +262,7 @@ def test_att1_duplicate_index_normal_signed(spec, state):
     indices.pop(1)  # remove an index, make room for the additional duplicate index.
     attester_slashing.attestation_1.attesting_indices = sorted(indices)
 
-    # sign it, the signature will be valid for a single occurence. If the transition accidentally ignores the duplicate.
+    # The signature will be valid for a single occurrence. If the transition accidentally ignores the duplicate.
     sign_indexed_attestation(spec, state, attester_slashing.attestation_1)
 
     indices.append(indices[0])  # add one of the indices a second time
@@ -282,7 +282,7 @@ def test_att2_duplicate_index_normal_signed(spec, state):
     indices.pop(2)  # remove an index, make room for the additional duplicate index.
     attester_slashing.attestation_2.attesting_indices = sorted(indices)
 
-    # sign it, the signature will be valid for a single occurence. If the transition accidentally ignores the duplicate.
+    # The signature will be valid for a single occurrence. If the transition accidentally ignores the duplicate.
     sign_indexed_attestation(spec, state, attester_slashing.attestation_2)
 
     indices.append(indices[1])  # add one of the indices a second time

test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py

@@ -155,7 +155,7 @@ def test_duplicate_attestation(spec, state):
     next_epoch(spec, single_state)
     next_epoch(spec, dup_state)
 
-    # Run non-duplicate inclusion rewards for comparision. Do not yield test vectors
+    # Run non-duplicate inclusion rewards for comparison. Do not yield test vectors
     for _ in run_process_rewards_and_penalties(spec, single_state):
         pass
 

test_libs/pyspec/eth2spec/test/sanity/test_blocks.py

@@ -82,7 +82,7 @@ def test_invalid_state_root(spec, state):
 
     expect_assertion_error(lambda: spec.state_transition(state, signed_block))
 
-    yield 'blocks', [block]
+    yield 'blocks', [signed_block]
     yield 'post', None
 
 
@@ -90,6 +90,8 @@ def test_invalid_state_root(spec, state):
 @spec_state_test
 @always_bls
 def test_zero_block_sig(spec, state):
+    yield 'pre', state
+
     block = build_empty_block_for_next_slot(spec, state)
     invalid_signed_block = spec.SignedBeaconBlock(message=block)
     expect_assertion_error(lambda: spec.state_transition(state, invalid_signed_block))
@@ -102,6 +104,8 @@ def test_zero_block_sig(spec, state):
 @spec_state_test
 @always_bls
 def test_invalid_block_sig(spec, state):
+    yield 'pre', state
+
     block = build_empty_block_for_next_slot(spec, state)
     domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot))
     message = spec.compute_domain_wrapper_root(block, domain)

test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py

@@ -451,10 +451,15 @@ class BaseBytes(bytes, Elements, metaclass=BytesType):
     @classmethod
     def extract_args(cls, *args):
         x = args
-        if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes)):
+        if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, str)):
             x = x[0]
         if isinstance(x, bytes):  # Includes BytesLike
             return x
+        if isinstance(x, str):
+            if x[:2] == '0x':
+                return bytes.fromhex(x[2:])
+            else:
+                return bytes.fromhex(x)
         else:
             return bytes(x)  # E.g. GeneratorType put into bytes.
 

test_libs/pyspec/requirements.txt

@@ -1,6 +1,6 @@
 eth-utils>=1.3.0,<2
 eth-typing>=2.1.0,<3.0.0
-pycryptodome==3.7.3
+pycryptodome==3.9.4
 py_ecc==1.7.1
 dataclasses==0.6
 ssz==0.1.3

test_libs/pyspec/setup.py

@@ -7,7 +7,7 @@ setup(
     install_requires=[
         "eth-utils>=1.3.0,<2",
         "eth-typing>=2.1.0,<3.0.0",
-        "pycryptodome==3.7.3",
+        "pycryptodome==3.9.4",
         "py_ecc==1.7.1",
         "ssz==0.1.3",
         "dataclasses==0.6",