Merge pull request #1512 from MrChico/toc_ci

Check table of content of markdown files with CI
2019-12-12 09:47:38 -07:00 · 2019-12-12 09:47:38 -07:00 · 7343baf85a
parent c1dc4a0063 49a8264f5d
commit 7343baf85a
14 changed files with 478 additions and 370 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -103,6 +103,15 @@ jobs:
          command: make citest
      - store_test_results:
          path: test_libs/pyspec/test-reports
+  table_of_contents:
+    docker:
+      - image: circleci/node:10.16.3
+    working_directory: ~/specs-repo
+    steps:
+      - checkout
+      - run:
+          name: Check table of contents
+          command: sudo npm install -g doctoc && make check_toc
  lint:
    docker:
      - image: circleci/python:3.6
@ -148,6 +157,7 @@ workflows:
      - test:
          requires:
            - install_pyspec_test
+      - table_of_contents
      - lint:
          requires:
            - test
--- a/18
+++ b/18
@ -17,18 +17,22 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])

 PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
-PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md
+PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/core/0_*.md)

 PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py
-PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md
+PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/core/1_*.md)
+
+PY_SPEC_ALL_DEPS = $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_1_DEPS)

 PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)

+MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SPEC_DIR)/light_client/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
+
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html

 .PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
-        install_deposit_contract_test test_deposit_contract compile_deposit_contract
+        install_deposit_contract_test test_deposit_contract compile_deposit_contract check_toc

 all: $(PY_SPEC_ALL_TARGETS)

@ -65,6 +69,14 @@ citest: $(PY_SPEC_ALL_TARGETS)
 open_cov:
 	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &

+check_toc: $(MARKDOWN_FILES:=.toc)
+
+%.toc:
+	cp $* $*.tmp && \
+	doctoc $* && \
+	diff -q $* $*.tmp && \
+	rm $*.tmp
+
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \
 	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \
--- a/specs/bls_signature.md
+++ b/specs/bls_signature.md
@ -6,9 +6,10 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [BLS signature verification](#bls-signature-verification)
-    - [Table of contents](#table-of-contents)
 - [Curve parameters](#curve-parameters)
 - [Point representations](#point-representations)
  - [G1 points](#g1-points)
@ -23,6 +24,7 @@
  - [`bls_verify`](#bls_verify)
  - [`bls_verify_multiple`](#bls_verify_multiple)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Curve parameters
--- a/specs/core/0_beacon-chain.md
+++ b/specs/core/0_beacon-chain.md
@ -4,9 +4,10 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 0 -- The Beacon Chain](#ethereum-20-phase-0----the-beacon-chain)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Notation](#notation)
 - [Custom types](#custom-types)
@ -61,8 +62,6 @@
    - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys)
  - [Predicates](#predicates)
    - [`is_active_validator`](#is_active_validator)
-            - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue)
-            - [`is_eligible_for_activation`](#is_eligible_for_activation)
    - [`is_slashable_validator`](#is_slashable_validator)
    - [`is_slashable_attestation_data`](#is_slashable_attestation_data)
    - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation)
@ -119,6 +118,7 @@
      - [Deposits](#deposits)
      - [Voluntary exits](#voluntary-exits)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/core/0_deposit-contract.md
+++ b/specs/core/0_deposit-contract.md
@ -4,9 +4,10 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 0 -- Deposit Contract](#ethereum-20-phase-0----deposit-contract)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Constants](#constants)
  - [Contract](#contract)
@ -17,6 +18,7 @@
    - [`DepositEvent` log](#depositevent-log)
 - [Vyper code](#vyper-code)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/core/0_fork-choice.md
+++ b/specs/core/0_fork-choice.md
@ -4,16 +4,17 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Fork choice](#fork-choice)
+  - [Configuration](#configuration)
  - [Helpers](#helpers)
    - [`LatestMessage`](#latestmessage)
    - [`Store`](#store)
    - [`get_genesis_store`](#get_genesis_store)
-            - [`get_slots_since_genesis`](#get_slots_since_genesis)
    - [`get_current_slot`](#get_current_slot)
    - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start)
    - [`get_ancestor`](#get_ancestor)
@ -27,6 +28,7 @@
    - [`on_block`](#on_block)
    - [`on_attestation`](#on_attestation)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/core/1_beacon-chain-misc.md
+++ b/specs/core/1_beacon-chain-misc.md
@ -3,9 +3,10 @@
 ## Table of contents

 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes)
-    - [Table of contents](#table-of-contents)
 - [Configuration](#configuration)
 - [Containers](#containers)
    - [`CompactCommittee`](#compactcommittee)
@ -26,6 +27,7 @@
  - [Persistent committees](#persistent-committees)
  - [Shard receipt processing](#shard-receipt-processing)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Configuration
--- a/specs/core/1_custody-game.md
+++ b/specs/core/1_custody-game.md
@ -5,9 +5,10 @@
 ## Table of contents

 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 1 -- Custody Game](#ethereum-20-phase-1----custody-game)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Terminology](#terminology)
 - [Constants](#constants)
@ -53,6 +54,7 @@
 - [Per-epoch processing](#per-epoch-processing)
  - [Handling of custody-related deadlines](#handling-of-custody-related-deadlines)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/core/1_shard-data-chains.md
+++ b/specs/core/1_shard-data-chains.md
@ -5,9 +5,10 @@
 ## Table of contents

 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Custom types](#custom-types)
 - [Configuration](#configuration)
@ -44,6 +45,7 @@
    - [Block body](#block-body)
 - [Shard fork choice rule](#shard-fork-choice-rule)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/light_client/merkle_proofs.md
+++ b/specs/light_client/merkle_proofs.md
@ -4,9 +4,10 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Merkle proof formats](#merkle-proof-formats)
-    - [Table of contents](#table-of-contents)
 - [Helper functions](#helper-functions)
 - [Generalized Merkle tree index](#generalized-merkle-tree-index)
 - [SSZ object to index](#ssz-object-to-index)
@ -19,6 +20,7 @@
    - [`generalized_index_parent`](#generalized_index_parent)
 - [Merkle multiproofs](#merkle-multiproofs)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Helper functions
--- a/specs/light_client/sync_protocol.md
+++ b/specs/light_client/sync_protocol.md
@ -5,9 +5,10 @@
 ## Table of contents

 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Minimal Light Client Design](#minimal-light-client-design)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Custom types](#custom-types)
 - [Constants](#constants)
@ -19,6 +20,7 @@
 - [Light client state updates](#light-client-state-updates)
 - [Data overhead](#data-overhead)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction
--- a/specs/networking/p2p-interface.md
+++ b/specs/networking/p2p-interface.md
@ -18,23 +18,88 @@ It consists of four main sections:

 - [Network fundamentals](#network-fundamentals)
  - [Transport](#transport)
+      - [Interop](#interop)
+      - [Mainnet](#mainnet)
  - [Encryption and identification](#encryption-and-identification)
-  - [Protocol negotiation](#protocol-negotiation)
+      - [Interop](#interop-1)
+      - [Mainnet](#mainnet-1)
+  - [Protocol Negotiation](#protocol-negotiation)
+      - [Interop](#interop-2)
+      - [Mainnet](#mainnet-2)
  - [Multiplexing](#multiplexing)
 - [Eth2 network interaction domains](#eth2-network-interaction-domains)
  - [Configuration](#configuration)
  - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
+    - [Topics and messages](#topics-and-messages)
+      - [Global topics](#global-topics)
+      - [Attestation subnets](#attestation-subnets)
+      - [Interop](#interop-3)
+      - [Mainnet](#mainnet-3)
+    - [Encodings](#encodings)
+      - [Interop](#interop-4)
+      - [Mainnet](#mainnet-4)
  - [The Req/Resp domain](#the-reqresp-domain)
+    - [Protocol identification](#protocol-identification)
+    - [Req/Resp interaction](#reqresp-interaction)
+      - [Requesting side](#requesting-side)
+      - [Responding side](#responding-side)
+    - [Encoding strategies](#encoding-strategies)
+      - [SSZ-encoding strategy (with or without Snappy)](#ssz-encoding-strategy-with-or-without-snappy)
+    - [Messages](#messages)
+      - [Status](#status)
+      - [Goodbye](#goodbye)
+      - [BeaconBlocksByRange](#beaconblocksbyrange)
+      - [BeaconBlocksByRoot](#beaconblocksbyroot)
  - [The discovery domain: discv5](#the-discovery-domain-discv5)
+    - [Integration into libp2p stacks](#integration-into-libp2p-stacks)
+    - [ENR structure](#enr-structure)
+      - [Interop](#interop-5)
+      - [Mainnet](#mainnet-5)
+    - [Topic advertisement](#topic-advertisement)
+      - [Interop](#interop-6)
+      - [Mainnet](#mainnet-6)
 - [Design decision rationale](#design-decision-rationale)
  - [Transport](#transport-1)
+    - [Why are we defining specific transports?](#why-are-we-defining-specific-transports)
+    - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec)
+    - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets)
+    - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport)
+    - [Why are we not using QUIC for mainnet from the start?](#why-are-we-not-using-quic-for-mainnet-from-the-start)
  - [Multiplexing](#multiplexing-1)
-  - [Protocol negotiation](#protocol-negotiation-1)
+    - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux)
+  - [Protocol Negotiation](#protocol-negotiation-1)
+    - [When is multiselect 2.0 due and why are we using it for mainnet?](#when-is-multiselect-20-due-and-why-are-we-using-it-for-mainnet)
+    - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation)
  - [Encryption](#encryption)
+    - [Why are we using SecIO for interop? Why not for mainnet?](#why-are-we-using-secio-for-interop-why-not-for-mainnet)
+    - [Why are we using Noise/TLS 1.3 for mainnet?](#why-are-we-using-noisetls-13-for-mainnet)
+    - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all)
+    - [Will mainnnet networking be untested when it launches?](#will-mainnnet-networking-be-untested-when-it-launches)
  - [Gossipsub](#gossipsub)
+    - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation)
+    - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding)
+    - [How do we upgrade gossip channels (e.g. changes in encoding, compression)?](#how-do-we-upgrade-gossip-channels-eg-changes-in-encoding-compression)
+    - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair)
+    - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes)
+    - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets)
+    - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots)
+    - [Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s?](#why-are-aggregate-attestations-broadcast-to-the-global-topic-as-aggregateandproofs-rather-than-just-as-attestations)
+    - [Why are we sending entire objects in the pubsub and not just hashes?](#why-are-we-sending-entire-objects-in-the-pubsub-and-not-just-hashes)
+    - [Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc?](#should-clients-gossip-blocks-if-they-cannot-validate-the-proposer-signature-due-to-not-yet-being-synced-not-knowing-the-head-block-etc)
+    - [How are we going to discover peers in a gossipsub topic?](#how-are-we-going-to-discover-peers-in-a-gossipsub-topic)
  - [Req/Resp](#reqresp)
+    - [Why segregate requests into dedicated protocol IDs?](#why-segregate-requests-into-dedicated-protocol-ids)
+    - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding)
+    - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver)
+    - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc)
  - [Discovery](#discovery)
+    - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht)
+    - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs)
  - [Compression/Encoding](#compressionencoding)
+    - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding)
+    - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers)
+    - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression)
+    - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes)
 - [libp2p implementations matrix](#libp2p-implementations-matrix)

 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@ -567,7 +632,7 @@ Conscious of that, the libp2p community conceptualized [mplex](https://github.co

 Overlay multiplexers are not necessary with QUIC since the protocol provides native multiplexing, but they need to be layered atop TCP, WebSockets, and other transports that lack such support.

-## Protocol negotiation
+## Protocol Negotiation

 ### When is multiselect 2.0 due and why are we using it for mainnet?

--- a/specs/simple-serialize.md
+++ b/specs/simple-serialize.md
@ -4,9 +4,10 @@

 ## Table of contents
 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [SimpleSerialize (SSZ)](#simpleserialize-ssz)
-    - [Table of contents](#table-of-contents)
 - [Constants](#constants)
 - [Typing](#typing)
  - [Basic types](#basic-types)
@ -28,6 +29,7 @@
 - [Summaries and expansions](#summaries-and-expansions)
 - [Implementations](#implementations)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Constants
--- a/specs/validator/0_beacon-chain-validator.md
+++ b/specs/validator/0_beacon-chain-validator.md
@ -5,9 +5,10 @@
 ## Table of contents

 <!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+

- [Ethereum 2.0 Phase 0 -- Honest Validator](#ethereum-20-phase-0----honest-validator)
-    - [Table of contents](#table-of-contents)
 - [Introduction](#introduction)
 - [Prerequisites](#prerequisites)
 - [Constants](#constants)
@ -24,19 +25,20 @@
  - [Lookahead](#lookahead)
 - [Beacon chain responsibilities](#beacon-chain-responsibilities)
  - [Block proposal](#block-proposal)
-            - [Block header](#block-header)
+    - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock)
      - [Slot](#slot)
      - [Parent root](#parent-root)
-                - [State root](#state-root)
+    - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody)
      - [Randao reveal](#randao-reveal)
      - [Eth1 Data](#eth1-data)
-                - [Signature](#signature)
-            - [Block body](#block-body)
      - [Proposer slashings](#proposer-slashings)
      - [Attester slashings](#attester-slashings)
      - [Attestations](#attestations)
      - [Deposits](#deposits)
      - [Voluntary exits](#voluntary-exits)
+    - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock)
+      - [State root](#state-root)
+      - [Signature](#signature)
  - [Attesting](#attesting)
    - [Attestation data](#attestation-data)
      - [General](#general)
@ -60,6 +62,7 @@
  - [Proposer slashing](#proposer-slashing)
  - [Attester slashing](#attester-slashing)

+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->

 ## Introduction