Merge branch 'dev' into start-at-zero

.circleci/config.yml

@@ -1,5 +1,4 @@
-# Python CircleCI 2.0 configuration file
+version: 2.1
-version: 2
 jobs:
   build:
     docker:
@@ -8,34 +7,83 @@ jobs:
 
     steps:
       - checkout
-      # Download and cache dependencies
+      - run:
-      - restore_cache:
+          name: Build pyspec
-          keys:
+          command: make pyspec
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
 
       - run:
-          name: install dependencies
+          name: Run py-tests
-          command: |
+          command: make test
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
-      - run:
-          name: build phase0 spec
-          command: make build/phase0
 
-      - save_cache:
+# TODO see #928: decide on CI triggering of yaml tests building,
-          paths:
+#  and destination of output (new yaml tests LFS-configured repository)
-            - ./venv
+#
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
+#      - run:
-
+#          name: Generate YAML tests
-      - run:
+#          command: make gen_yaml_tests
-          name: run tests
+#
-          command: |
+#      - store_artifacts:
-            . venv/bin/activate
+#          path: test-reports
-            pytest tests
+#          destination: test-reports
-
+#
-      - store_artifacts:
+#      - run:
-          path: test-reports
+#          name: Save YAML tests for deployment
-          destination: test-reports
+#          command: |
+#            mkdir /tmp/workspace
+#            cp -r yaml_tests /tmp/workspace/
+#            git log -1 >> /tmp/workspace/latest_commit_message
+#      - persist_to_workspace:
+#          root: /tmp/workspace
+#          paths:
+#            - yaml_tests
+#            - latest_commit_message
+#  commit:
+#    docker:
+#      - image: circleci/python:3.6
+#    steps:
+#      - attach_workspace:
+#          at: /tmp/workspace
+#      - add_ssh_keys:
+#          fingerprints:
+#            - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa"
+#      - run:
+#          name: Checkout test repository
+#          command: |
+#            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+#            git clone git@github.com:ethereum/eth2.0-tests.git
+#      - run:
+#          name: Commit and push generated YAML tests
+#          command: |
+#            cd eth2.0-tests
+#            git config user.name 'eth2TestGenBot'
+#            git config user.email '47188154+eth2TestGenBot@users.noreply.github.com'
+#            for filename in /tmp/workspace/yaml_tests/*; do
+#              rm -rf $(basename $filename)
+#              cp -r $filename .
+#            done
+#            git add .
+#            if git diff --cached --exit-code >& /dev/null; then
+#              echo "No changes to commit"
+#            else
+#              echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message
+#              cat /tmp/workspace/latest_commit_message >> commit_message
+#              git commit -F commit_message
+#              git push origin master
+#            fi
+#workflows:
+#  version: 2.1
+#
+#  build_and_commit:
+#    jobs:
+#      - build:
+#          filters:
+#            tags:
+#              only: /.*/
+#      - commit:
+#          requires:
+#            - build
+#          filters:
+#            tags:
+#              only: /.*/
+#            branches:
+#              ignore: /.*/

.gitignore

@@ -1,7 +1,15 @@
 *.pyc
 /__pycache__
-/venv
+venv
+.venvs
+.venv
 /.pytest_cache
 
 build/
 output/
+
+yaml_tests/
+.pytest_cache
+
+# Dynamically built from Markdown spec
+test_libs/pyspec/eth2spec/phase0/spec.py

Makefile

@@ -1,29 +1,74 @@
 SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
-BUILD_DIR = ./build
+TEST_LIBS_DIR = ./test_libs
-UTILS_DIR = ./utils
+PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+PY_TEST_DIR = ./py_tests
+YAML_TEST_DIR = ./yaml_tests
+GENERATOR_DIR = ./test_generators
+CONFIGS_DIR = ./configs
+
+# Collect a list of generator names
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
+# Map this list of generator paths to a list of test output paths
+YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
+
+PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
+PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS)
 
 
-.PHONY: clean all test
+.PHONY: clean all test gen_yaml_tests pyspec phase0
-
-
-all: $(BUILD_DIR)/phase0
 
+all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 clean:
-	rm -rf $(BUILD_DIR)
+	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(GENERATOR_VENVS)
+	rm -rf $(PY_TEST_DIR)/venv $(PY_TEST_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_ALL_TARGETS)
 
+# "make gen_yaml_tests" to run generators
+gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
 # runs a limited set of tests against a minimal config
-# run pytest with `-m` option to full suite
+test: $(PY_SPEC_ALL_TARGETS)
-test: all
+	cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config .
-	pytest -m minimal_config tests/
+
+# "make pyspec" to create the pyspec for all phases.
+pyspec: $(PY_SPEC_ALL_TARGETS)
+
+# "make phase0" to create pyspec for phase0
+phase0: $(PY_SPEC_PHASE_0_TARGETS)
 
 
-$(BUILD_DIR)/phase0: $(SPEC_DIR)/core/0_beacon-chain.md $(SCRIPT_DIR)/phase0/*.py $(UTILS_DIR)/phase0/*.py
+$(PY_SPEC_DIR)/eth2spec/phase0/spec.py:
+	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@
+
+
+CURRENT_DIR = ${CURDIR}
+
+# The function that builds a set of suite files, by calling a generator for the given type (param 1)
+define build_yaml_tests
+	$(info running generator $(1))
+	# Create the output
+	mkdir -p $(YAML_TEST_DIR)$(1)
+
+	# 1) Create a virtual environment
+	# 2) Activate the venv, this is where dependencies are installed for the generator
+	# 3) Install all the necessary requirements
+	# 4) Run the generator. The generator is assumed to have an "main.py" file.
+	# 5) We output to the tests dir (generator program should accept a "-o <filepath>" argument.
+	cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR)
+
+	$(info generator $(1) finished)
+endef
+
+# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
+$(YAML_TEST_DIR):
+	$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
 	mkdir -p $@
-	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py
+
-	mkdir -p $@/utils
+# For any target within the tests dir, build it using the build_yaml_tests function.
-	cp $(UTILS_DIR)/phase0/* $@/utils
+# (creation of output dir is a dependency)
-	cp $(UTILS_DIR)/phase0/state_transition.py $@
+$(YAML_TEST_DIR)%: $(YAML_TEST_DIR)
-	touch $@/__init__.py $@/utils/__init__.py
+	$(call build_yaml_tests,$*)

README.md

@@ -6,7 +6,8 @@ To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https:/
 
 This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests.
 
-# Specs
+
+## Specs
 
 Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
 * [Phase 0 -- The Beacon Chain](specs/core/0_beacon-chain.md)
@@ -21,10 +22,21 @@ Accompanying documents can be found in [specs](specs) and include
 * [Merkle proof formats](specs/light_client/merkle_proofs.md)
 * [Light client syncing protocol](specs/light_client/sync_protocol.md)
 
-## Design goals
+
+### Design goals
+
 The following are the broad design goals for Ethereum 2.0:
 * to minimize complexity, even at the cost of some losses in efficiency
 * to remain live through major network partitions and when very large portions of nodes go offline
 * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available
 * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time
 * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain)
+
+
+## For spec contributors
+
+Documentation on the different components used during spec writing can be found here:
+* [YAML Test Generators](test_generators/README.md)
+* [Executable Python Spec](test_libs/pyspec/README.md)
+* [Py-tests](py_tests/README.md)
+

configs/constant_presets/mainnet.yaml

@@ -5,16 +5,16 @@
 
 # Misc
 # ---------------------------------------------------------------
-# 2**10 ` (= 1,024)
+# 2**10 (= 1,024)
 SHARD_COUNT: 1024
-# 2**7 ` (= 128)
+# 2**7 (= 128)
 TARGET_COMMITTEE_SIZE: 128
-# 2**5 ` (= 32)
+# 2**12 (= 4,096)
-MAX_BALANCE_CHURN_QUOTIENT: 32
-# 2**12 ` (= 4,096)
 MAX_ATTESTATION_PARTICIPANTS: 4096
-# 2**2 ` (= 4)
+# 2**2 (= 4)
-MAX_EXIT_DEQUEUES_PER_EPOCH: 4
+MIN_PER_EPOCH_CHURN_LIMIT: 4
+# 2**16 (= 65,536)
+CHURN_LIMIT_QUOTIENT: 65536
 # See issue 563
 SHUFFLE_ROUND_COUNT: 90
 
@@ -22,20 +22,20 @@ SHUFFLE_ROUND_COUNT: 90
 # Deposit contract
 # ---------------------------------------------------------------
 # **TBD**
-DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 DEPOSIT_CONTRACT_TREE_DEPTH: 32
 
 
 # Gwei values
 # ---------------------------------------------------------------
-# 2**0 * 10**9 ` (= 1,000,000,000) Gwei
+# 2**0 * 10**9 (= 1,000,000,000) Gwei
 MIN_DEPOSIT_AMOUNT: 1000000000
-# 2**5 * 10**9 ` (= 32,000,000,000) Gwei
+# 2**5 * 10**9 (= 32,000,000,000) Gwei
 MAX_DEPOSIT_AMOUNT: 32000000000
-# 2**4 * 10**9 ` (= 16,000,000,000) Gwei
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
 EJECTION_BALANCE: 16000000000
-# 2**0 * 10**9 ` (= 1,000,000,000) Gwei
+# 2**0 * 10**9 (= 1,000,000,000) Gwei
 HIGH_BALANCE_INCREMENT: 1000000000
 
 
@@ -54,63 +54,63 @@ BLS_WITHDRAWAL_PREFIX_BYTE: 0x00
 # ---------------------------------------------------------------
 # 6 seconds 6 seconds
 SECONDS_PER_SLOT: 6
-# 2**2 ` (= 4) slots 24 seconds
+# 2**2 (= 4) slots 24 seconds
 MIN_ATTESTATION_INCLUSION_DELAY: 4
-# 2**6 ` (= 64) slots 6.4 minutes
+# 2**6 (= 64) slots 6.4 minutes
 SLOTS_PER_EPOCH: 64
-# 2**0 ` (= 1) epochs 6.4 minutes
+# 2**0 (= 1) epochs 6.4 minutes
 MIN_SEED_LOOKAHEAD: 1
-# 2**2 ` (= 4) epochs 25.6 minutes
+# 2**2 (= 4) epochs 25.6 minutes
 ACTIVATION_EXIT_DELAY: 4
-# 2**4 ` (= 16) epochs ~1.7 hours
+# 2**4 (= 16) epochs ~1.7 hours
 EPOCHS_PER_ETH1_VOTING_PERIOD: 16
-# 2**13 ` (= 8,192) slots ~13 hours
+# 2**13 (= 8,192) slots ~13 hours
 SLOTS_PER_HISTORICAL_ROOT: 8192
-# 2**8 ` (= 256) epochs ~27 hours
+# 2**8 (= 256) epochs ~27 hours
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
-# 2**11 ` (= 2,048) epochs 9 days
+# 2**11 (= 2,048) epochs 9 days
 PERSISTENT_COMMITTEE_PERIOD: 2048
-# 2**6 ` (= 64)
+# 2**6 (= 64) epochs ~7 hours
 MAX_CROSSLINK_EPOCHS: 64
 
 
 # State list lengths
 # ---------------------------------------------------------------
-# 2**13 ` (= 8,192) epochs ~36 days
+# 2**13 (= 8,192) epochs ~36 days
 LATEST_RANDAO_MIXES_LENGTH: 8192
-# 2**13 ` (= 8,192) epochs ~36 days
+# 2**13 (= 8,192) epochs ~36 days
 LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 8192
-# 2**13 ` (= 8,192) epochs ~36 days
+# 2**13 (= 8,192) epochs ~36 days
 LATEST_SLASHED_EXIT_LENGTH: 8192
 
 
 # Reward and penalty quotients
 # ---------------------------------------------------------------
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 BASE_REWARD_QUOTIENT: 32
-# 2**9 ` (= 512)
+# 2**9 (= 512)
 WHISTLEBLOWING_REWARD_QUOTIENT: 512
-# 2**3 ` (= 8)
+# 2**3 (= 8)
 PROPOSER_REWARD_QUOTIENT: 8
-# 2**24 ` (= 16,777,216)
+# 2**24 (= 16,777,216)
 INACTIVITY_PENALTY_QUOTIENT: 16777216
 
 
 # Max operations per block
 # ---------------------------------------------------------------
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 MIN_PENALTY_QUOTIENT: 32
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_PROPOSER_SLASHINGS: 16
-# 2**0 ` (= 1)
+# 2**0 (= 1)
 MAX_ATTESTER_SLASHINGS: 1
-# 2**7 ` (= 128)
+# 2**7 (= 128)
 MAX_ATTESTATIONS: 128
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_DEPOSITS: 16
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_VOLUNTARY_EXITS: 16
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_TRANSFERS: 16
 
 

configs/constant_presets/minimal.yaml

@@ -9,33 +9,33 @@ SHARD_COUNT: 8
 
 # [customized] unsecure, but fast
 TARGET_COMMITTEE_SIZE: 4
-# 2**5 ` (= 32)
+# 2**12 (= 4,096)
-MAX_BALANCE_CHURN_QUOTIENT: 32
-# 2**12 ` (= 4,096)
 MAX_ATTESTATION_PARTICIPANTS: 4096
-# 2**2 ` (= 4)
+# 2**2 (= 4)
-MAX_EXIT_DEQUEUES_PER_EPOCH: 4
+MIN_PER_EPOCH_CHURN_LIMIT: 4
-# See issue 563
+# 2**16 (= 65,536)
-SHUFFLE_ROUND_COUNT: 90
+CHURN_LIMIT_QUOTIENT: 65536
+# [customized] Faster, but unsecure.
+SHUFFLE_ROUND_COUNT: 10
 
 
 # Deposit contract
 # ---------------------------------------------------------------
 # **TBD**
-DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 DEPOSIT_CONTRACT_TREE_DEPTH: 32
 
 
 # Gwei values
 # ---------------------------------------------------------------
-# 2**0 * 10**9 ` (= 1,000,000,000) Gwei
+# 2**0 * 10**9 (= 1,000,000,000) Gwei
 MIN_DEPOSIT_AMOUNT: 1000000000
-# 2**5 * 10**9 ` (= 32,000,000,000) Gwei
+# 2**5 * 10**9 (= 32,000,000,000) Gwei
 MAX_DEPOSIT_AMOUNT: 32000000000
-# 2**4 * 10**9 ` (= 16,000,000,000) Gwei
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
 EJECTION_BALANCE: 16000000000
-# 2**0 * 10**9 ` (= 1,000,000,000) Gwei
+# 2**0 * 10**9 (= 1,000,000,000) Gwei
 HIGH_BALANCE_INCREMENT: 1000000000
 
 
@@ -58,19 +58,19 @@ SECONDS_PER_SLOT: 6
 MIN_ATTESTATION_INCLUSION_DELAY: 2
 # [customized] fast epochs
 SLOTS_PER_EPOCH: 8
-# 2**0 ` (= 1) epochs 6.4 minutes
+# 2**0 (= 1) epochs 6.4 minutes
 MIN_SEED_LOOKAHEAD: 1
-# 2**2 ` (= 4) epochs 25.6 minutes
+# 2**2 (= 4) epochs 25.6 minutes
 ACTIVATION_EXIT_DELAY: 4
-# 2**4 ` (= 16) epochs ~1.7 hours
+# [customized] higher frequency new deposits from eth1 for testing
-EPOCHS_PER_ETH1_VOTING_PERIOD: 16
+EPOCHS_PER_ETH1_VOTING_PERIOD: 2
 # [customized] smaller state
 SLOTS_PER_HISTORICAL_ROOT: 64
-# 2**8 ` (= 256) epochs ~27 hours
+# 2**8 (= 256) epochs ~27 hours
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
-# 2**11 ` (= 2,048) epochs 9 days
+# 2**11 (= 2,048) epochs 9 days
 PERSISTENT_COMMITTEE_PERIOD: 2048
-# 2**6 ` (= 64)
+# 2**6 (= 64) epochs ~7 hours
 MAX_CROSSLINK_EPOCHS: 64
 
 
@@ -86,31 +86,31 @@ LATEST_SLASHED_EXIT_LENGTH: 64
 
 # Reward and penalty quotients
 # ---------------------------------------------------------------
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 BASE_REWARD_QUOTIENT: 32
-# 2**9 ` (= 512)
+# 2**9 (= 512)
 WHISTLEBLOWING_REWARD_QUOTIENT: 512
-# 2**3 ` (= 8)
+# 2**3 (= 8)
 PROPOSER_REWARD_QUOTIENT: 8
-# 2**24 ` (= 16,777,216)
+# 2**24 (= 16,777,216)
 INACTIVITY_PENALTY_QUOTIENT: 16777216
 
 
 # Max operations per block
 # ---------------------------------------------------------------
-# 2**5 ` (= 32)
+# 2**5 (= 32)
 MIN_PENALTY_QUOTIENT: 32
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_PROPOSER_SLASHINGS: 16
-# 2**0 ` (= 1)
+# 2**0 (= 1)
 MAX_ATTESTER_SLASHINGS: 1
-# 2**7 ` (= 128)
+# 2**7 (= 128)
 MAX_ATTESTATIONS: 128
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_DEPOSITS: 16
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_VOLUNTARY_EXITS: 16
-# 2**4 ` (= 16)
+# 2**4 (= 16)
 MAX_TRANSFERS: 16
 
 

configs/fork_timelines/testing.yaml

@@ -0,0 +1,6 @@
+# Testing fork timeline
+
+# Equal to GENESIS_EPOCH
+phase0: 536870912
+
+# No other forks considered in testing yet (to be implemented)

py_tests/README.md

@@ -0,0 +1,30 @@
+# ETH 2.0 py-tests
+
+These tests are not intended for client-consumption.
+These tests are sanity tests, to verify if the spec itself is consistent.
+
+There are ideas to port these tests to the YAML test suite,
+ but we are still looking for inputs on how this should work.
+
+## How to run tests
+
+### Automated
+
+Run `make test` from the root of the spec repository.
+
+### Manual
+
+From within the py_tests folder:
+
+Install dependencies:
+```bash
+python3 -m venv venv
+. venv/bin/activate
+pip3 install -r requirements.txt
+```
+Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement.
+
+Run the tests:
+```
+pytest -m minimal_config .
+```

py_tests/phase0/block_processing/test_process_attestation.py

@@ -1,17 +1,17 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
 
-from build.phase0.state_transition import (
+from eth2spec.phase0.state_transition import (
     state_transition,
 )
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_current_epoch,
     process_attestation,
     slot_to_epoch,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
     get_valid_attestation,
 )

py_tests/phase0/block_processing/test_process_attester_slashing.py

@@ -1,13 +1,13 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_balance,
     get_beacon_proposer_index,
     process_attester_slashing,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     get_valid_attester_slashing,
     next_epoch,
 )
@@ -40,7 +40,7 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True):
         get_balance(post_state, slashed_index) <
         get_balance(state, slashed_index)
     )
-    proposer_index = get_beacon_proposer_index(state, state.slot)
+    proposer_index = get_beacon_proposer_index(state)
     # gained whistleblower reward
     assert (
         get_balance(post_state, proposer_index) >

py_tests/phase0/block_processing/test_process_block_header.py

@@ -2,14 +2,15 @@ from copy import deepcopy
 import pytest
 
 
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_beacon_proposer_index,
     cache_state,
     advance_slot,
     process_block_header,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_empty_block_for_next_slot,
+    next_slot,
 )
 
 # mark entire file as 'header'
@@ -61,8 +62,12 @@ def test_invalid_previous_block_root(state):
 
 
 def test_proposer_slashed(state):
+    # use stub state to get proposer index of next slot
+    stub_state = deepcopy(state)
+    next_slot(stub_state)
+    proposer_index = get_beacon_proposer_index(stub_state)
+
     # set proposer to slashed
-    proposer_index = get_beacon_proposer_index(state, state.slot + 1)
     state.validator_registry[proposer_index].slashed = True
 
     block = build_empty_block_for_next_slot(state)

py_tests/phase0/block_processing/test_process_deposit.py

@@ -1,14 +1,14 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
 
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_balance,
     ZERO_HASH,
     process_deposit,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_deposit,
     privkeys,
     pubkeys,

py_tests/phase0/block_processing/test_process_proposer_slashing.py

@@ -1,13 +1,13 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_balance,
     get_current_epoch,
     process_proposer_slashing,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     get_valid_proposer_slashing,
 )
 

py_tests/phase0/block_processing/test_voluntary_exit.py

@@ -1,15 +1,15 @@
 from copy import deepcopy
 import pytest
 
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
 
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     get_active_validator_indices,
     get_churn_limit,
     get_current_epoch,
     process_voluntary_exit,
 )
-from tests.phase0.helpers import (
+from phase0.helpers import (
     build_voluntary_exit,
     pubkey_to_privkey,
 )

py_tests/phase0/conftest.py

@@ -1,8 +1,8 @@
 import pytest
 
-from build.phase0 import spec
+from eth2spec.phase0 import spec
 
-from tests.phase0.helpers import (
+from .helpers import (
     create_genesis_state,
 )
 

py_tests/phase0/helpers.py

@@ -2,12 +2,12 @@ from copy import deepcopy
 
 from py_ecc import bls
 
-import build.phase0.spec as spec
+from eth2spec.phase0.state_transition import (
-from build.phase0.utils.minimal_ssz import signing_root
-from build.phase0.state_transition import (
     state_transition,
 )
-from build.phase0.spec import (
+import eth2spec.phase0.spec as spec
+from eth2spec.utils.minimal_ssz import signing_root
+from eth2spec.phase0.spec import (
     # constants
     EMPTY_SIGNATURE,
     ZERO_HASH,
@@ -39,7 +39,7 @@ from build.phase0.spec import (
     verify_merkle_branch,
     hash,
 )
-from build.phase0.utils.merkle_minimal import (
+from eth2spec.utils.merkle_minimal import (
     calc_merkle_tree_from_leaves,
     get_merkle_proof,
     get_merkle_root,

py_tests/phase0/test_sanity.py

@@ -3,10 +3,10 @@ from copy import deepcopy
 import pytest
 
 from py_ecc import bls
-import build.phase0.spec as spec
+import eth2spec.phase0.spec as spec
 
-from build.phase0.utils.minimal_ssz import signing_root
+from eth2spec.utils.minimal_ssz import signing_root
-from build.phase0.spec import (
+from eth2spec.phase0.spec import (
     # constants
     EMPTY_SIGNATURE,
     ZERO_HASH,
@@ -28,15 +28,15 @@ from build.phase0.spec import (
     verify_merkle_branch,
     hash,
 )
-from build.phase0.state_transition import (
+from eth2spec.phase0.state_transition import (
     state_transition,
 )
-from build.phase0.utils.merkle_minimal import (
+from eth2spec.utils.merkle_minimal import (
     calc_merkle_tree_from_leaves,
     get_merkle_proof,
     get_merkle_root,
 )
-from tests.phase0.helpers import (
+from .helpers import (
     build_deposit_data,
     build_empty_block_for_next_slot,
     fill_aggregate_attestation,
@@ -216,7 +216,7 @@ def test_attester_slashing(state):
     # lost whistleblower reward
     assert get_balance(test_state, validator_index) < get_balance(state, validator_index)
 
-    proposer_index = get_beacon_proposer_index(test_state, test_state.slot)
+    proposer_index = get_beacon_proposer_index(test_state)
     # gained whistleblower reward
     assert (
         get_balance(test_state, proposer_index) >
@@ -316,6 +316,9 @@ def test_attestation(state):
 
     assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1
 
+    proposer_index = get_beacon_proposer_index(test_state)
+    assert test_state.balances[proposer_index] > state.balances[proposer_index]
+
     #
     # Epoch transition should move to previous_epoch_attestations
     #

py_tests/requirements.txt

@@ -4,3 +4,4 @@ oyaml==0.7
 pycryptodome==3.7.3
 py_ecc>=1.6.0
 pytest>=3.6,<3.7
+../test_libs/pyspec

scripts/phase0/build_spec.py

@@ -2,24 +2,32 @@ import sys
 import function_puller
 
 
-def build_spec(sourcefile, outfile):
+def build_phase0_spec(sourcefile, outfile):
     code_lines = []
-
-    code_lines.append("from build.phase0.utils.minimal_ssz import *")
-    code_lines.append("from build.phase0.utils.bls_stub import *")
-    for i in (1, 2, 3, 4, 8, 32, 48, 96):
-        code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i))
-    code_lines.append("SLOTS_PER_EPOCH = 64")  # stub, will get overwritten by real var
-    code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH")
-
     code_lines.append("""
+    
 from typing import (
     Any,
     Callable,
+    Dict,
     List,
     NewType,
     Tuple,
 )
+from eth2spec.utils.minimal_ssz import *
+from eth2spec.utils.bls_stub import *
+
+
+    """)
+    for i in (1, 2, 3, 4, 8, 32, 48, 96):
+        code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i))
+
+    code_lines.append("""
+# stub, will get overwritten by real var
+SLOTS_PER_EPOCH = 64
+
+
+def slot_to_epoch(x): return x // SLOTS_PER_EPOCH
 
 
 Slot = NewType('Slot', int)  # uint64
@@ -34,12 +42,14 @@ Any = None
 Store = None
     """)
 
-    code_lines += function_puller.get_lines(sourcefile)
+    code_lines += function_puller.get_spec(sourcefile)
 
     code_lines.append("""
 # Monkey patch validator get committee code
 _compute_committee = compute_committee
 committee_cache = {}
+
+
 def compute_committee(validator_indices: List[ValidatorIndex],
                       seed: Bytes32,
                       index: int,
@@ -60,6 +70,8 @@ def compute_committee(validator_indices: List[ValidatorIndex],
 # Monkey patch hash cache
 _hash = hash
 hash_cache = {}
+
+
 def hash(x):
     if x in hash_cache:
         return hash_cache[x]
@@ -67,6 +79,18 @@ def hash(x):
         ret = _hash(x)
         hash_cache[x] = ret
         return ret
+
+# Access to overwrite spec constants based on configuration
+def apply_constants_preset(preset: Dict[str, Any]):
+    global_vars = globals()
+    for k, v in preset.items():
+        global_vars[k] = v
+
+    # Deal with derived constants
+    global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
+    
+    # Initialize SSZ types again, to account for changed lengths
+    init_SSZ_types()
 """)
 
     with open(outfile, 'w') as out:
@@ -75,5 +99,6 @@ def hash(x):
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:
-        print("Error: spec source and outfile must defined")
+        print("Usage: <source phase0> <output phase0 pyspec>")
-    build_spec(sys.argv[1], sys.argv[2])
+    build_phase0_spec(sys.argv[1], sys.argv[2])
+

scripts/phase0/function_puller.py

@@ -1,11 +1,13 @@
 import sys
+from typing import List
 
 
-def get_lines(file_name):
+def get_spec(file_name: str) -> List[str]:
     code_lines = []
     pulling_from = None
     current_name = None
-    processing_typedef = False
+    current_typedef = None
+    type_defs = []
     for linenum, line in enumerate(open(sys.argv[1]).readlines()):
         line = line.rstrip()
         if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
@@ -17,17 +19,26 @@ def get_lines(file_name):
             if pulling_from is None:
                 pulling_from = linenum
             else:
-                if processing_typedef:
+                if current_typedef is not None:
                     assert code_lines[-1] == '}'
                     code_lines[-1] = '})'
+                    current_typedef[-1] = '})'
+                    type_defs.append((current_name, current_typedef))
                 pulling_from = None
-                processing_typedef = False
+                current_typedef = None
         else:
             if pulling_from == linenum and line == '{':
                 code_lines.append('%s = SSZType({' % current_name)
-                processing_typedef = True
+                current_typedef = ['global_vars["%s"] = SSZType({' % current_name]
             elif pulling_from is not None:
+                # Add some whitespace between functions
+                if line[:3] == 'def':
+                    code_lines.append('')
+                    code_lines.append('')
                 code_lines.append(line)
+                # Remember type def lines
+                if current_typedef is not None:
+                    current_typedef.append(line)
             elif pulling_from is None and len(line) > 0 and line[0] == '|':
                 row = line[1:].split('|')
                 if len(row) >= 2:
@@ -42,5 +53,13 @@ def get_lines(file_name):
                         if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789':
                             eligible = False
                     if eligible:
-                        code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890')))
+                        code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')))
+    # Build type-def re-initialization
+    code_lines.append('')
+    code_lines.append('def init_SSZ_types():')
+    code_lines.append('    global_vars = globals()')
+    for ssz_type_name, ssz_type in type_defs:
+        code_lines.append('')
+        for type_line in ssz_type:
+            code_lines.append('    ' + type_line)
     return code_lines

specs/core/0_beacon-chain.md

@@ -984,25 +984,19 @@ def generate_seed(state: BeaconState,
 ### `get_beacon_proposer_index`
 
 ```python
-def get_beacon_proposer_index(state: BeaconState,
+def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex:
-                              slot: Slot) -> ValidatorIndex:
     """
-    Return the beacon proposer index for the ``slot``.
+    Return the beacon proposer index at ``state.slot``.
-    Due to proposer selection being based upon the validator balances during
-    the epoch in question, this can only be run for the current epoch.
     """
     current_epoch = get_current_epoch(state)
     # assert slot_to_epoch(slot) == current_epoch
 
-    first_committee, _ = get_crosslink_committees_at_slot(state, slot)[0]
+    first_committee, _ = get_crosslink_committees_at_slot(state, state.slot)[0]
     i = 0
     while True:
-        rand_byte = hash(
-            generate_seed(state, current_epoch) +
-            int_to_bytes8(i // 32)
-        )[i % 32]
         candidate = first_committee[(current_epoch + i) % len(first_committee)]
-        if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * rand_byte:
+        random_byte = hash(generate_seed(state, current_epoch) + int_to_bytes8(i // 32))[i % 32]
+        if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * random_byte:
             return candidate
         i += 1
 ```
@@ -1051,16 +1045,8 @@ def get_attestation_participants(state: BeaconState,
     Return the sorted participant indices corresponding to ``attestation_data`` and ``bitfield``.
     """
     crosslink_committee = get_crosslink_committee_for_attestation(state, attestation_data)
-
     assert verify_bitfield(bitfield, len(crosslink_committee))
-
+    return sorted([index for i, index in enumerate(crosslink_committee) if get_bitfield_bit(bitfield, i) == 0b1])
-    # Find the participating attesters in the committee
-    participants = []
-    for i, validator_index in enumerate(crosslink_committee):
-        aggregation_bit = get_bitfield_bit(bitfield, i)
-        if aggregation_bit == 0b1:
-            participants.append(validator_index)
-    return sorted(participants)
 ```
 
 ### `int_to_bytes1`, `int_to_bytes2`, ...
@@ -1343,7 +1329,7 @@ def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistlebl
     slashed_balance = get_effective_balance(state, slashed_index)
     state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += slashed_balance
 
-    proposer_index = get_beacon_proposer_index(state, state.slot)
+    proposer_index = get_beacon_proposer_index(state)
     if whistleblower_index is None:
         whistleblower_index = proposer_index
     whistleblowing_reward = slashed_balance // WHISTLEBLOWING_REWARD_QUOTIENT
@@ -1493,9 +1479,9 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit],
         process_deposit(state, deposit)
 
     # Process genesis activations
-    for validator_index in range(len(state.validator_registry)):
+    for index in range(len(state.validator_registry)):
-        if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT:
+        if get_effective_balance(state, index) >= MAX_DEPOSIT_AMOUNT:
-            activate_validator(state, validator_index)
+            activate_validator(state, index)
 
     genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH))
     for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH):
@@ -1546,8 +1532,8 @@ def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock:
         return get_ancestor(store, store.get_parent(block), slot)
 ```
 
-* Let `get_latest_attestation(store: Store, validator_index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `validator_index`. If several such attestations exist, use the one the [validator](#dfn-validator) `v` observed first.
+* Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the [validator](#dfn-validator) `v` observed first.
-* Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, validator_index)`.
+* Let `get_latest_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`.
 * Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` returns the child blocks of the given `block`.
 * Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`.
 * The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count.
@@ -1559,10 +1545,7 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock)
     """
     validators = start_state.validator_registry
     active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot))
-    attestation_targets = [
+    attestation_targets = [(i, get_latest_attestation_target(store, i)) for i in active_validator_indices]
-        (validator_index, get_latest_attestation_target(store, validator_index))
-        for validator_index in active_validator_indices
-    ]
 
     # Use the rounded-balance-with-hysteresis supplied by the protocol for fork
     # choice voting. This reduces the number of recomputations that need to be
@@ -1627,7 +1610,7 @@ The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SL
 
 #### Helper functions
 
-We define some helper functions utilized when processing an epoch transition:
+We define epoch transition helper functions:
 
 ```python
 def get_current_total_balance(state: BeaconState) -> Gwei:
@@ -1701,24 +1684,12 @@ def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple
 ```
 
 ```python
-def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation:
+def get_earliest_attestation(state: BeaconState, attestations: List[PendingAttestation], index: ValidatorIndex) -> PendingAttestation:
     return min([
-        a for a in state.previous_epoch_attestations if
+        a for a in attestations if index in get_attestation_participants(state, a.data, a.aggregation_bitfield)
-        validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield)
     ], key=lambda a: a.inclusion_slot)
 ```
 
-```python
-def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot:
-    return earliest_attestation(state, validator_index).inclusion_slot
-```
-
-```python
-def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int:
-    attestation = earliest_attestation(state, validator_index)
-    return attestation.inclusion_slot - attestation.data.slot
-```
-
 #### Justification
 
 Run the following function:
@@ -1728,47 +1699,42 @@ def update_justification_and_finalization(state: BeaconState) -> None:
     if get_current_epoch(state) == GENESIS_EPOCH:
         return
 
-    new_justified_epoch = state.current_justified_epoch
+    antepenultimate_justified_epoch = state.previous_justified_epoch
-    new_finalized_epoch = state.finalized_epoch
 
-    # Rotate the justification bitfield up one epoch to make room for the current epoch (and limit to 64 bits)
+    # Process justifications
+    state.previous_justified_epoch = state.current_justified_epoch
+    state.previous_justified_root = state.current_justified_root
     state.justification_bitfield = (state.justification_bitfield << 1) % 2**64
-    # If the previous epoch gets justified, fill the second last bit
     previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state))
     if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2:
-        new_justified_epoch = get_current_epoch(state) - 1
+        state.current_justified_epoch = get_previous_epoch(state)
-        state.justification_bitfield |= 2
+        state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch))
-    # If the current epoch gets justified, fill the last bit
+        state.justification_bitfield |= (1 << 1)
     current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state))
     if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2:
-        new_justified_epoch = get_current_epoch(state)
+        state.current_justified_epoch = get_current_epoch(state)
-        state.justification_bitfield |= 1
+        state.current_justified_root = get_block_root(state, get_epoch_start_slot(state.current_justified_epoch))
+        state.justification_bitfield |= (1 << 0)
 
     # Process finalizations
     bitfield = state.justification_bitfield
     current_epoch = get_current_epoch(state)
-    # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source
+    # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source
-    if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3:
+    if (bitfield >> 1) % 8 == 0b111 and antepenultimate_justified_epoch == current_epoch - 3:
-        new_finalized_epoch = state.previous_justified_epoch
+        state.finalized_epoch = antepenultimate_justified_epoch
-    # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source
+        state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch))
-    if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2:
+    # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source
-        new_finalized_epoch = state.previous_justified_epoch
+    if (bitfield >> 1) % 4 == 0b11 and antepenultimate_justified_epoch == current_epoch - 2:
-    # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source
+        state.finalized_epoch = antepenultimate_justified_epoch
-    if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2:
+        state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch))
-        new_finalized_epoch = state.current_justified_epoch
+    # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source
-    # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source
+    if (bitfield >> 0) % 8 == 0b111 and state.previous_justified_root == current_epoch - 2:
-    if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1:
+        state.finalized_epoch = state.previous_justified_root
-        new_finalized_epoch = state.current_justified_epoch
+        state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch))
-
+    # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source
-    # Update state jusification/finality fields
+    if (bitfield >> 0) % 4 == 0b11 and state.previous_justified_root == current_epoch - 1:
-    state.previous_justified_epoch = state.current_justified_epoch
+        state.finalized_epoch = state.previous_justified_root
-    state.previous_justified_root = state.current_justified_root
+        state.finalized_root = get_block_root(state, get_epoch_start_slot(state.finalized_epoch))
-    if new_justified_epoch != state.current_justified_epoch:
-        state.current_justified_epoch = new_justified_epoch
-        state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch))
-    if new_finalized_epoch != state.finalized_epoch:
-        state.finalized_epoch = new_finalized_epoch
-        state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch))
 ```
 
 #### Crosslinks
@@ -1811,14 +1777,19 @@ def maybe_reset_eth1_period(state: BeaconState) -> None:
 First, we define some additional helpers:
 
 ```python
-def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei:
+def get_base_reward_from_total_balance(state: BeaconState, total_balance: Gwei, index: ValidatorIndex) -> Gwei:
-    if get_previous_total_balance(state) == 0:
+    if total_balance == 0:
         return 0
 
-    adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT
+    adjusted_quotient = integer_squareroot(total_balance) // BASE_REWARD_QUOTIENT
     return get_effective_balance(state, index) // adjusted_quotient // 5
 ```
 
+```python
+def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei:
+    return get_base_reward_from_total_balance(state, get_previous_total_balance(state), index)
+```
+
 ```python
 def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei:
     if epochs_since_finality <= 4:
@@ -1859,10 +1830,9 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[
         if index in get_unslashed_attesting_indices(state, state.previous_epoch_attestations):
             rewards[index] += base_reward * total_attesting_balance // total_balance
             # Inclusion speed bonus
-            rewards[index] += (
+            earliest_attestation = get_earliest_attestation(state, state.previous_epoch_attestations, index)
-                base_reward * MIN_ATTESTATION_INCLUSION_DELAY //
+            inclusion_delay = earliest_attestation.inclusion_slot - earliest_attestation.data.slot
-                inclusion_distance(state, index)
+            rewards[index] += base_reward * MIN_ATTESTATION_INCLUSION_DELAY // inclusion_delay
-            )
         else:
             penalties[index] += base_reward
         # Expected FFG target
@@ -1875,10 +1845,6 @@ def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[
             rewards[index] += base_reward * matching_head_balance // total_balance
         else:
             penalties[index] += base_reward
-        # Proposer bonus
-        if index in get_unslashed_attesting_indices(state, state.previous_epoch_attestations):
-            proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index))
-            rewards[proposer_index] += base_reward // PROPOSER_REWARD_QUOTIENT
         # Take away max rewards if we're not finalizing
         if epochs_since_finality > 4:
             penalties[index] += base_reward * 4
@@ -1918,14 +1884,8 @@ def apply_rewards(state: BeaconState) -> None:
     rewards1, penalties1 = get_justification_and_finalization_deltas(state)
     rewards2, penalties2 = get_crosslink_deltas(state)
     for i in range(len(state.validator_registry)):
-        set_balance(
+        increase_balance(state, i, rewards1[i] + rewards2[i])
-            state,
+        decrease_balance(state, i, penalties1[i] + penalties2[i])
-            i,
-            max(
-                0,
-                get_balance(state, i) + rewards1[i] + rewards2[i] - penalties1[i] - penalties2[i],
-            ),
-        )
 ```
 
 #### Balance-driven status transitions
@@ -1941,7 +1901,7 @@ def process_balance_driven_status_transitions(state: BeaconState) -> None:
     for index, validator in enumerate(state.validator_registry):
         balance = get_balance(state, index)
         if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and balance >= MAX_DEPOSIT_AMOUNT:
-            state.activation_eligibility_epoch = get_current_epoch(state)
+            validator.activation_eligibility_epoch = get_current_epoch(state)
 
         if is_active_validator(validator, get_current_epoch(state)) and balance < EJECTION_BALANCE:
             initiate_validator_exit(state, index)
@@ -1954,7 +1914,7 @@ Run the following function:
 ```python
 def update_registry(state: BeaconState) -> None:
     activation_queue = sorted([
-        validator for validator in state.validator_registry if
+        index for index, validator in enumerate(state.validator_registry) if
         validator.activation_eligibility_epoch != FAR_FUTURE_EPOCH and
         validator.activation_epoch >= get_delayed_activation_exit_epoch(state.finalized_epoch)
     ], key=lambda index: state.validator_registry[index].activation_eligibility_epoch)
@@ -2051,7 +2011,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None:
     # Save current block as the new latest block
     state.latest_block_header = get_temporary_block_header(block)
     # Verify proposer is not slashed
-    proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]
+    proposer = state.validator_registry[get_beacon_proposer_index(state)]
     assert not proposer.slashed
     # Verify proposer signature
     assert bls_verify(
@@ -2066,7 +2026,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None:
 
 ```python
 def process_randao(state: BeaconState, block: BeaconBlock) -> None:
-    proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)]
+    proposer = state.validator_registry[get_beacon_proposer_index(state)]
     # Verify that the provided randao value is valid
     assert bls_verify(
         pubkey=proposer.pubkey,
@@ -2213,6 +2173,18 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
         state.previous_epoch_attestations.append(pending_attestation)
 ```
 
+Run `process_proposer_attestation_rewards(state)`.
+
+```python
+def process_proposer_attestation_rewards(state: BeaconState) -> None:
+    proposer_index = get_beacon_proposer_index(state)
+    for pending_attestations in (state.previous_epoch_attestations, state.current_epoch_attestations):
+        for index in get_unslashed_attesting_indices(state, pending_attestations):
+            if get_earliest_attestation(state, pending_attestations, index).inclusion_slot == state.slot:
+                base_reward = get_base_reward_from_total_balance(state, get_current_total_balance(state), index)
+                increase_balance(state, proposer_index, base_reward // PROPOSER_REWARD_QUOTIENT)
+```
+
 ##### Deposits
 
 Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`.
@@ -2274,7 +2246,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None:
             activation_epoch=FAR_FUTURE_EPOCH,
             exit_epoch=FAR_FUTURE_EPOCH,
             withdrawable_epoch=FAR_FUTURE_EPOCH,
-            initiated_exit=False,
             slashed=False,
             high_balance=0
         )
@@ -2337,14 +2308,6 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None:
     """
     # Verify the amount and fee aren't individually too big (for anti-overflow purposes)
     assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee)
-    # Verify that we have enough ETH to send, and that after the transfer the balance will be either
-    # exactly zero or at least MIN_DEPOSIT_AMOUNT
-    assert (
-        get_balance(state, transfer.sender) == transfer.amount + transfer.fee or
-        get_balance(state, transfer.sender) >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT
-    )
-    # No self-transfers (to enforce >= MIN_DEPOSIT_AMOUNT or zero balance invariant)
-    assert transfer.sender != transfer.recipient
     # A transfer is valid in only one slot
     assert state.slot == transfer.slot
     # Only withdrawn or not-yet-deposited accounts can transfer
@@ -2367,7 +2330,10 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None:
     # Process the transfer
     decrease_balance(state, transfer.sender, transfer.amount + transfer.fee)
     increase_balance(state, transfer.recipient, transfer.amount)
-    increase_balance(state, get_beacon_proposer_index(state, state.slot), transfer.fee)
+    increase_balance(state, get_beacon_proposer_index(state), transfer.fee)
+    # Verify balances are not dust
+    assert not (0 < get_balance(state, transfer.sender) < MIN_DEPOSIT_AMOUNT)
+    assert not (0 < get_balance(state, transfer.recipient) < MIN_DEPOSIT_AMOUNT)
 ```
 
 #### State root verification

specs/core/1_custody-game.md

@@ -283,7 +283,7 @@ def process_custody_reveal(state: BeaconState,
         assert is_active_validator(revealer, get_current_epoch(state)) or revealer.exit_epoch > get_current_epoch(state)
         revealer.custody_reveal_index += 1
         revealer.max_reveal_lateness = max(revealer.max_reveal_lateness, current_custody_period - reveal.period)
-        proposer_index = get_beacon_proposer_index(state, state.slot)
+        proposer_index = get_beacon_proposer_index(state)
         increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT)
 
     # Case 2: masked punitive early reveal
@@ -323,7 +323,7 @@ def process_chunk_challenge(state: BeaconState,
     # Add new chunk challenge record
     state.custody_chunk_challenge_records.append(CustodyChunkChallengeRecord(
         challenge_index=state.custody_challenge_index,
-        challenger_index=get_beacon_proposer_index(state, state.slot),
+        challenger_index=get_beacon_proposer_index(state),
         responder_index=challenge.responder_index
         deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE,
         crosslink_data_root=challenge.attestation.data.crosslink_data_root,
@@ -436,7 +436,7 @@ def process_chunk_challenge_response(state: BeaconState,
     # Clear the challenge
     state.custody_chunk_challenge_records.remove(challenge)
     # Reward the proposer
-    proposer_index = get_beacon_proposer_index(state, state.slot)
+    proposer_index = get_beacon_proposer_index(state)
     increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT)
 ```
 

specs/core/1_shard-data-chains.md

@@ -406,4 +406,4 @@ def is_valid_beacon_attestation(shard: Shard,
 
 ## Shard fork choice rule
 
-The fork choice rule for any shard is LMD GHOST using the shard attestations of the persistent committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than a slot.)
+The fork choice rule for any shard is LMD GHOST using the shard attestations of the persistent committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than that slot.)

specs/light_client/sync_protocol.md

@@ -1,6 +1,6 @@
 # Beacon Chain Light Client Syncing
 
-__NOTICE__: This document is a work-in-progress for researchers and implementers. One of the design goals of the eth2 beacon chain is light-client friendlines, both to allow low-resource clients (mobile phones, IoT, etc) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the eth2 beacon chain and other chains.
+__NOTICE__: This document is a work-in-progress for researchers and implementers. One of the design goals of the eth2 beacon chain is light-client friendliness, both to allow low-resource clients (mobile phones, IoT, etc) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the eth2 beacon chain and other chains.
 
 ## Table of Contents
 

specs/simple-serialize.md

@@ -1,4 +1,4 @@
-# SimpleSerialiZe (SSZ)
+# SimpleSerialize (SSZ)
 
 This is a **work in progress** describing typing, serialization and Merkleization of Ethereum 2.0 objects.
 

specs/test_formats/README.md

@@ -17,8 +17,17 @@ This document defines the YAML format and structure used for ETH 2.0 testing.
 
 Ethereum 2.0 uses YAML as the format for all cross client tests. This document describes at a high level the general format to which all test files should conform.
 
+### Test-case formats
+
 The particular formats of specific types of tests (test suites) are defined in separate documents.
 
+Test formats:
+- [`bls`](./bls/README.md)
+- [`operations`](./operations/README.md)
+- [`shuffling`](./shuffling/README.md)
+- [`ssz`](./ssz/README.md)
+- More formats are planned, see tracking issues for CI/testing
+
 ## Glossary
 
 - `generator`: a program that outputs one or more `suite` files.
@@ -66,16 +75,11 @@ There are two types of fork-data:
 The first is neat to have as a separate form: we prevent duplication, and can run with different presets
  (e.g. fork timeline for a minimal local test, for a public testnet, or for mainnet)
 
-The second is still somewhat ambiguous: some tests may want cover multiple forks, and can do so in different ways:
+The second does not affect the result of the tests, it just states what is covered by the tests,
-- run one test, transitioning from one to the other
+ so that the right suites can be executed to see coverage for a certain fork.
-- run the same test for both
+For some types of tests, it may be beneficial to ensure it runs exactly the same, with any given fork "active".
-- run a test for every transition from one fork to the other
+Test-formats can be explicit on the need to repeat a test with different forks being "active",
-- more
+ but generally tests run only once.
-
-There is a common factor here however: the options are exclusive, and give a clear idea on what test suites need to be ran to cover testing for a specific fork.
-The way this list of forks is interpreted, is up to the test-runner:
-State-transition test suites may want to just declare forks that are being covered in the test suite,
- whereas shuffling test suites may want to declare a list of forks to test the shuffling algorithm for individually. 
 
 ### Test completeness
 
@@ -89,14 +93,13 @@ The aim is to provide clients with a well-defined scope of work to run a particu
 ## Test Suite
 
 ```
-title: <required, string, short, one line> -- Display name for the test suite
+title: <string, short, one line> -- Display name for the test suite
-summary: <required, string, average, 1-3 lines> -- Summarizes the test suite
+summary: <string, average, 1-3 lines> -- Summarizes the test suite
-forks_timeline: <required, string, reference to a fork definition file, without extension> -- Used to determine the forking timeline
+forks_timeline: <string, reference to a fork definition file, without extension> -- Used to determine the forking timeline
-forks: <required, list of strings> -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc.
+forks: <list of strings> -- Defines the coverage. Test-runner code may decide to re-run with the different forks "activated", when applicable.
-  - ... <required, string, first the fork name, then the spec version>
+config: <string, reference to a config file, without extension> -- Used to determine which set of constants to run (possibly compile time) with
-config: <required, string, reference to a config file, without extension> -- Used to determine which set of constants to run (possibly compile time) with
+runner: <string, no spaces, python-like naming format> *MUST be consistent with folder structure*
-runner: <required, string, no spaces, python-like naming format> *MUST be consistent with folder structure*
+handler: <string, no spaces, python-like naming format> *MUST be consistent with folder structure*
-handler: <optional, string, no spaces, python-like naming format> *MUST be consistent with folder structure*
 
 test_cases: <list, values being maps defining a test case each>
    ...
@@ -163,8 +166,11 @@ To prevent parsing of hundreds of different YAML files to test a specific test t
 ```
 .                             <--- root of eth2.0 tests repository
 ├── bls                       <--- collection of handler for a specific test-runner, example runner: "bls"
-│   ├── signing          <--- collection of test suites for a specific handler, example handler: "signing". If no handler, use a dummy folder "main"
+│   ├── verify_msg            <--- collection of test suites for a specific handler, example handler: "verify_msg". If no multiple handlers, use a dummy folder (e.g. "core"), and specify that in the yaml.
-│   │   ├── sign_msg.yml <--- an entry list of test suites
+│   │   ├── verify_valid.yml    .
+│   │   ├── special_cases.yml   . a list of test suites
+│   │   ├── domains.yml         .
+│   │   ├── invalid.yml         .
 │   │   ...                   <--- more suite files (optional)
 │   ...                       <--- more handlers
 ...                           <--- more test types

specs/test_formats/bls/README.md

@@ -0,0 +1,15 @@
+# BLS tests
+
+A test type for BLS. Primarily geared towards verifying the *integration* of any BLS library.
+We do not recommend to roll your own crypto, or use an untested BLS library.
+
+The BLS test suite runner has the following handlers:
+
+- [`aggregate_pubkeys`](./aggregate_pubkeys.md)
+- [`aggregate_sigs`](./aggregate_sigs.md)
+- [`msg_hash_g2_compressed`](./msg_hash_g2_compressed.md)
+- [`msg_hash_g2_uncompressed`](./msg_hash_g2_uncompressed.md)
+- [`priv_to_pub`](./priv_to_pub.md)
+- [`sign_msg`](./sign_msg.md)
+
+Note: signature-verification and aggregate-verify test cases are not yet supported.

specs/test_formats/bls/aggregate_pubkeys.md

@@ -0,0 +1,17 @@
+# Test format: BLS pubkey aggregation
+
+A BLS pubkey aggregation combines a series of pubkeys into a single pubkey.
+
+## Test case format
+
+```yaml
+input: List[BLS Pubkey] -- list of input BLS pubkeys
+output: BLS Pubkey -- expected output, single BLS pubkey
+```
+
+`BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`.
+
+
+## Condition
+
+The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`.

specs/test_formats/bls/aggregate_sigs.md

@@ -0,0 +1,17 @@
+# Test format: BLS signature aggregation
+
+A BLS signature aggregation combines a series of signatures into a single signature.
+
+## Test case format
+
+```yaml
+input: List[BLS Signature] -- list of input BLS signatures
+output: BLS Signature -- expected output, single BLS signature
+```
+
+`BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`.
+
+
+## Condition
+
+The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`.

specs/test_formats/bls/msg_hash_g2_compressed.md

@@ -0,0 +1,19 @@
+# Test format: BLS hash-compressed
+
+A BLS compressed-hash to G2.
+
+## Test case format
+
+```yaml
+input: 
+  message: bytes32,
+  domain: bytes -- any number
+output: List[bytes48] -- length of two
+```
+
+All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`
+
+
+## Condition
+
+The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`.

specs/test_formats/bls/msg_hash_g2_uncompressed.md

@@ -0,0 +1,19 @@
+# Test format: BLS hash-uncompressed
+
+A BLS uncompressed-hash to G2. 
+
+## Test case format
+
+```yaml
+input: 
+  message: bytes32,
+  domain: bytes -- any number
+output: List[List[bytes48]] -- 3 lists, each a length of two
+```
+
+All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`
+
+
+## Condition
+
+The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`.

specs/test_formats/bls/priv_to_pub.md

@@ -0,0 +1,17 @@
+# Test format: BLS private key to pubkey
+
+A BLS private key to public key conversion.
+
+## Test case format
+
+```yaml
+input: bytes32 -- the private key
+output: bytes48 -- the public key
+```
+
+All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`
+
+
+## Condition
+
+The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`.

specs/test_formats/bls/sign_msg.md

@@ -0,0 +1,20 @@
+# Test format: BLS sign message
+
+Message signing with BLS should produce a signature. 
+
+## Test case format
+
+```yaml
+input:
+  privkey: bytes32 -- the private key used for signing
+  message: bytes32 -- input message to sign (a hash)
+  domain: bytes    -- BLS domain
+output: bytes96    -- expected signature
+```
+
+All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`
+
+
+## Condition
+
+The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`.

specs/test_formats/operations/README.md

@@ -0,0 +1,10 @@
+# Operations tests
+
+The different kinds of operations ("transactions") are tested individually with test handlers.
+
+The tested operation kinds are:
+- [`deposits`](./deposits.md)
+- More tests are work-in-progress.
+
+
+

specs/test_formats/operations/deposits.md

@@ -0,0 +1,18 @@
+# Test format: Deposit operations
+
+A deposit is a form of an operation (or "transaction"), modifying the state.
+
+## Test case format
+
+```yaml
+description: string  -- description of test case, purely for debugging purposes
+pre: BeaconState     -- state before applying the deposit
+deposit: Deposit     -- the deposit
+post: BeaconState    -- state after applying the deposit. No value if deposit processing is aborted.
+```
+
+## Condition
+
+A `deposits` handler of the `operations` should process these cases, 
+ calling the implementation of the `process_deposit(state, deposit)` functionality described in the spec.
+The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the inputs as invalid.

specs/test_formats/shuffling/README.md

@@ -0,0 +1,32 @@
+# Test format: shuffling
+
+The runner of the Shuffling test type has only one handler: `core`
+
+This does not mean however that testing is limited.
+Clients may take different approaches to shuffling, for optimizing,
+ and supporting advanced lookup behavior back in older history.
+
+For implementers, possible test runners implementing testing can include:
+1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation)
+2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`)
+3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go.
+4) test complete shuffle in reverse (reverse rounds, same as 2)
+
+## Test case format
+
+```yaml
+seed: bytes32
+count: int
+shuffled: List[int]
+```
+
+- The `bytes32` is encoded a string, hexadecimal encoding, prefixed with `0x`.
+- Integers are validator indices. These are `uint64`, but realistically they are not as big.
+
+The `count` specifies the validator registry size. One should compute the shuffling for indices `0, 1, 2, 3, ..., count (exclusive)`.
+Seed is the raw shuffling seed, passed to permute-index (or optimized shuffling approach). 
+
+## Condition
+
+The resulting list should match the expected output `shuffled` after shuffling the implied input, using the given `seed`.
+

specs/test_formats/ssz/README.md

@@ -0,0 +1,15 @@
+# SSZ tests
+
+SSZ has changed throughout the development of ETH 2.0.
+
+## Contents
+
+A minimal but useful series of tests covering `uint` encoding and decoding is provided.
+This is a direct port of the older SSZ `uint` tests (minus outdated test cases).
+
+[uint test format](./uint.md).
+
+Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc.
+The exact uint lengths to support may be redefined in the future.
+
+Extension of the SSZ tests collection is planned, see CI/testing issues for progress tracking.

specs/test_formats/ssz/uint.md

@@ -0,0 +1,19 @@
+# Test format: SSZ uints
+
+SSZ supports encoding of uints up to 32 bytes. These are considered to be basic types.
+
+## Test case format
+
+```yaml
+type: "uintN"      -- string, where N is one of [8, 16, 32, 64, 128, 256]
+valid: bool        -- expected validity of the input data
+value: string      -- string, decimal encoding, to support up to 256 bit integers
+ssz: bytes         -- string, input data, hex encoded, with prefix 0x
+tags: List[string] -- description of test case, in the form of a list of labels
+```
+
+## Condition
+
+Two-way testing can be implemented in the test-runner:
+- Encoding: After encoding the given input number `value`, the output should match `ssz`
+- Decoding: After decoding the given `ssz` bytes, it should match the input number `value` 

specs/validator/0_beacon-chain-validator.md

@@ -369,24 +369,23 @@ def get_committee_assignment(
             return assignment
 ```
 
-A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the epoch of the slot in question and can not reliably be used to predict an epoch in advance.
+A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question and can not reliably be used to predict in advance.
 
 ```python
 def is_proposer_at_slot(state: BeaconState,
                         slot: Slot,
                         validator_index: ValidatorIndex) -> bool:
-    current_epoch = get_current_epoch(state)
+    assert state.slot == slot
-    assert slot_to_epoch(slot) == current_epoch
 
-    return get_beacon_proposer_index(state, slot) == validator_index
+    return get_beacon_proposer_index(state) == validator_index
 ```
 
-_Note_: If a validator is assigned to the 0th slot of an epoch, the validator must run an empty slot transition from the previous epoch into the 0th slot of the epoch to be able to check if they are a proposer at that slot.
+_Note_: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot.
 
 
 ### Lookahead
 
-The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing which must checked during the epoch in question.
+The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing which must checked during the slot in question.
 
 `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in phase 1+).
 

test_generators/README.md

@@ -0,0 +1,170 @@
+# Eth2.0 Test Generators
+
+This directory contains all the generators for YAML tests, consumed by Eth 2.0 client implementations.
+
+Any issues with the generators and/or generated tests should be filed
+ in the repository that hosts the generator outputs, here: [ethereum/eth2.0-tests](https://github.com/ethereum/eth2.0-tests/).
+
+Whenever a release is made, the new tests are automatically built and
+[eth2TestGenBot](https://github.com/eth2TestGenBot) commits the changes to the test repository.
+
+## How to run generators
+
+pre-requisites:
+- Python 3 installed
+- PIP 3
+- GNU make
+
+### Cleaning
+
+This removes the existing virtual environments (`/test_generators/<generator>/venv`), and generated tests (`/yaml_tests/`).
+
+```bash
+make clean 
+```
+
+### Running all test generators
+
+This runs all the generators.
+
+```bash
+make gen_yaml_tests
+```
+
+### Running a single generator
+
+The make file auto-detects generators in the `test_generators/` directory,
+ and provides a tests-gen target for each generator, see example.
+
+```bash
+make ./yaml_tests/shuffling/
+```
+
+## Developing a generator
+
+Simply open up the generator (not all at once) of choice in your favorite IDE/editor, and run:
+
+```bash
+# From the root of the generator directory:
+# Create a virtual environment (any venv/.venv/.venvs is git-ignored)
+python3 -m venv venv
+# Activate the venv, this is where dependencies are installed for the generator
+. venv/bin/activate
+```
+
+Now that you have a virtual environment, write your generator.
+It's recommended to extend the base-generator.
+
+Create a `requirements.txt` in the root of your generator directory:
+```
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec
+```
+The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself, to prevent code duplication and outdated tests.
+Applying configurations to the spec is simple, and enables you to create test suites with different contexts.
+
+Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement.
+
+Install all the necessary requirements (re-run when you add more):
+```bash
+pip3 install -r requirements.txt
+```
+
+And write your initial test generator, extending the base generator:
+
+Write a `main.py` file, here's an example:
+
+```python
+from gen_base import gen_runner, gen_suite, gen_typing
+
+from eth_utils import (
+    to_dict, to_tuple
+)
+
+from preset_loader import loader
+from eth2spec.phase0 import spec
+
+@to_dict
+def example_test_case(v: int):
+    yield "spec_SHARD_COUNT", spec.SHARD_COUNT
+    yield "example", v
+
+
+@to_tuple
+def generate_example_test_cases():
+    for i in range(10):
+        yield example_test_case(i)
+
+
+def example_minimal_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'minimal')
+    spec.apply_constants_preset(presets)
+
+    return ("mini", "core", gen_suite.render_suite(
+        title="example_minimal",
+        summary="Minimal example suite, testing bar.",
+        forks_timeline="testing",
+        forks=["phase0"],
+        config="minimal",
+        handler="main",
+        test_cases=generate_example_test_cases()))
+
+
+def example_mainnet_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'mainnet')
+    spec.apply_constants_preset(presets)
+
+    return ("full", "core", gen_suite.render_suite(
+        title="example_main_net",
+        summary="Main net based example suite.",
+        forks_timeline= "mainnet",
+        forks=["phase0"],
+        config="testing",
+        handler="main",
+        test_cases=generate_example_test_cases()))
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("example", [example_minimal_suite, example_mainnet_suite])
+```
+
+Recommendations:
+- you can have more than just 1 suite creator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])`
+- you can concatenate lists of test cases, if you don't want to split it up in suites, however make sure they could be run with one handler.
+- you can split your suite creators into different python files/packages, good for code organization.
+- use config "minimal" for performance. But also implement a suite with the default config where necessary. 
+- you may be able to write your test suite creator in a way where it does not make assumptions on constants.
+  If so, you can generate test suites with different configurations for the same scenario (see example). 
+- the test-generator accepts `--output` and `--force` (overwrite output)
+
+## How to add a new test generator
+
+In order to add a new test generator that builds `New Tests`:
+
+1. Create a new directory `new_tests`, within the `test_generators` directory.
+ Note that `new_tests` is also the name of the directory in which the tests will appear in the tests repository later.
+2. Your generator is assumed to have a `requirements.txt` file,
+ with any dependencies it may need. Leave it empty if your generator has none.
+3. Your generator is assumed to have a `main.py` file in its root.
+ By adding the base generator to your requirements, you can make a generator really easily. See docs below.
+4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`.
+ The base generator helps you handle this; you only have to define suite headers,
+ and a list of tests for each suite you generate.
+5. Finally, add any linting or testing commands to the
+ [circleci config file](https://github.com/ethereum/eth2.0-test-generators/blob/master/.circleci/config.yml)
+ if desired to increase code quality.
+
+Note: you do not have to change the makefile.
+However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case.
+Do note that generators should be easy to maintain, lean, and based on the spec.
+
+
+## How to remove a test generator
+
+If a test generator is not needed anymore, undo the steps described above and make a new release:
+
+1. remove the generator directory
+2. remove the generated tests in the `eth2.0-tests` repository by opening a PR there.
+3. make a new release

test_generators/bls/README.md

@@ -0,0 +1,21 @@
+# BLS Test Generator
+
+Explanation of BLS12-381 type hierarchy
+The base unit is bytes48 of which only 381 bits are used
+
+- FQ: uint381 modulo field modulus
+- FQ2: (FQ, FQ)
+- G2: (FQ2, FQ2, FQ2)
+
+## Resources
+
+- [Eth2.0 spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/bls_signature.md)
+- [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf)
+- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone 
+- [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381)
+- [Trinity implementation](https://github.com/ethereum/trinity/blob/master/eth2/_utils/bls.py)
+
+## Comments
+
+Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set).
+Also note that pubkeys and privkeys are reversed.

test_generators/bls/main.py

@@ -0,0 +1,243 @@
+"""
+BLS test vectors generator
+"""
+
+from typing import Tuple
+
+from eth_utils import (
+    to_tuple, int_to_big_endian
+)
+from gen_base import gen_runner, gen_suite, gen_typing
+
+from py_ecc import bls
+
+
+def int_to_hex(n: int) -> str:
+    return '0x' + int_to_big_endian(n).hex()
+
+
+def hex_to_int(x: str) -> int:
+    return int(x, 16)
+
+
+# Note: even though a domain is only an uint64,
+# To avoid issues with YAML parsers that are limited to 53-bit (JS language limit)
+# It is serialized as an hex string as well.
+DOMAINS = [
+    0,
+    1,
+    1234,
+    2**32-1,
+    2**64-1
+]
+
+MESSAGES = [
+    bytes(b'\x00' * 32),
+    bytes(b'\x56' * 32),
+    bytes(b'\xab' * 32),
+]
+
+PRIVKEYS = [
+    # Curve order is 256 so private keys are 32 bytes at most.
+    # Also not all integers is a valid private key, so using pre-generated keys
+    hex_to_int('0x00000000000000000000000000000000263dbd792f5b1be47ed85f8938c0f29586af0d3ac7b977f21c278fe1462040e3'),
+    hex_to_int('0x0000000000000000000000000000000047b8192d77bf871b62e87859d653922725724a5c031afeabc60bcef5ff665138'),
+    hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'),
+]
+
+
+def hash_message(msg: bytes,
+                 domain: int) ->Tuple[Tuple[str, str], Tuple[str, str], Tuple[str, str]]:
+    """
+    Hash message
+    Input:
+        - Message as bytes
+        - domain as uint64
+    Output:
+        - Message hash as a G2 point
+    """
+    return [
+        [
+            int_to_hex(fq2.coeffs[0]),
+            int_to_hex(fq2.coeffs[1]),
+        ]
+        for fq2 in bls.utils.hash_to_G2(msg, domain)
+    ]
+
+
+def hash_message_compressed(msg: bytes, domain: int) -> Tuple[str, str]:
+    """
+    Hash message
+    Input:
+        - Message as bytes
+        - domain as uint64
+    Output:
+        - Message hash as a compressed G2 point
+    """
+    z1, z2 = bls.utils.compress_G2(bls.utils.hash_to_G2(msg, domain))
+    return [int_to_hex(z1), int_to_hex(z2)]
+
+
+
+@to_tuple
+def case01_message_hash_G2_uncompressed():
+    for msg in MESSAGES:
+        for domain in DOMAINS:
+            yield {
+                'input': {
+                    'message': '0x' + msg.hex(),
+                    'domain': int_to_hex(domain)
+                },
+                'output': hash_message(msg, domain)
+            }
+
+@to_tuple
+def case02_message_hash_G2_compressed():
+    for msg in MESSAGES:
+        for domain in DOMAINS:
+            yield {
+                'input': {
+                    'message': '0x' + msg.hex(),
+                    'domain': int_to_hex(domain)
+                },
+                'output': hash_message_compressed(msg, domain)
+            }
+
+@to_tuple
+def case03_private_to_public_key():
+    pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS]
+    pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys]
+    for privkey, pubkey_serial in zip(PRIVKEYS, pubkeys_serial):
+        yield {
+            'input': int_to_hex(privkey),
+            'output': pubkey_serial,
+        }
+
+@to_tuple
+def case04_sign_messages():
+    for privkey in PRIVKEYS:
+        for message in MESSAGES:
+            for domain in DOMAINS:
+                sig = bls.sign(message, privkey, domain)
+                yield {
+                    'input': {
+                        'privkey': int_to_hex(privkey),
+                        'message': '0x' + message.hex(),
+                        'domain': int_to_hex(domain)
+                    },
+                    'output': '0x' + sig.hex()
+                }
+
+# TODO: case05_verify_messages: Verify messages signed in case04
+# It takes too long, empty for now
+
+
+@to_tuple
+def case06_aggregate_sigs():
+    for domain in DOMAINS:
+        for message in MESSAGES:
+            sigs = [bls.sign(message, privkey, domain) for privkey in PRIVKEYS]
+            yield {
+                'input': ['0x' + sig.hex() for sig in sigs],
+                'output': '0x' + bls.aggregate_signatures(sigs).hex(),
+            }
+
+@to_tuple
+def case07_aggregate_pubkeys():
+    pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS]
+    pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys]
+    yield {
+        'input': pubkeys_serial,
+        'output': '0x' + bls.aggregate_pubkeys(pubkeys).hex(),
+    }
+
+
+# TODO
+# Aggregate verify
+
+# TODO
+# Proof-of-possession
+
+
+def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("g2_uncompressed", "msg_hash_g2_uncompressed", gen_suite.render_suite(
+        title="BLS G2 Uncompressed msg hash",
+        summary="BLS G2 Uncompressed msg hash",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="msg_hash_uncompressed",
+        test_cases=case01_message_hash_G2_uncompressed()))
+
+
+def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("g2_compressed", "msg_hash_g2_compressed", gen_suite.render_suite(
+        title="BLS G2 Compressed msg hash",
+        summary="BLS G2 Compressed msg hash",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="msg_hash_compressed",
+        test_cases=case02_message_hash_G2_compressed()))
+
+
+
+def bls_priv_to_pub_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("priv_to_pub", "priv_to_pub", gen_suite.render_suite(
+        title="BLS private key to pubkey",
+        summary="BLS Convert private key to public key",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="priv_to_pub",
+        test_cases=case03_private_to_public_key()))
+
+
+def bls_sign_msg_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("sign_msg", "sign_msg", gen_suite.render_suite(
+        title="BLS sign msg",
+        summary="BLS Sign a message",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="sign_msg",
+        test_cases=case04_sign_messages()))
+
+
+def bls_aggregate_sigs_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("aggregate_sigs", "aggregate_sigs", gen_suite.render_suite(
+        title="BLS aggregate sigs",
+        summary="BLS Aggregate signatures",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="aggregate_sigs",
+        test_cases=case06_aggregate_sigs()))
+
+
+def bls_aggregate_pubkeys_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("aggregate_pubkeys", "aggregate_pubkeys", gen_suite.render_suite(
+        title="BLS aggregate pubkeys",
+        summary="BLS Aggregate public keys",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="bls",
+        handler="aggregate_pubkeys",
+        test_cases=case07_aggregate_pubkeys()))
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("bls", [
+        bls_msg_hash_compressed_suite,
+        bls_msg_hash_uncompressed_suite,
+        bls_priv_to_pub_suite,
+        bls_sign_msg_suite,
+        bls_aggregate_sigs_suite,
+        bls_aggregate_pubkeys_suite
+    ])

test_generators/bls/requirements.txt

@@ -0,0 +1,3 @@
+py-ecc==1.6.0
+eth-utils==1.4.1
+../../test_libs/gen_helpers

test_generators/operations/README.md

@@ -0,0 +1,13 @@
+# Operations
+
+Operations (or "transactions" in previous spec iterations),
+ are atomic changes to the state, introduced by embedding in blocks.
+
+This generator provides a series of test suites, divided into handler, for each operation type.
+An operation test-runner can consume these operation test-suites,
+ and handle different kinds of operations by processing the cases using the specified test handler.
+
+Information on the format of the tests can be found in the [operations test formats documentation](../../specs/test_formats/operations/README.md).
+
+ 
+

test_generators/operations/deposits.py

@@ -0,0 +1,181 @@
+from eth2spec.phase0 import spec
+from eth_utils import (
+    to_dict, to_tuple
+)
+from gen_base import gen_suite, gen_typing
+from preset_loader import loader
+from eth2spec.debug.encode import encode
+from eth2spec.utils.minimal_ssz import signing_root
+from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof
+
+from typing import List, Tuple
+
+import genesis
+import keys
+from py_ecc import bls
+
+
+def build_deposit_data(state,
+                       pubkey: spec.BLSPubkey,
+                       withdrawal_cred: spec.Bytes32,
+                       privkey: int,
+                       amount: int):
+    deposit_data = spec.DepositData(
+        pubkey=pubkey,
+        withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:],
+        amount=amount,
+        proof_of_possession=spec.EMPTY_SIGNATURE,
+    )
+    deposit_data.proof_of_possession = bls.sign(
+        message_hash=signing_root(deposit_data),
+        privkey=privkey,
+        domain=spec.get_domain(
+            state.fork,
+            spec.get_current_epoch(state),
+            spec.DOMAIN_DEPOSIT,
+        )
+    )
+    return deposit_data
+
+
+def build_deposit(state,
+                  deposit_data_leaves: List[spec.Bytes32],
+                  pubkey: spec.BLSPubkey,
+                  withdrawal_cred: spec.Bytes32,
+                  privkey: int,
+                  amount: int) -> spec.Deposit:
+
+    deposit_data = build_deposit_data(state, pubkey, withdrawal_cred, privkey, amount)
+
+    item = spec.hash(deposit_data.serialize())
+    index = len(deposit_data_leaves)
+    deposit_data_leaves.append(item)
+    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
+    proof = list(get_merkle_proof(tree, item_index=index))
+
+    deposit = spec.Deposit(
+        proof=list(proof),
+        index=index,
+        data=deposit_data,
+    )
+    assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, get_merkle_root(tuple(deposit_data_leaves)))
+
+    return deposit
+
+
+def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState]:
+    genesis_deposits = genesis.create_deposits(
+        keys.pubkeys[:initial_validator_count],
+        keys.withdrawal_creds[:initial_validator_count]
+    )
+    state = genesis.create_genesis_state(genesis_deposits)
+
+    deposit_data_leaves = [spec.hash(dep.data.serialize()) for dep in genesis_deposits]
+
+    deposit = build_deposit(
+        state,
+        deposit_data_leaves,
+        keys.pubkeys[index],
+        keys.withdrawal_creds[index],
+        keys.privkeys[index],
+        spec.MAX_DEPOSIT_AMOUNT,
+    )
+
+    state.latest_eth1_data.deposit_root = get_merkle_root(tuple(deposit_data_leaves))
+    state.latest_eth1_data.deposit_count = len(deposit_data_leaves)
+
+    return deposit, state
+
+
+@to_dict
+def valid_deposit():
+    new_dep, state = build_deposit_for_index(10, 10)
+    yield 'description', 'valid deposit to add new validator'
+    yield 'pre', encode(state, spec.BeaconState)
+    yield 'deposit', encode(new_dep, spec.Deposit)
+    spec.process_deposit(state, new_dep)
+    yield 'post', encode(state, spec.BeaconState)
+
+
+@to_dict
+def valid_topup():
+    new_dep, state = build_deposit_for_index(10, 3)
+    yield 'description', 'valid deposit to top-up existing validator'
+    yield 'pre', encode(state, spec.BeaconState)
+    yield 'deposit', encode(new_dep, spec.Deposit)
+    spec.process_deposit(state, new_dep)
+    yield 'post', encode(state, spec.BeaconState)
+
+
+@to_dict
+def invalid_deposit_index():
+    new_dep, state = build_deposit_for_index(10, 10)
+    # Mess up deposit index, 1 too small
+    state.deposit_index = 9
+
+    yield 'description', 'invalid deposit index'
+    yield 'pre', encode(state, spec.BeaconState)
+    yield 'deposit', encode(new_dep, spec.Deposit)
+    try:
+        spec.process_deposit(state, new_dep)
+    except AssertionError:
+        # expected
+        yield 'post', None
+        return
+    raise Exception('invalid_deposit_index has unexpectedly allowed deposit')
+
+
+@to_dict
+def invalid_deposit_proof():
+    new_dep, state = build_deposit_for_index(10, 10)
+    # Make deposit proof invalid (at bottom of proof)
+    new_dep.proof[-1] = spec.ZERO_HASH
+
+    yield 'description', 'invalid deposit proof'
+    yield 'pre', encode(state, spec.BeaconState)
+    yield 'deposit', encode(new_dep, spec.Deposit)
+    try:
+        spec.process_deposit(state, new_dep)
+    except AssertionError:
+        # expected
+        yield 'post', None
+        return
+    raise Exception('invalid_deposit_index has unexpectedly allowed deposit')
+
+
+@to_tuple
+def deposit_cases():
+    yield valid_deposit()
+    yield valid_topup()
+    yield invalid_deposit_index()
+    yield invalid_deposit_proof()
+
+
+def mini_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'minimal')
+    spec.apply_constants_preset(presets)
+
+    return ("deposit_minimal", "deposits", gen_suite.render_suite(
+        title="deposit operation",
+        summary="Test suite for deposit type operation processing",
+        forks_timeline="testing",
+        forks=["phase0"],
+        config="minimal",
+        runner="operations",
+        handler="deposits",
+        test_cases=deposit_cases()))
+
+
+def full_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'mainnet')
+    spec.apply_constants_preset(presets)
+
+    return ("deposit_full", "deposits", gen_suite.render_suite(
+        title="deposit operation",
+        summary="Test suite for deposit type operation processing",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="operations",
+        handler="deposits",
+        test_cases=deposit_cases()))

test_generators/operations/genesis.py

@@ -0,0 +1,44 @@
+from eth2spec.phase0 import spec
+from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof
+from typing import List
+
+
+def create_genesis_state(deposits: List[spec.Deposit]) -> spec.BeaconState:
+    deposit_root = get_merkle_root((tuple([spec.hash(dep.data.serialize()) for dep in deposits])))
+
+    return spec.get_genesis_beacon_state(
+        deposits,
+        genesis_time=0,
+        genesis_eth1_data=spec.Eth1Data(
+            deposit_root=deposit_root,
+            deposit_count=len(deposits),
+            block_hash=spec.ZERO_HASH,
+        ),
+    )
+
+
+def create_deposits(pubkeys: List[spec.BLSPubkey], withdrawal_cred: List[spec.Bytes32]) -> List[spec.Deposit]:
+
+    # Mock proof of possession
+    proof_of_possession = b'\x33' * 96
+
+    deposit_data = [
+        spec.DepositData(
+            pubkey=pubkeys[i],
+            withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[i][1:],
+            amount=spec.MAX_DEPOSIT_AMOUNT,
+            proof_of_possession=proof_of_possession,
+        ) for i in range(len(pubkeys))
+    ]
+
+    # Fill tree with existing deposits
+    deposit_data_leaves = [spec.hash(data.serialize()) for data in deposit_data]
+    tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves))
+
+    return [
+        spec.Deposit(
+            proof=list(get_merkle_proof(tree, item_index=i)),
+            index=i,
+            data=deposit_data[i]
+        ) for i in range(len(deposit_data))
+    ]

test_generators/operations/keys.py

@@ -0,0 +1,7 @@
+from py_ecc import bls
+from eth2spec.phase0.spec import hash
+
+privkeys = list(range(1, 101))
+pubkeys = [bls.privtopub(k) for k in privkeys]
+# Insecure, but easier to follow
+withdrawal_creds = [hash(bls.privtopub(k)) for k in privkeys]

test_generators/operations/main.py

@@ -0,0 +1,9 @@
+from gen_base import gen_runner
+
+from deposits import mini_deposits_suite, full_deposits_suite
+
+if __name__ == "__main__":
+    gen_runner.run_generator("operations", [
+        mini_deposits_suite,
+        full_deposits_suite
+    ])

test_generators/operations/requirements.txt

@@ -0,0 +1,5 @@
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec
+py_ecc

test_generators/shuffling/README.md

@@ -0,0 +1,10 @@
+# Shuffling Tests
+
+Tests for the swap-or-not shuffling in ETH 2.0.
+
+Tips for initial shuffling write:
+- run with `round_count = 1` first, do the same with pyspec.
+- start with permute index
+- optimized shuffling implementations:
+  - vitalik, Python: https://github.com/ethereum/eth2.0-specs/pull/576#issue-250741806
+  - protolambda, Go: https://github.com/protolambda/eth2-shuffle

test_generators/shuffling/main.py

@@ -0,0 +1,54 @@
+from eth2spec.phase0 import spec
+from eth_utils import (
+    to_dict, to_tuple
+)
+from gen_base import gen_runner, gen_suite, gen_typing
+from preset_loader import loader
+
+
+@to_dict
+def shuffling_case(seed: spec.Bytes32, count: int):
+    yield 'seed', '0x' + seed.hex()
+    yield 'count', count
+    yield 'shuffled', [spec.get_permuted_index(i, count, seed) for i in range(count)]
+
+
+@to_tuple
+def shuffling_test_cases():
+    for seed in [spec.hash(spec.int_to_bytes4(seed_init_value)) for seed_init_value in range(30)]:
+        for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000]:
+            yield shuffling_case(seed, count)
+
+
+def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'minimal')
+    spec.apply_constants_preset(presets)
+
+    return ("shuffling_minimal", "core", gen_suite.render_suite(
+        title="Swap-or-Not Shuffling tests with minimal config",
+        summary="Swap or not shuffling, with minimally configured testing round-count",
+        forks_timeline="testing",
+        forks=["phase0"],
+        config="minimal",
+        runner="shuffling",
+        handler="core",
+        test_cases=shuffling_test_cases()))
+
+
+def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    presets = loader.load_presets(configs_path, 'mainnet')
+    spec.apply_constants_preset(presets)
+
+    return ("shuffling_full", "core", gen_suite.render_suite(
+        title="Swap-or-Not Shuffling tests with mainnet config",
+        summary="Swap or not shuffling, with normal configured (secure) mainnet round-count",
+        forks_timeline="mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="shuffling",
+        handler="core",
+        test_cases=shuffling_test_cases()))
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("shuffling", [mini_shuffling_suite, full_shuffling_suite])

test_generators/shuffling/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+../../test_libs/pyspec

test_generators/ssz/main.py

@@ -0,0 +1,47 @@
+from uint_test_cases import (
+    generate_random_uint_test_cases,
+    generate_uint_wrong_length_test_cases,
+    generate_uint_bounds_test_cases,
+    generate_uint_out_of_bounds_test_cases
+)
+
+from gen_base import gen_runner, gen_suite, gen_typing
+
+def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("uint_random", "uint", gen_suite.render_suite(
+        title="UInt Random",
+        summary="Random integers chosen uniformly over the allowed value range",
+        forks_timeline= "mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="ssz",
+        handler="uint",
+        test_cases=generate_random_uint_test_cases()))
+
+
+def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("uint_wrong_length", "uint", gen_suite.render_suite(
+        title="UInt Wrong Length",
+        summary="Serialized integers that are too short or too long",
+        forks_timeline= "mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="ssz",
+        handler="uint",
+        test_cases=generate_uint_wrong_length_test_cases()))
+
+
+def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput:
+    return ("uint_bounds", "uint", gen_suite.render_suite(
+        title="UInt Bounds",
+        summary="Integers right at or beyond the bounds of the allowed value range",
+        forks_timeline= "mainnet",
+        forks=["phase0"],
+        config="mainnet",
+        runner="ssz",
+        handler="uint",
+        test_cases=generate_uint_bounds_test_cases() + generate_uint_out_of_bounds_test_cases()))
+
+
+if __name__ == "__main__":
+    gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite])

test_generators/ssz/renderers.py

@@ -0,0 +1,93 @@
+from collections.abc import (
+    Mapping,
+    Sequence,
+)
+
+from eth_utils import (
+    encode_hex,
+    to_dict,
+)
+
+from ssz.sedes import (
+    BaseSedes,
+    Boolean,
+    Bytes,
+    BytesN,
+    Container,
+    List,
+    UInt,
+)
+
+
+def render_value(value):
+    if isinstance(value, bool):
+        return value
+    elif isinstance(value, int):
+        return str(value)
+    elif isinstance(value, bytes):
+        return encode_hex(value)
+    elif isinstance(value, Sequence):
+        return tuple(render_value(element) for element in value)
+    elif isinstance(value, Mapping):
+        return render_dict_value(value)
+    else:
+        raise ValueError(f"Cannot render value {value}")
+
+
+@to_dict
+def render_dict_value(value):
+    for key, value in value.items():
+        yield key, render_value(value)
+
+
+def render_type_definition(sedes):
+    if isinstance(sedes, Boolean):
+        return "bool"
+
+    elif isinstance(sedes, UInt):
+        return f"uint{sedes.length * 8}"
+
+    elif isinstance(sedes, BytesN):
+        return f"bytes{sedes.length}"
+
+    elif isinstance(sedes, Bytes):
+        return f"bytes"
+
+    elif isinstance(sedes, List):
+        return [render_type_definition(sedes.element_sedes)]
+
+    elif isinstance(sedes, Container):
+        return {
+            field_name: render_type_definition(field_sedes)
+            for field_name, field_sedes in sedes.fields
+        }
+
+    elif isinstance(sedes, BaseSedes):
+        raise Exception("Unreachable: All sedes types have been checked")
+
+    else:
+        raise TypeError("Expected BaseSedes")
+
+
+@to_dict
+def render_test_case(*, sedes, valid, value=None, serial=None, description=None, tags=None):
+    value_and_serial_given = value is not None and serial is not None
+    if valid:
+        if not value_and_serial_given:
+            raise ValueError("For valid test cases, both value and ssz must be present")
+    else:
+        if value_and_serial_given:
+            raise ValueError("For invalid test cases, one of either value or ssz must not be present")
+
+    if tags is None:
+        tags = []
+
+    yield "type", render_type_definition(sedes)
+    yield "valid", valid
+    if value is not None:
+        yield "value", render_value(value)
+    if serial is not None:
+        yield "ssz", encode_hex(serial)
+    if description is not None:
+        yield description
+    yield "tags", tags

test_generators/ssz/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils==1.4.1
+../../test_libs/gen_helpers
+../../test_libs/config_helpers
+ssz==0.1.0a2

test_generators/ssz/uint_test_cases.py

@@ -0,0 +1,98 @@
+import random
+
+from eth_utils import (
+    to_tuple,
+)
+
+import ssz
+from ssz.sedes import (
+    UInt,
+)
+from renderers import (
+    render_test_case,
+)
+
+random.seed(0)
+
+
+BIT_SIZES = [8, 16, 32, 64, 128, 256]
+RANDOM_TEST_CASES_PER_BIT_SIZE = 10
+RANDOM_TEST_CASES_PER_LENGTH = 3
+
+
+def get_random_bytes(length):
+    return bytes(random.randint(0, 255) for _ in range(length))
+
+
+@to_tuple
+def generate_random_uint_test_cases():
+    for bit_size in BIT_SIZES:
+        sedes = UInt(bit_size)
+
+        for _ in range(RANDOM_TEST_CASES_PER_BIT_SIZE):
+            value = random.randrange(0, 2**bit_size)
+            serial = ssz.encode(value, sedes)
+            # note that we need to create the tags in each loop cycle, otherwise ruamel will use
+            # YAML references which makes the resulting file harder to read
+            tags = tuple(["atomic", "uint", "random"])
+            yield render_test_case(
+                sedes=sedes,
+                valid=True,
+                value=value,
+                serial=serial,
+                tags=tags,
+            )
+
+
+@to_tuple
+def generate_uint_wrong_length_test_cases():
+    for bit_size in BIT_SIZES:
+        sedes = UInt(bit_size)
+        lengths = sorted({
+            0,
+            sedes.length // 2,
+            sedes.length - 1,
+            sedes.length + 1,
+            sedes.length * 2,
+        })
+        for length in lengths:
+            for _ in range(RANDOM_TEST_CASES_PER_LENGTH):
+                tags = tuple(["atomic", "uint", "wrong_length"])
+                yield render_test_case(
+                    sedes=sedes,
+                    valid=False,
+                    serial=get_random_bytes(length),
+                    tags=tags,
+                )
+
+
+@to_tuple
+def generate_uint_bounds_test_cases():
+    common_tags = ("atomic", "uint")
+    for bit_size in BIT_SIZES:
+        sedes = UInt(bit_size)
+
+        for value, tag in ((0, "uint_lower_bound"), (2 ** bit_size - 1, "uint_upper_bound")):
+            serial = ssz.encode(value, sedes)
+            yield render_test_case(
+                sedes=sedes,
+                valid=True,
+                value=value,
+                serial=serial,
+                tags=common_tags + (tag,),
+            )
+
+
+@to_tuple
+def generate_uint_out_of_bounds_test_cases():
+    common_tags = ("atomic", "uint")
+    for bit_size in BIT_SIZES:
+        sedes = UInt(bit_size)
+
+        for value, tag in ((-1, "uint_underflow"), (2 ** bit_size, "uint_overflow")):
+            yield render_test_case(
+                sedes=sedes,
+                valid=False,
+                value=value,
+                tags=common_tags + (tag,),
+            )

test_libs/config_helpers/README.md

@@ -0,0 +1,19 @@
+# ETH 2.0 config helpers
+
+`preset_loader`: A util to load constants-presets with.
+See [Constants-presets documentation](../../configs/constants_presets/README.md).
+
+Usage:
+
+```python
+configs_path = 'configs/'
+
+...
+
+import preset_loader
+from eth2spec.phase0 import spec
+my_presets = preset_loader.load_presets(configs_path, 'mainnet')
+spec.apply_constants_preset(my_presets)
+```
+
+WARNING: this overwrites globals, make sure to prevent accidental collisions with other usage of the same imported specs package.

test_libs/config_helpers/preset_loader/loader.py

@@ -0,0 +1,25 @@
+from typing import Dict, Any
+
+from ruamel.yaml import (
+    YAML,
+)
+from pathlib import Path
+from os.path import join
+
+
+def load_presets(configs_dir, presets_name) -> Dict[str, Any]:
+    """
+    Loads the given preset
+    :param presets_name: The name of the generator. (lowercase snake_case)
+    :return: Dictionary, mapping of constant-name -> constant-value
+    """
+    path = Path(join(configs_dir, 'constant_presets', presets_name+'.yaml'))
+    yaml = YAML(typ='base')
+    loaded = yaml.load(path)
+    out = dict()
+    for k, v in loaded.items():
+        if v.startswith("0x"):
+            out[k] = bytes.fromhex(v[2:])
+        else:
+            out[k] = int(v)
+    return out

test_libs/config_helpers/requirements.txt

@@ -0,0 +1 @@
+ruamel.yaml==0.15.87

test_libs/config_helpers/setup.py

@@ -0,0 +1,9 @@
+from distutils.core import setup
+
+setup(
+    name='config_helpers',
+    packages=['preset_loader'],
+    install_requires=[
+        "ruamel.yaml==0.15.87"
+    ]
+)

test_libs/gen_helpers/README.md

@@ -0,0 +1,5 @@
+# ETH 2.0 test generator helpers
+
+`gen_base`: A util to quickly write new test suite generators with.
+See [Generators documentation](../../test_generators/README.md).
+

test_libs/gen_helpers/gen_base/gen_runner.py

@@ -0,0 +1,115 @@
+import argparse
+from pathlib import Path
+import sys
+from typing import List
+
+from ruamel.yaml import (
+    YAML,
+)
+
+from gen_base.gen_typing import TestSuiteCreator
+
+
+def validate_output_dir(path_str):
+    path = Path(path_str)
+
+    if not path.exists():
+        raise argparse.ArgumentTypeError("Output directory must exist")
+
+    if not path.is_dir():
+        raise argparse.ArgumentTypeError("Output path must lead to a directory")
+
+    return path
+
+
+def validate_configs_dir(path_str):
+    path = Path(path_str)
+
+    if not path.exists():
+        raise argparse.ArgumentTypeError("Configs directory must exist")
+
+    if not path.is_dir():
+        raise argparse.ArgumentTypeError("Config path must lead to a directory")
+
+    if not Path(path, "constant_presets").exists():
+        raise argparse.ArgumentTypeError("Constant Presets directory must exist")
+
+    if not Path(path, "constant_presets").is_dir():
+        raise argparse.ArgumentTypeError("Constant Presets path must lead to a directory")
+
+    if not Path(path, "fork_timelines").exists():
+        raise argparse.ArgumentTypeError("Fork Timelines directory must exist")
+
+    if not Path(path, "fork_timelines").is_dir():
+        raise argparse.ArgumentTypeError("Fork Timelines path must lead to a directory")
+
+    return path
+
+
+def run_generator(generator_name, suite_creators: List[TestSuiteCreator]):
+    """
+    Implementation for a general test generator.
+    :param generator_name: The name of the generator. (lowercase snake_case)
+    :param suite_creators: A list of suite creators, each of these builds a list of test cases.
+    :return:
+    """
+
+    parser = argparse.ArgumentParser(
+        prog="gen-" + generator_name,
+        description=f"Generate YAML test suite files for {generator_name}",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        dest="output_dir",
+        required=True,
+        type=validate_output_dir,
+        help="directory into which the generated YAML files will be dumped"
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        default=False,
+        help="if set overwrite test files if they exist",
+    )
+    parser.add_argument(
+        "-c",
+        "--configs-path",
+        dest="configs_path",
+        required=True,
+        type=validate_configs_dir,
+        help="specify the path of the configs directory (containing constants_presets and fork_timelines)",
+    )
+
+    args = parser.parse_args()
+    output_dir = args.output_dir
+    if not args.force:
+        file_mode = "x"
+    else:
+        file_mode = "w"
+
+    yaml = YAML(pure=True)
+    yaml.default_flow_style = None
+
+    print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...")
+    print(f"Reading config presets and fork timelines from {args.configs_path}")
+    for suite_creator in suite_creators:
+        (output_name, handler, suite) = suite_creator(args.configs_path)
+
+        handler_output_dir = Path(output_dir) / Path(handler)
+        try:
+            if not handler_output_dir.exists():
+                handler_output_dir.mkdir()
+        except FileNotFoundError as e:
+            sys.exit(f'Error when creating handler dir {handler} for test "{suite["title"]}" ({e})')
+
+        out_path = handler_output_dir / Path(output_name + '.yaml')
+
+        try:
+            with out_path.open(file_mode) as f:
+                yaml.dump(suite, f)
+        except IOError as e:
+            sys.exit(f'Error when dumping test "{suite["title"]}" ({e})')
+
+    print("done.")

test_libs/gen_helpers/gen_base/gen_suite.py

@@ -0,0 +1,22 @@
+from typing import Iterable
+
+from eth_utils import to_dict
+from gen_base.gen_typing import TestCase
+
+
+@to_dict
+def render_suite(*,
+                 title: str, summary: str,
+                 forks_timeline: str, forks: Iterable[str],
+                 config: str,
+                 runner: str,
+                 handler: str,
+                 test_cases: Iterable[TestCase]):
+    yield "title", title
+    yield "summary", summary
+    yield "forks_timeline", forks_timeline,
+    yield "forks", forks
+    yield "config", config
+    yield "runner", runner
+    yield "handler", handler
+    yield "test_cases", test_cases

test_libs/gen_helpers/gen_base/gen_typing.py

@@ -0,0 +1,8 @@
+from typing import Callable, Dict, Tuple, Any
+
+TestCase = Dict[str, Any]
+TestSuite = Dict[str, Any]
+# Tuple: (output name, handler name, suite) -- output name excl. ".yaml"
+TestSuiteOutput = Tuple[str, str, TestSuite]
+# Args: <presets path>
+TestSuiteCreator = Callable[[str], TestSuiteOutput]

test_libs/gen_helpers/requirements.txt

@@ -0,0 +1,2 @@
+ruamel.yaml==0.15.87
+eth-utils==1.4.1

test_libs/gen_helpers/setup.py

@@ -0,0 +1,10 @@
+from distutils.core import setup
+
+setup(
+    name='gen_helpers',
+    packages=['gen_base'],
+    install_requires=[
+        "ruamel.yaml==0.15.87",
+        "eth-utils==1.4.1"
+    ]
+)

test_libs/pyspec/README.md

@@ -0,0 +1,27 @@
+# ETH 2.0 PySpec
+
+The Python executable spec is built from the ETH 2.0 specification, 
+ complemented with the necessary helper functions for hashing, BLS, and more.
+
+With this executable spec,
+ test-generators can easily create test-vectors for client implementations,
+ and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest.
+
+## Building
+
+All the dynamic parts of the spec can be build at once with `make pyspec`.
+
+Alternatively, you can build a sub-set of the pyspec: `make phase0`.
+
+Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py`
+
+## Contributing
+
+Contributions are welcome, but consider implementing your idea as part of the spec itself first.
+The pyspec is not a replacement.
+If you see opportunity to include any of the `pyspec/eth2spec/utils/` code in the spec,
+ please submit an issue or PR.
+
+## License
+
+Same as the spec itself, see LICENSE file in spec repository root.

test_libs/pyspec/eth2spec/debug/decode.py

@@ -0,0 +1,28 @@
+from eth2spec.utils.minimal_ssz import hash_tree_root
+
+
+def decode(json, typ):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return json
+    elif typ == 'bool':
+        assert json in (True, False)
+        return json
+    elif isinstance(typ, list):
+        return [decode(element, typ[0]) for element in json]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return bytes.fromhex(json[2:])
+    elif hasattr(typ, 'fields'):
+        temp = {}
+        for field, subtype in typ.fields.items():
+            temp[field] = decode(json[field], subtype)
+            if field + "_hash_tree_root" in json:
+                assert(json[field + "_hash_tree_root"][2:] == 
+                       hash_tree_root(temp[field], subtype).hex())
+        ret = typ(**temp)
+        if "hash_tree_root" in json:
+            assert(json["hash_tree_root"][2:] == 
+                   hash_tree_root(ret, typ).hex())
+        return ret
+    else:
+        print(json, typ)
+        raise Exception("Type not recognized")

test_libs/pyspec/eth2spec/debug/encode.py

@@ -0,0 +1,26 @@
+from eth2spec.utils.minimal_ssz import hash_tree_root
+
+
+def encode(value, typ, include_hash_tree_roots=False):
+    if isinstance(typ, str) and typ[:4] == 'uint':
+        return value
+    elif typ == 'bool':
+        assert value in (True, False)
+        return value
+    elif isinstance(typ, list):
+        return [encode(element, typ[0], include_hash_tree_roots) for element in value]
+    elif isinstance(typ, str) and typ[:4] == 'byte':
+        return '0x' + value.hex()
+    elif hasattr(typ, 'fields'):
+        ret = {}
+        for field, subtype in typ.fields.items():
+            ret[field] = encode(getattr(value, field), subtype, include_hash_tree_roots)
+            if include_hash_tree_roots:
+                ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex()
+        if include_hash_tree_roots:
+            ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex()
+        return ret
+    else:
+        print(value, typ)
+        raise Exception("Type not recognized")
+

test_libs/pyspec/eth2spec/phase0/state_transition.py

@@ -1,17 +1,17 @@
 from . import spec
 
 
-from typing import (  # noqa: F401
+from typing import (
     Any,
     Callable,
-    List,
+    List
-    NewType,
-    Tuple,
 )
 
 from .spec import (
     BeaconState,
     BeaconBlock,
+    Slot,
+    process_proposer_attestation_rewards,
 )
 
 
@@ -52,6 +52,7 @@ def process_operations(state: BeaconState, block: BeaconBlock) -> None:
         spec.MAX_ATTESTATIONS,
         spec.process_attestation,
     )
+    process_proposer_attestation_rewards(state)
 
     assert len(block.body.deposits) == expected_deposit_count(state)
     process_operation_type(
@@ -100,13 +101,16 @@ def process_epoch_transition(state: BeaconState) -> None:
     spec.finish_epoch_update(state)
 
 
-def state_transition(state: BeaconState,
+def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState:
-                     block: BeaconBlock,
+    while state.slot < up_to:
-                     verify_state_root: bool=False) -> BeaconState:
-    while state.slot < block.slot:
         spec.cache_state(state)
         if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0:
             process_epoch_transition(state)
         spec.advance_slot(state)
-        if block.slot == state.slot:
+
+
+def state_transition(state: BeaconState,
+                     block: BeaconBlock,
+                     verify_state_root: bool=False) -> BeaconState:
+    state_transition_to(state, block.slot)
     process_block(state, block, verify_state_root)

test_libs/pyspec/requirements.txt

@@ -0,0 +1,4 @@
+eth-utils>=1.3.0,<2
+eth-typing>=2.1.0,<3.0.0
+pycryptodome==3.7.3
+py_ecc>=1.6.0

test_libs/pyspec/setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='pyspec',
+    packages=find_packages(),
+    install_requires=[
+        "eth-utils>=1.3.0,<2",
+        "eth-typing>=2.1.0,<3.0.0",
+        "pycryptodome==3.7.3",
+        "py_ecc>=1.6.0",
+    ]
+)

utils/phase0/jsonize.py

@@ -1,52 +0,0 @@
-from .minimal_ssz import hash_tree_root
-
-
-def jsonize(value, typ, include_hash_tree_roots=False):
-    if isinstance(typ, str) and typ[:4] == 'uint':
-        return value
-    elif typ == 'bool':
-        assert value in (True, False)
-        return value
-    elif isinstance(typ, list):
-        return [jsonize(element, typ[0], include_hash_tree_roots) for element in value]
-    elif isinstance(typ, str) and typ[:4] == 'byte':
-        return '0x' + value.hex()
-    elif hasattr(typ, 'fields'):
-        ret = {}
-        for field, subtype in typ.fields.items():
-            ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots)
-            if include_hash_tree_roots:
-                ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex()
-        if include_hash_tree_roots:
-            ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex()
-        return ret
-    else:
-        print(value, typ)
-        raise Exception("Type not recognized")
-
-
-def dejsonize(json, typ):
-    if isinstance(typ, str) and typ[:4] == 'uint':
-        return json
-    elif typ == 'bool':
-        assert json in (True, False)
-        return json
-    elif isinstance(typ, list):
-        return [dejsonize(element, typ[0]) for element in json]
-    elif isinstance(typ, str) and typ[:4] == 'byte':
-        return bytes.fromhex(json[2:])
-    elif hasattr(typ, 'fields'):
-        temp = {}
-        for field, subtype in typ.fields.items():
-            temp[field] = dejsonize(json[field], subtype)
-            if field + "_hash_tree_root" in json:
-                assert(json[field + "_hash_tree_root"][2:] == 
-                       hash_tree_root(temp[field], subtype).hex())
-        ret = typ(**temp)
-        if "hash_tree_root" in json:
-            assert(json["hash_tree_root"][2:] == 
-                   hash_tree_root(ret, typ).hex())
-        return ret
-    else:
-        print(json, typ)
-        raise Exception("Type not recognized")