update ci config and makefile

.circleci/config.yml

@@ -1,5 +1,4 @@
-# Python CircleCI 2.0 configuration file
+version: 2.1
-version: 2
 jobs:
   build:
     docker:
@@ -8,34 +7,82 @@ jobs:
 
     steps:
       - checkout
-      # Download and cache dependencies
+      - run:
-      - restore_cache:
+          name: Build phase0 spec
-          keys:
+          command: make phase0
-            - v1-dependencies-{{ checksum "requirements.txt" }}
-            # fallback to using the latest cache if no exact match is found
-            - v1-dependencies-
 
       - run:
-          name: install dependencies
+          name: run py-tests
-          command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install -r requirements.txt
-      - run:
-          name: build phase0 spec
-          command: make build/phase0
-
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
-      - run:
-          name: run tests
           command: |
             . venv/bin/activate
             pytest tests
 
+      - run:
+          name: Generate YAML tests
+          command: make yaml_tests
+
       - store_artifacts:
           path: test-reports
           destination: test-reports
+
+      - run:
+          name: Save YAML tests for deployment
+          command: |
+            mkdir /tmp/workspace
+            cp -r yaml_tests /tmp/workspace/
+            git log -1 >> /tmp/workspace/latest_commit_message
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - yaml_tests
+            - latest_commit_message
+  commit:
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - add_ssh_keys:
+          fingerprints:
+            - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa"
+      - run:
+          name: Checkout test repository
+          command: |
+            ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+            git clone git@github.com:ethereum/eth2.0-tests.git
+      - run:
+          name: Commit and push generated YAML tests
+          command: |
+            cd eth2.0-tests
+            git config user.name 'eth2TestGenBot'
+            git config user.email '47188154+eth2TestGenBot@users.noreply.github.com'
+            for filename in /tmp/workspace/yaml_tests/*; do
+              rm -rf $(basename $filename)
+              cp -r $filename .
+            done
+            git add .
+            if git diff --cached --exit-code >& /dev/null; then
+              echo "No changes to commit"
+            else
+              echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message
+              cat /tmp/workspace/latest_commit_message >> commit_message
+              git commit -F commit_message
+              git push origin master
+            fi
+workflows:
+  version: 2.1
+
+  build_and_commit:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/
+      - commit:
+          requires:
+            - build
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              ignore: /.*/

Makefile

@@ -1,29 +1,74 @@
 SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
-BUILD_DIR = ./build
+TEST_LIBS_DIR = ./test_libs
-UTILS_DIR = ./utils
+PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
+
+YAML_TEST_DIR = ./yaml_tests
+GENERATOR_DIR = ./test_generators
+GENERATOR_VENVS_DIR = $(GENERATOR_DIR)/.venvs
+
+# Collect a list of generator names
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
+# Map this list of generator paths to a list of test output paths
+YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+
+PY_SPEC_PHASE_0_TARGET = $(PY_SPEC_DIR)/phase0/spec.py
+PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGET)
 
 
-.PHONY: clean all test
+.PHONY: clean all test yaml_tests pyspec phase0
-
-
-all: $(BUILD_DIR)/phase0
 
+all: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) $(PY_SPEC_ALL_TARGETS)
 
 clean:
-	rm -rf $(BUILD_DIR)
+	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(GENERATOR_VENVS_DIR)
+	rm -rf $(PY_SPEC_ALL_TARGETS)
 
+# "make yaml_tests" to run generators
+yaml_tests: $(YAML_TEST_TARGETS)
 
 # runs a limited set of tests against a minimal config
 # run pytest with `-m` option to full suite
-test:
+test: $(PY_SPEC_TARGETS)
 	pytest -m minimal_config tests/
 
+# "make pyspec" to create the pyspec for all phases.
+pyspec: $(PY_SPEC_TARGETS)
 
-$(BUILD_DIR)/phase0:
+# "make phase0" to create pyspec for phase0
-	mkdir -p $@
+phase0: $(PY_SPEC_DIR)/phase0/spec.py
+
+
+$(PY_SPEC_DIR)/phase0/spec.py:
 	python3 $(SCRIPT_DIR)/phase0/build_spec.py  $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py
-	mkdir -p $@/utils
+
-	cp $(UTILS_DIR)/phase0/* $@/utils
+
-	cp $(UTILS_DIR)/phase0/state_transition.py $@
+
-	touch $@/__init__.py $@/utils/__init__.py
+# The function that builds a set of suite files, by calling a generator for the given type (param 1)
+define build_yaml_tests
+	$(info running generator $(1))
+	# Create the output
+	mkdir -p $(YAML_TEST_DIR)$(1)
+
+	# Create a virtual environment
+	python3 -m venv $(VENV_DIR)$(1)
+	# Activate the venv, this is where dependencies are installed for the generator
+	. $(GENERATOR_VENVS_DIR)$(1)bin/activate
+	# Install all the necessary requirements
+	pip3 install -r $(GENERATOR_DIR)$(1)requirements.txt --user
+
+	# Run the generator. The generator is assumed to have an "main.py" file.
+	# We output to the tests dir (generator program should accept a "-p <filepath>" argument.
+	python3 $(GENERATOR_DIR)$(1)main.py -o $(YAML_TEST_DIR)$(1)
+	$(info generator $(1) finished)
+endef
+
+# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
+$(YAML_TEST_DIR):
+	$(info ${YAML_TEST_TARGETS})
+	mkdir -p $@
+
+# For any target within the tests dir, build it using the build_yaml_tests function.
+$(YAML_TEST_DIR)%:
+	$(call build_yaml_tests,$*)