Test vectors: use tarballs, allow multiple spec versions for progressive spec updates (#514)

* update test submodule [skip ci]

* rename the process_lfs script to setup_official_tests [skip ci]

* clarify that the existing LFS is for json files [skip ci]

* Update again to use download/unpack function library

* update the test downloading script [skip ci]

* Update EF fixtures with new test path

* fix symlink order

* Fix relative:absolute symlink issue

* add symlink existence check

* setup_official_tests.sh: refactoring

.appveyor.yml

@@ -31,7 +31,7 @@ install:
   - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
 
   # LFS test fixtures
-  - bash scripts\process_lfs.sh jsonTestsCache
+  - bash scripts\setup_official_tests.sh jsonTestsCache
 
 build_script:
   # the 32-bit build is done on a 64-bit image, so we need to override the architecture

.travis.yml

@@ -49,11 +49,10 @@ install:
   - vendor/nimbus-build-system/scripts/build_rocksdb.sh rocksdbCache
 
   # LFS test fixtures
-  - scripts/process_lfs.sh jsonTestsCache
+  - scripts/setup_official_tests.sh jsonTestsCache
 
 script:
   - set -e # fail fast
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" update # to allow a newer Nim version to be detected
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_LFS_SCRIPT=1 test
-

Makefile

@@ -55,7 +55,7 @@ p2pd: | go-checks
 DISABLE_LFS_SCRIPT := 0
 test: | build deps
 ifeq ($(DISABLE_LFS_SCRIPT), 0)
-	V=$(V) scripts/process_lfs.sh
+	V=$(V) scripts/setup_official_tests.sh
 endif
 	$(ENV_SCRIPT) nim test $(NIM_PARAMS) beacon_chain.nims && rm -f 0000-*.json
 
@@ -81,4 +81,3 @@ clean-testnet1:
 
 clean: | clean-common
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node}
-

azure-pipelines.yml

@@ -66,11 +66,10 @@ jobs:
           git config --global core.longpaths true
           export GIT_LFS_SKIP_SMUDGE=1
           git submodule --quiet update --init --recursive
-          scripts/process_lfs.sh jsonTestsCache
+          scripts/setup_official_tests.sh jsonTestsCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
           file build/beacon_node
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_LFS_SCRIPT=1 test
         displayName: 'build and test'
-

beacon_chain.nimble

@@ -61,4 +61,4 @@ task test, "Run all tests":
 
 task sync_lfs_tests, "Sync LFS json tests":
   # Syncs the json test files (but not the EF yaml tests)
-  exec "scripts/process_lfs.sh"
+  exec "scripts/setup_official_tests.sh"

scripts/setup_official_tests.sh

@@ -13,7 +13,7 @@ TMP_CACHE_DIR="tmpcache"
 SUBREPO_DIR="tests/official/fixtures"
 # verbosity level
 [[ -z "$V" ]] && V=0
-[[ -z "$BUILD_MSG" ]] && BUILD_MSG="Downloading LFS files"
+[[ -z "$BUILD_MSG" ]] && BUILD_MSG="Downloading official test vectors"
 CACHE_DIR="$1" # optional parameter pointing to a CI cache dir. Without it, we just download the LFS files for a local `make test`.
 
 [[ -d "${SUBREPO_DIR}" ]] || { echo "This script should be run from the \"nim-beacon-chain\" repo top dir."; exit 1; }
@@ -33,13 +33,19 @@ COMPRESS_XZ="false"
 which 7z &>/dev/null && { DECOMPRESS_XZ="7z e -txz -bd -so"; COMPRESS_XZ="7z a -txz -an -bd -si -so"; }
 which xz &>/dev/null && { DECOMPRESS_XZ="xz -d -c -T 0"; COMPRESS_XZ="xz -c -T 0"; }
 
-download_lfs_files() {
-	[[ -z "$1" ]] && { echo "usage: download_lfs_files() subdir_name"; exit 1; }
+# script output
+echo -e "$BUILD_MSG"
+[[ "$V" == "0" ]] && exec 3>&1 4>&2 &>/dev/null # save stdout and stderr before sending them into oblivion
+
+#############################################
+# JSON test files (SSZ v0.8.1) - TODO migrate
+
+download_lfs_json_files() {
+	[[ -z "$1" ]] && { echo "usage: download_lfs_json_files() subdir_name"; exit 1; }
 	LFS_DIR="$1"
 
-	echo -e "$BUILD_MSG"
-	which git-lfs &>/dev/null || { echo "Error: 'git-lfs' not found. Please install the corresponding package."; exit 1; }
-	[[ "$V" == "0" ]] && exec &>/dev/null
+	# restore stdout and stderr to make sure this error message is shown
+	which git-lfs &>/dev/null || { [[ "$V" == "0" ]] && exec 1>&3 2>&4; echo "Error: 'git-lfs' not found. Please install the corresponding package."; exit 1; }
 
 	pushd "${SUBREPO_DIR}"
 	git lfs install # redundant after running it once per repo, but fast enough not to worry about detecting whether it ran before
@@ -47,8 +53,8 @@ download_lfs_files() {
 	popd
 }
 
-process_subdir() {
-	[[ -z "$1" ]] && { echo "usage: process_subdir subdir_name"; exit 1; }
+process_json_subdir() {
+	[[ -z "$1" ]] && { echo "usage: process_json_subdir subdir_name"; exit 1; }
 	LFS_DIR="$1"
 	ARCHIVE_NAME="${LFS_DIR}.tar.xz"
 
@@ -57,7 +63,7 @@ process_subdir() {
 		# compare the archive's mtime to the date of the last commit
 		if [[ $(stat ${STAT_FORMAT} "${CACHE_DIR}/${ARCHIVE_NAME}") -gt $(cd "${SUBREPO_DIR}"; git log --pretty=format:%cd -n 1 --date=unix "${LFS_DIR}") ]]; then
 			# the cache is valid
-			echo "Copying cached files into ${SUBREPO_DIR}/${LFS_DIR}/"
+			echo "Copying cached json files into ${SUBREPO_DIR}/${LFS_DIR}/"
 			mkdir -p "${TMP_CACHE_DIR}"
 			${DECOMPRESS_XZ} "${CACHE_DIR}/${ARCHIVE_NAME}" | tar -x -C "${TMP_CACHE_DIR}" -f -
 			cp -a "${TMP_CACHE_DIR}/${LFS_DIR}"/* "${SUBREPO_DIR}/${LFS_DIR}/"
@@ -76,7 +82,7 @@ process_subdir() {
 		if [[ "${ON_MACOS}" == "1" ]]; then
 			HOMEBREW_NO_AUTO_UPDATE=1 brew install git-lfs # this takes almost 5 minutes on Travis, so only run it if needed
 		fi
-		download_lfs_files "$LFS_DIR"
+		download_lfs_json_files "$LFS_DIR"
 		echo "Updating the cache."
 		pushd "${SUBREPO_DIR}"
 		# the archive will contain ${LFS_DIR} as its top dir
@@ -87,12 +93,20 @@ process_subdir() {
 }
 
 if [[ -n "${CACHE_DIR}" ]]; then
-	process_subdir "json_tests_v0.8.1"
-	process_subdir "json_tests_v0.8.3"
+	process_json_subdir "json_tests_v0.8.1"
+	process_json_subdir "json_tests_v0.8.3"
 
+	# Ethereum Foundation test vectors
+	mkdir -p "${CACHE_DIR}/tarballs"
+	rm -rf "${SUBREPO_DIR}/tarballs"
+	ln -s "$(pwd -P)/${CACHE_DIR}/tarballs" "${SUBREPO_DIR}"
 else
 	# no caching
-	download_lfs_files "json_tests_v0.8.1"
-	download_lfs_files "json_tests_v0.8.3"
+	download_lfs_json_files "json_tests_v0.8.1"
+	download_lfs_json_files "json_tests_v0.8.3"
 fi
 
+pushd "${SUBREPO_DIR}"
+./download_test_vectors.sh
+popd
+

tests/official/fixtures

@@ -1 +1 @@
-Subproject commit 59830c90e11291c886c71b8cdf83a81651d5fbc0
+Subproject commit 3062a33b265d449d3adcac73de8a5b5d377f1cb8

tests/official/fixtures_utils.nim

@@ -37,7 +37,7 @@ proc readValue*(r: var JsonReader, a: var seq[byte]) {.inline.} =
 const
   FixturesDir* = currentSourcePath.rsplit(DirSep, 1)[0] / "fixtures"
   JsonTestsDir* = FixturesDir/"json_tests_v0.8.3"
-  SszTestsDir* = FixturesDir/"eth2.0-spec-tests"/"tests"
+  SszTestsDir* = FixturesDir/"tests-v0.8.3"
 
 proc parseTest*(path: string, Format: typedesc[Json or SSZ], T: typedesc): T =
   try: