refactor download_test_vectors.sh (#5)

README.md

@@ -5,8 +5,8 @@ This repo used to reformat the official Ethereum 2 tests to a format suitable fo
 Currently it is used for:
 
 - Having multiple test vectors versions side-by-side for progressive update between spec versions
-- SSZ test vectors are still using a json format from 0.8.1.
-- SSZ test vectors in json requires LFS
+- SSZ test vectors that are still using a JSON format from 0.8.1.
+- SSZ test vectors in JSON that require LFS
 
 ## Cloning the repo
 
@@ -18,7 +18,7 @@ git lfs install
 ```
 Afterwards any `git clone` will also do the required `git lfs pull` implicitly
 
-It you do not want to download the LFS files, exporting the following variable before `git clone`:
+It you do not want to download the LFS files, export the following variable before `git clone`:
 ```
 export GIT_LFS_SKIP_SMUDGE=1
 ```
@@ -33,5 +33,6 @@ git clone https://github.com/status-im/nim-eth2-official-tests
 cd nim-eth2-official-tests
 
 # Download versioned test vectors
-sh download_test_vectors.sh
+./download_test_vectors.sh
 ```
+

download_test_vectors.sh

@@ -2,16 +2,58 @@
 
 set -eu
 
-source scripts/download_functions.sh
+FLAVOURS=(
+	"general"
+	"minimal"
+	"mainnet"
+)
 
-dl_version v0.8.3
-dl_version v0.9.0
+dl_version() {
+	[[ -z "$1" ]] && { echo "usage: dl_version() vX.Y.Z"; exit 1; }
+	version="$1"
 
-echo "Ignore the warnings \"unknown extended header keyword 'SCHILY.{dev,ino,nlink}'\" on Linux."
-# tar: Ignoring unknown extended header keyword 'SCHILY.dev'
-# tar: Ignoring unknown extended header keyword 'SCHILY.ino'
-# tar: Ignoring unknown extended header keyword 'SCHILY.nlink'
-echo "Those are due to the test vectors being packed with OSX BSD tar."
+	mkdir -p "tarballs/${version}"
+	pushd "tarballs/${version}" >/dev/null
+	for flavour in "${FLAVOURS[@]}"; do
+		if [[ ! -e "${flavour}.tar.gz" ]]; then
+			echo "Downloading: ${version}/${flavour}.tar.gz"
+			curl --location --remote-name --silent --show-error \
+				"https://github.com/ethereum/eth2.0-spec-tests/releases/download/${version}/${flavour}.tar.gz" \
+				|| {
+					echo "Curl failed. Aborting"
+					rm -f "${flavour}.tar.gz"
+					exit 1
+				}
+		fi
+	done
+	popd >/dev/null
+}
+
+unpack_version() {
+	[[ -z "$1" ]] && { echo "usage: unpack_version() vX.Y.Z"; exit 1; }
+	version="$1"
+
+	dl_version "$version"
+
+	# suppress warnings when unpacking with GNU tar an archive created with BSD tar (probably on macOS)
+	EXTRA_TAR_PARAMS=""
+	tar --version | grep -qi 'gnu' && EXTRA_TAR_PARAMS="--warning=no-unknown-keyword"
+
+	if [[ ! -d "tests-${version}" ]]; then
+		for flavour in "${FLAVOURS[@]}"; do
+			echo "Unpacking: ${version}/${flavour}.tar.gz"
+			tar --one-top-level="tests-${version}" --strip-components 1 --ignore-zeros ${EXTRA_TAR_PARAMS} -xzf \
+				"tarballs/${version}/${flavour}.tar.gz" \
+				|| {
+					echo "Tar failed. Aborting."
+					rm -rf "tests-${version}"
+					exit 1
+				}
+		done
+	fi
+}
+
+for version in "v0.8.3" "v0.9.0"; do
+	unpack_version "$version"
+done
 
-unpack_version v0.8.3
-unpack_version v0.9.0

scripts/download_functions.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-dl_version() {
-  [[ -z "$1" ]] && { echo "usage: dl_version() vX.Y.Z"; exit 1; }
-
-  [[ -d "tarballs/$1" ]] || {
-    mkdir -p "tarballs/$1"
-    pushd "tarballs/$1"
-    curl -L --remote-name-all "https://github.com/ethereum/eth2.0-spec-tests/releases/download/$1/{general,minimal,mainnet}.tar.gz"
-    popd
-  }
-}
-
-unpack_version() {
-  [[ -z "$1" ]] && { echo "usage: unpack_version() vX.Y.Z"; exit 1; }
-
-  [[ -d "tests-$1" ]] || {
-    cat "tarballs/$1"/{general,minimal,mainnet}.tar.gz | tar --one-top-level="tests-$1" --strip-components 1 -xzf - -i
-  }
-}