From 1ec4c5aef7a8db64e9a694d30158745bbec68566 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Fri, 30 Aug 2019 17:50:46 +0200 Subject: [PATCH] Prepare for v0.8.3 tests: submodules + LFS script + Appveyor updates (#368) * Update fixtures to 0.8.3 * v0.8.1 tests use new test path * LFS caching: handle multiple versions of json_tests * Enable paths > 260 characters in appveyor * Appveyor: use git clone core.longpaths + the env variable at startup * improve process_lfs.sh and cleanup submodules --- .appveyor.yml | 6 ++++- .gitmodules | 15 ------------ scripts/process_lfs.sh | 24 ++++++++++++++----- tests/official/fixtures | 2 +- ...es_utils.nim => fixtures_utils_v0_8_1.nim} | 6 +++-- tests/official/test_fixture_bls.nim | 15 ++++++------ tests/official/test_fixture_shuffling.nim | 12 ++++------ tests/official/test_fixture_ssz_static.nim | 2 +- tests/official/test_fixture_ssz_uint.nim | 2 +- 9 files changed, 42 insertions(+), 42 deletions(-) rename tests/official/{fixtures_utils.nim => fixtures_utils_v0_8_1.nim} (95%) diff --git a/.appveyor.yml b/.appveyor.yml index 327f7266d..3d15a1df0 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,6 +6,11 @@ environment: # disable LFS file downloading during regular cloning GIT_LFS_SKIP_SMUDGE: 1 +init: # Scripts called at the very beginning + # Enable paths > 260 characters + - ps: Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1 + - git config --global core.longpaths true + cache: - NimBinaries - p2pdCache @@ -39,4 +44,3 @@ test_script: - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_LFS_SCRIPT=1 DISABLE_GO_CHECKS=1 test deploy: off - diff --git a/.gitmodules b/.gitmodules index 223b8a535..0c40d7b48 100644 --- a/.gitmodules +++ b/.gitmodules @@ -36,21 +36,6 @@ url = https://github.com/status-im/nim-web3.git ignore = dirty branch = master -[submodule "vendor/Nim"] - path = vendor/Nim - url = https://github.com/status-im/Nim.git - ignore = dirty - branch = nimbus -[submodule "vendor/Nim-csources"] - ignore = dirty - branch = master - path = vendor/Nim-csources - url = https://github.com/nim-lang/csources.git -[submodule "vendor/nimble"] - path = vendor/nimble - url = https://github.com/nim-lang/nimble.git - ignore = dirty - branch = master [submodule "vendor/nim-nat-traversal"] path = vendor/nim-nat-traversal url = https://github.com/status-im/nim-nat-traversal.git diff --git a/scripts/process_lfs.sh b/scripts/process_lfs.sh index 4f63d5570..0760ee259 100755 --- a/scripts/process_lfs.sh +++ b/scripts/process_lfs.sh @@ -9,10 +9,8 @@ set -e -ARCHIVE_NAME="json_tests.tar.xz" TMP_CACHE_DIR="tmpcache" SUBREPO_DIR="tests/official/fixtures" -LFS_DIR="json_tests" # verbosity level [[ -z "$V" ]] && V=0 [[ -z "$BUILD_MSG" ]] && BUILD_MSG="Downloading LFS files" @@ -36,6 +34,9 @@ which 7z &>/dev/null && { DECOMPRESS_XZ="7z e -txz -bd -so"; COMPRESS_XZ="7z a - which xz &>/dev/null && { DECOMPRESS_XZ="xz -d -c -T 0"; COMPRESS_XZ="xz -c -T 0"; } download_lfs_files() { + [[ -z "$1" ]] && { echo "usage: download_lfs_files() subdir_name"; exit 1; } + LFS_DIR="$1" + echo -e "$BUILD_MSG" which git-lfs &>/dev/null || { echo "Error: 'git-lfs' not found. Please install the corresponding package."; exit 1; } [[ "$V" == "0" ]] && exec &>/dev/null @@ -46,8 +47,12 @@ download_lfs_files() { popd } -UPDATE_CACHE=0 -if [[ -n "${CACHE_DIR}" ]]; then +process_subdir() { + [[ -z "$1" ]] && { echo "usage: process_subdir subdir_name"; exit 1; } + LFS_DIR="$1" + ARCHIVE_NAME="${LFS_DIR}.tar.xz" + + UPDATE_CACHE=0 if [[ -e "${CACHE_DIR}/${ARCHIVE_NAME}" ]]; then # compare the archive's mtime to the date of the last commit if [[ $(stat ${STAT_FORMAT} "${CACHE_DIR}/${ARCHIVE_NAME}") -gt $(cd "${SUBREPO_DIR}"; git log --pretty=format:%cd -n 1 --date=unix "${LFS_DIR}") ]]; then @@ -71,7 +76,7 @@ if [[ -n "${CACHE_DIR}" ]]; then if [[ "${ON_MACOS}" == "1" ]]; then brew install git-lfs # this takes almost 5 minutes on Travis, so only run it if needed fi - download_lfs_files + download_lfs_files "$LFS_DIR" echo "Updating the cache." pushd "${SUBREPO_DIR}" # the archive will contain ${LFS_DIR} as its top dir @@ -79,8 +84,15 @@ if [[ -n "${CACHE_DIR}" ]]; then popd mv "${SUBREPO_DIR}/${ARCHIVE_NAME}" "${CACHE_DIR}/" fi +} + +if [[ -n "${CACHE_DIR}" ]]; then + process_subdir "json_tests_v0.8.1" + process_subdir "json_tests_v0.8.3" + else # no caching - download_lfs_files + download_lfs_files "json_tests_v0.8.1" + download_lfs_files "json_tests_v0.8.3" fi diff --git a/tests/official/fixtures b/tests/official/fixtures index de468c07c..f4fb3ab55 160000 --- a/tests/official/fixtures +++ b/tests/official/fixtures @@ -1 +1 @@ -Subproject commit de468c07c2518cf1546c4cb615418738a2918577 +Subproject commit f4fb3ab5558916ae4f2834542719f5a1555ee096 diff --git a/tests/official/fixtures_utils.nim b/tests/official/fixtures_utils_v0_8_1.nim similarity index 95% rename from tests/official/fixtures_utils.nim rename to tests/official/fixtures_utils_v0_8_1.nim index bfae18088..c5873d0ed 100644 --- a/tests/official/fixtures_utils.nim +++ b/tests/official/fixtures_utils_v0_8_1.nim @@ -13,12 +13,14 @@ export # Workaround: # - https://github.com/nim-lang/Nim/issues/11225 serialization.readValue +# Process legacy EF test format (up to 0.8.1) +# ------------------------------------------- + type # TODO: use ref object to avoid allocating # so much on the stack - pending https://github.com/status-im/nim-json-serialization/issues/3 TestConstants* = object - # TODO - 0.5.1 constants SHARD_COUNT*: int TARGET_COMMITTEE_SIZE*: int MAX_BALANCE_CHURN_QUOTIENT*: int @@ -76,7 +78,7 @@ type const FixturesDir* = currentSourcePath.rsplit(DirSep, 1)[0] / "fixtures" - JsonTestsDir* = FixturesDir / "json_tests" + JsonTestsDir* = FixturesDir / "json_tests_v0.8.1" # ####################### # Default init diff --git a/tests/official/test_fixture_bls.nim b/tests/official/test_fixture_bls.nim index 3d49f03ba..ebdc71e3b 100644 --- a/tests/official/test_fixture_bls.nim +++ b/tests/official/test_fixture_bls.nim @@ -7,13 +7,13 @@ import # Standard libs - ospaths, strutils, unittest, endians, + ospaths, unittest, endians, # Status libs blscurve, stew/byteutils, # Beacon chain internals ../../beacon_chain/spec/crypto, # Test utilities - ./fixtures_utils + ./fixtures_utils_v0_8_1 type # # TODO - but already tested in nim-blscurve @@ -65,8 +65,7 @@ proc readValue*(r: var JsonReader, a: var Domain) {.inline.} = let be_uint = hexToPaddedByteArray[8](r.readValue(string)) bigEndian64(a.addr, be_uint.unsafeAddr) -const TestFolder = currentSourcePath.rsplit(DirSep, 1)[0] -const TestsPath = "fixtures" / "json_tests" / "bls" +const TestsPath = JsonTestsDir / "bls" var blsPrivToPubTests: Tests[BLSPrivToPub] @@ -76,10 +75,10 @@ var suite "Official - BLS tests": test "Parsing the official BLS tests": - blsPrivToPubTests = parseTests(TestFolder / TestsPath / "priv_to_pub" / "priv_to_pub.json", BLSPrivToPub) - blsSignMsgTests = parseTests(TestFolder / TestsPath / "sign_msg" / "sign_msg.json", BLSSignMsg) - blsAggSigTests = parseTests(TestFolder / TestsPath / "aggregate_sigs" / "aggregate_sigs.json", BLSAggSig) - blsAggPubKeyTests = parseTests(TestFolder / TestsPath / "aggregate_pubkeys" / "aggregate_pubkeys.json", BLSAggPubKey) + blsPrivToPubTests = parseTests(TestsPath / "priv_to_pub" / "priv_to_pub.json", BLSPrivToPub) + blsSignMsgTests = parseTests(TestsPath / "sign_msg" / "sign_msg.json", BLSSignMsg) + blsAggSigTests = parseTests(TestsPath / "aggregate_sigs" / "aggregate_sigs.json", BLSAggSig) + blsAggPubKeyTests = parseTests(TestsPath / "aggregate_pubkeys" / "aggregate_pubkeys.json", BLSAggPubKey) test "Private to public key conversion": for t in blsPrivToPubTests.test_cases: diff --git a/tests/official/test_fixture_shuffling.nim b/tests/official/test_fixture_shuffling.nim index b0c25c247..2c44c883c 100644 --- a/tests/official/test_fixture_shuffling.nim +++ b/tests/official/test_fixture_shuffling.nim @@ -7,12 +7,12 @@ import # Standard library - ospaths, strutils, unittest, + ospaths, unittest, # Beacon chain internals ../../beacon_chain/spec/[datatypes, validator, digest], # Test utilities ../testutil, - ./fixtures_utils + ./fixtures_utils_v0_8_1 type Shuffling* = object @@ -20,18 +20,16 @@ type count*: uint64 shuffled*: seq[ValidatorIndex] -const TestFolder = currentSourcePath.rsplit(DirSep, 1)[0] - when const_preset == "mainnet": - const TestsPath = "fixtures" / "json_tests" / "shuffling" / "core" / "shuffling_full.json" + const TestsPath = JsonTestsDir / "shuffling" / "core" / "shuffling_full.json" elif const_preset == "minimal": - const TestsPath = "fixtures" / "json_tests" / "shuffling" / "core" / "shuffling_minimal.json" + const TestsPath = JsonTestsDir / "shuffling" / "core" / "shuffling_minimal.json" var shufflingTests: Tests[Shuffling] suite "Official - Shuffling tests [Preset: " & preset(): test "Parsing the official shuffling tests [Preset: " & preset(): - shufflingTests = parseTests(TestFolder / TestsPath, Shuffling) + shufflingTests = parseTests(TestsPath, Shuffling) test "Shuffling a sequence of N validators" & preset(): for t in shufflingTests.test_cases: diff --git a/tests/official/test_fixture_ssz_static.nim b/tests/official/test_fixture_ssz_static.nim index 085312855..c4f8bc107 100644 --- a/tests/official/test_fixture_ssz_static.nim +++ b/tests/official/test_fixture_ssz_static.nim @@ -17,7 +17,7 @@ import ../../beacon_chain/spec/[datatypes, validator, digest, crypto], # Test utilities ../testutil, - ./fixtures_utils + ./fixtures_utils_v0_8_1 const failFast = defined(debug) and false diff --git a/tests/official/test_fixture_ssz_uint.nim b/tests/official/test_fixture_ssz_uint.nim index 18a488292..b42f38aaa 100644 --- a/tests/official/test_fixture_ssz_uint.nim +++ b/tests/official/test_fixture_ssz_uint.nim @@ -15,7 +15,7 @@ import ../../beacon_chain/spec/[datatypes, validator], # Test utilities ../testutil, - ./fixtures_utils + ./fixtures_utils_v0_8_1 type SSZUint* = object