diff --git a/.appveyor.yml b/.appveyor.yml index 3e05fd5c4..7d74c1d9a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,6 +3,10 @@ version: '{build}' # use an image with recent Mingw-w64 versions available on both architectures: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin image: Visual Studio 2015 +environment: + # disable LFS file downloading during regular cloning + GIT_LFS_SKIP_SMUDGE: 1 + cache: - NimBinaries diff --git a/.travis.yml b/.travis.yml index 9828e5489..de7cc25bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ cache: git: # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" depth: 10 + # disable LFS file downloading during regular cloning + lfs_skip_smudge: true matrix: include: diff --git a/Makefile b/Makefile index d9d73402b..8ef08864d 100644 --- a/Makefile +++ b/Makefile @@ -28,15 +28,6 @@ ENV_SCRIPT := "$(CURDIR)/env.sh" # duplicated in "env.sh" to prepend NIM_DIR/bin to PATH NIM_DIR := vendor/Nim -#- forces a rebuild of csources, Nimble and a complete compiler rebuild, in case we're called after pulling a new Nim version -#- uses our Git submodules for csources and Nimble (Git doesn't let us place them in another submodule) -#- build_all.sh looks at the parent dir to decide whether to copy the resulting csources binary there, -# but this is broken when using symlinks, so build csources separately (we get parallel compiling as a bonus) -#- Windows is a special case, as usual -#- macOS is also a special case, with its "ln" not supporting "-r" -#- the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection with ARCH_OVERRIDE -BUILD_NIM := echo -e $(BUILD_MSG) "Nim compiler" && \ - V=$(V) CC=$(CC) MAKE=$(MAKE) ARCH_OVERRIDE=$(ARCH_OVERRIDE) "$(CURDIR)/build_nim.sh" "$(NIM_DIR)" ../Nim-csources ../nimble ifeq ($(OS), Windows_NT) EXE_SUFFIX := .exe else @@ -49,7 +40,7 @@ ifeq ($(shell uname), Darwin) else MD5SUM := md5sum endif -# AppVeyor cache of $(NIM_DIR)/bin that doesn't get unpacked after all submodules have been checked out, like on Travis +# AppVeyor/Travis cache of $(NIM_DIR)/bin CI_CACHE := # debugging tools + testing tools @@ -154,33 +145,34 @@ github-ssh: git config url."git@github.com:".insteadOf "https://github.com/" git submodule foreach --recursive 'git config url."git@github.com:".insteadOf "https://github.com/"' -#- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary) +#- conditionally re-builds the Nim compiler (not usually needed, because `make update` calls this rule; delete $(NIM_BINARY) to force it) #- allows parallel building with the '+' prefix -build-nim: | deps - + $(BUILD_NIM) +#- forces a rebuild of csources, Nimble and a complete compiler rebuild, in case we're called after pulling a new Nim version +#- uses our Git submodules for csources and Nimble (Git doesn't let us place them in another submodule) +#- build_all.sh looks at the parent dir to decide whether to copy the resulting csources binary there, +# but this is broken when using symlinks, so build csources separately (we get parallel compiling as a bonus) +#- Windows is a special case, as usual +#- macOS is also a special case, with its "ln" not supporting "-r" +#- the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection with ARCH_OVERRIDE +build-nim: | sanity-checks + + NIM_BUILD_MSG="$(BUILD_MSG) Nim compiler" \ + V=$(V) \ + CC=$(CC) \ + MAKE=$(MAKE) \ + ARCH_OVERRIDE=$(ARCH_OVERRIDE) \ + "$(CURDIR)/build_nim.sh" "$(NIM_DIR)" ../Nim-csources ../nimble "$(CI_CACHE)" #- initialises and updates the Git submodules #- manages the AppVeyor cache of Nim compiler binaries #- deletes the ".nimble" dir to force the execution of the "deps" target #- deletes and recreates "nimbus.nims" which on Windows is a copy instead of a proper symlink #- allows parallel building with the '+' prefix -#- rebuilds the Nim compiler after the corresponding submodule is updated (keep in mind that Git doesn't preserve file timestamps) +#- rebuilds the Nim compiler if the corresponding submodule is updated $(NIM_BINARY) update: | sanity-checks git submodule update --init --recursive - [[ -n "$(CI_CACHE)" && -d "$(CI_CACHE)" ]] && \ - cp -a "$(CI_CACHE)"/* $(NIM_DIR)/bin/ || \ - true rm -rf $(NIMBLE_DIR) nimbus.nims && \ $(MAKE) nimbus.nims - + "$(CURDIR)/nim_needs_rebuilding.sh" "$(NIM_DIR)" $(NIM_BINARY) && \ - { \ - $(BUILD_NIM); \ - [[ -n "$(CI_CACHE)" ]] && \ - rm -rf "$(CI_CACHE)" && \ - mkdir $(CI_CACHE) && \ - cp -a $(NIM_DIR)/bin/* "$(CI_CACHE)"/ || \ - true; \ - } || true + + $(MAKE) build-nim # don't use this target, or you risk updating dependency repos that are not ready to be used in Nimbus update-remote: diff --git a/README.md b/README.md index 6d92d59b4..900e2b502 100644 --- a/README.md +++ b/README.md @@ -148,12 +148,19 @@ make LOG_LEVEL=TRACE nimbus # log everything make NIMFLAGS="-d:release" ``` -- if you want to use SSH keys with GitHub: +- if you want to use SSH keys with GitHub (also handles submodules): ```bash make github-ssh ``` +- force a Nim compiler rebuild: + +```bash +rm vendor/Nim/bin/nim +make -j8 build-nim +``` + #### Git submodule workflow Working on a dependency: diff --git a/build_nim.sh b/build_nim.sh index 1bc287be9..b56fe3869 100755 --- a/build_nim.sh +++ b/build_nim.sh @@ -7,77 +7,120 @@ CSOURCES_COMMIT="b56e49bbedf62db22eb26388f98262e2948b2cbc" # 0.19.0 NIMBLE_COMMIT="c8d79fc0228682677330a9f57d14389aaa641153" # Mar 26 10:06:06 2019 # script arguments -[[ $# -ne 3 ]] && { echo "usage: $0 nim_dir csources_dir nimble_dir"; exit 1; } +[[ $# -ne 4 ]] && { echo "usage: $0 nim_dir csources_dir nimble_dir ci_cache_dir"; exit 1; } NIM_DIR="$1" CSOURCES_DIR="$2" NIMBLE_DIR="$3" +CI_CACHE="$4" ## env vars # verbosity level [[ -z "$V" ]] && V=0 -[[ "$V" == "0" ]] && exec &>/dev/null [[ -z "$CC" ]] && CC="gcc" # to build csources in parallel, set MAKE="make -jN" [[ -z "$MAKE" ]] && MAKE="make" # for 32-bit binaries on a 64-bit Windows host UCPU="" [[ "$ARCH_OVERRIDE" == "x86" ]] && UCPU="ucpu=i686" +[[ -z "$NIM_BUILD_MSG" ]] && NIM_BUILD_MSG="Building the Nim compiler" # Windows detection -ON_WINDOWS=0 -uname | grep -qi mingw && ON_WINDOWS=1 +if uname | grep -qi mingw; then + ON_WINDOWS=1 + EXE_SUFFIX=".exe" +else + ON_WINDOWS=0 + EXE_SUFFIX="" +fi -# working directory -cd "$NIM_DIR" +NIM_BINARY="${NIM_DIR}/bin/nim${EXE_SUFFIX}" -# Git repos for csources and Nimble -[[ -d "$CSOURCES_DIR" ]] || { \ - mkdir -p "$CSOURCES_DIR" && \ - cd "$CSOURCES_DIR" && \ - git clone https://github.com/nim-lang/csources.git . && \ - git checkout $CSOURCES_COMMIT && \ - cd - >/dev/null; \ -} -[[ "$CSOURCES_DIR" != "csources" ]] && \ -rm -rf csources && \ -ln -s "$CSOURCES_DIR" csources +nim_needs_rebuilding() { + REBUILD=0 + NO_REBUILD=1 -# we have to delete .git or koch.nim will checkout a branch tip -[[ -d "$NIMBLE_DIR" ]] || { \ - mkdir -p "$NIMBLE_DIR" && \ - cd "$NIMBLE_DIR" && \ - git clone https://github.com/nim-lang/nimble.git . && \ - git checkout $NIMBLE_COMMIT && \ - rm -rf .git && \ - cd - >/dev/null; \ -} -[[ "$NIMBLE_DIR" != "dist/nimble" ]] && \ -mkdir -p dist && \ -rm -rf dist/nimble && \ -ln -s ../"$NIMBLE_DIR" dist/nimble + if [[ -n "$CI_CACHE" && -d "$CI_CACHE" ]]; then + if [[ ! -e "$NIM_DIR" ]]; then + git clone --depth=1 https://github.com/status-im/Nim.git "$NIM_DIR" + fi + cp -a "$CI_CACHE"/* "$NIM_DIR"/bin/ + fi -# bootstrap the Nim compiler and build the tools -rm -rf bin/nim_csources && \ -cd csources && { \ - [[ "$ON_WINDOWS" == "0" ]] && { \ - $MAKE clean && \ - $MAKE LD=$CC; \ - } || { \ - $MAKE myos=windows $UCPU clean && \ - $MAKE myos=windows $UCPU CC=gcc LD=gcc; \ - }; \ -} && \ -cd - >/dev/null && { \ - [ -e csources/bin ] && { \ - cp -a csources/bin/nim bin/nim && \ - cp -a csources/bin/nim bin/nim_csources && \ - rm -rf csources/bin; \ - } || { \ - cp -a bin/nim bin/nim_csources; \ - }; \ -} && { \ - sed 's/koch tools/koch --stable tools/' build_all.sh > build_all_custom.sh; \ - sh build_all_custom.sh; \ - rm build_all_custom.sh; \ + # compare binary mtime to the date of the last commit (keep in mind that Git doesn't preserve file timestamps) + if [[ -e "$NIM_BINARY" && $(stat -c%Y "$NIM_BINARY") -gt $(cd "$NIM_DIR"; git log --pretty=format:%cd -n 1 --date=unix) ]]; then + return $NO_REBUILD + else + return $REBUILD + fi } +build_nim() { + echo -e "$NIM_BUILD_MSG" + [[ "$V" == "0" ]] && exec &>/dev/null + + # working directory + cd "$NIM_DIR" + + # Git repos for csources and Nimble + if [[ ! -d "$CSOURCES_DIR" ]]; then + mkdir -p "$CSOURCES_DIR" + cd "$CSOURCES_DIR" + git clone https://github.com/nim-lang/csources.git . + git checkout $CSOURCES_COMMIT + cd - >/dev/null + fi + if [[ "$CSOURCES_DIR" != "csources" ]]; then + rm -rf csources + ln -s "$CSOURCES_DIR" csources + fi + + if [[ ! -d "$NIMBLE_DIR" ]]; then + mkdir -p "$NIMBLE_DIR" + cd "$NIMBLE_DIR" + git clone https://github.com/nim-lang/nimble.git . + git checkout $NIMBLE_COMMIT + # we have to delete .git or koch.nim will checkout a branch tip, overriding our target commit + rm -rf .git + cd - >/dev/null + fi + if [[ "$NIMBLE_DIR" != "dist/nimble" ]]; then + mkdir -p dist + rm -rf dist/nimble + ln -s ../"$NIMBLE_DIR" dist/nimble + fi + + # bootstrap the Nim compiler and build the tools + rm -rf bin/nim_csources + cd csources + if [[ "$ON_WINDOWS" == "0" ]]; then + $MAKE clean + $MAKE LD=$CC + else + $MAKE myos=windows $UCPU clean + $MAKE myos=windows $UCPU CC=gcc LD=gcc + fi + cd - >/dev/null + if [[ -e csources/bin ]]; then + cp -a csources/bin/nim bin/nim + cp -a csources/bin/nim bin/nim_csources + rm -rf csources/bin + else + cp -a bin/nim bin/nim_csources + fi + sed 's/koch tools/koch --stable tools/' build_all.sh > build_all_custom.sh + sh build_all_custom.sh + rm build_all_custom.sh + + # update the CI cache + cd - >/dev/null # we were in $NIM_DIR + if [[ -n "$CI_CACHE" ]]; then + rm -rf "$CI_CACHE" + mkdir "$CI_CACHE" + cp -a "$NIM_DIR"/bin/* "$CI_CACHE"/ + fi +} + +if nim_needs_rebuilding; then + build_nim +fi + diff --git a/nim_needs_rebuilding.sh b/nim_needs_rebuilding.sh deleted file mode 100755 index a1553597b..000000000 --- a/nim_needs_rebuilding.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# this one is required -set -e - -# script arguments -[[ $# -ne 2 ]] && { echo "usage: $0 nim_dir nim_binary"; exit 1; } -NIM_DIR="$1" -NIM_BINARY="$2" - -# compare binary mtime to the date of the last commit -! [[ -e $NIM_BINARY && $(stat -c%Y $NIM_BINARY) -gt $(cd "$NIM_DIR"; git log --pretty=format:%cd -n 1 --date=unix) ]] -