Merge branch 'build' into devel

.appveyor.yml

@@ -9,7 +9,6 @@ environment:
 
 cache:
   - NimBinaries
-  - jsonTestsCache
 
 matrix:
   # We always want 32-bit and 64-bit compilation
@@ -23,12 +22,12 @@ platform:
 clone_depth: 10
 
 install:
+  - git submodule update --init --recursive
   # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
   - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
   - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
 
 build_script:
-  #- cd C:\projects\%APPVEYOR_PROJECT_SLUG%
   # the 32-bit build is done on a 64-bit image, so we need to override the architecture
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% CI_CACHE=NimBinaries update
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% fetch-dlls
@@ -39,10 +38,6 @@ test_script:
   - build\nimbus.exe --help
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% test
   - IF "%PLATFORM%" == "x64" mingw32-make -j2 test-reproducibility
-  # LFS test fixtures
-  - cd vendor\nim-beacon-chain & bash scripts\process_lfs.sh ..\..\jsonTestsCache & cd ..\..
-  - mingw32-make -C vendor/nim-beacon-chain DISABLE_LFS_SCRIPT=1 -j2 test
-  #- mingw32-make go-checks
 
 deploy: off
 

.gitmodules

@@ -152,3 +152,8 @@
 	url = https://github.com/flyx/NimYAML.git
 	ignore = dirty
 	branch = master
+[submodule "vendor/nimbus-build-system"]
+	path = vendor/nimbus-build-system
+	url = https://github.com/status-im/nimbus-build-system.git
+	ignore = dirty
+	branch = master

.travis.yml

@@ -1,11 +1,10 @@
-language: go
+language: c
 
 # https://docs.travis-ci.com/user/caching/
 cache:
   directories:
     - rocksdbCache
     - vendor/Nim/bin
-    - jsonTestsCache
 
 git:
   # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X"
@@ -13,9 +12,6 @@ git:
   # disable LFS file downloading during regular cloning
   lfs_skip_smudge: true
 
-go:
-  - "1.12.x"
-
 matrix:
   include:
     - os: linux
@@ -28,16 +24,11 @@ matrix:
 
 install:
   # build our own rocksdb to test with a fixed version that we think works
-  - ./build_rocksdb.sh rocksdbCache
+  - vendor/nimbus-build-system/scripts/build_rocksdb.sh rocksdbCache
-
-  # LFS test fixtures
-  - cd vendor/nim-beacon-chain; scripts/process_lfs.sh ../../jsonTestsCache; cd ../..
 
 script:
   - set -e # fail fast
   - make -j2
-  - ./build/nimbus --help
+  - build/nimbus --help
   - make -j2 test test-reproducibility
-  - make -C vendor/nim-beacon-chain DISABLE_LFS_SCRIPT=1 -j2 test
-  - make go-checks
 

Makefile

@@ -5,43 +5,11 @@
 # at your option. This file may not be copied, modified, or distributed except
 # according to those terms.
 
-# common parts
+# used inside the included makefiles
-include common.mk
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 
-GIT_CLONE := git clone --quiet --recurse-submodules
+# we don't want an error here, so we can handle things later, in the build-system-checks target
-GIT_PULL := git pull --recurse-submodules
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
-GIT_STATUS := git status
-#- the Nimble dir can't be "[...]/vendor", or Nimble will start looking for
-#  version numbers in repo dirs (because those would be in its subdirectories)
-#- duplicated in "env.sh" for the env var with the same name
-NIMBLE_DIR := vendor/.nimble
-REPOS_DIR := vendor
-ifeq ($(OS), Windows_NT)
-  PWD := pwd -W
-else
-  PWD := pwd
-endif
-# we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes
-RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$($(PWD)):\e[39m"; $(CMD)
-# absolute path, since it will be run at various subdirectory depths
-ENV_SCRIPT := "$(CURDIR)/env.sh"
-# duplicated in "env.sh" to prepend NIM_DIR/bin to PATH
-NIM_DIR := vendor/Nim
-
-ifeq ($(OS), Windows_NT)
-  EXE_SUFFIX := .exe
-else
-  EXE_SUFFIX :=
-endif
-NIM_BINARY := $(NIM_DIR)/bin/nim$(EXE_SUFFIX)
-# md5sum - macOS is a special case
-ifeq ($(shell uname), Darwin)
-  MD5SUM := md5 -r
-else
-  MD5SUM := md5sum
-endif
-# AppVeyor/Travis cache of $(NIM_DIR)/bin
-CI_CACHE :=
 
 # debugging tools + testing tools
 TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTestGen
@@ -49,15 +17,29 @@ TOOLS_DIRS := premix tests
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 
-.PHONY: all $(TOOLS) deps sanity-checks github-ssh build-nim update status ntags ctags nimbus testsuite test clean mrproper fetch-dlls test-libp2p-daemon nat-libs libminiupnpc.a libnatpmp.a go-checks libnimbus.so libnimbus.a wrappers
+.PHONY: all $(TOOLS) build-system-checks deps update nimbus test test-reproducibility clean libnimbus.so libnimbus.a wrappers wrappers-static
 
 # default target, because it's the first one that doesn't start with '.'
-all: $(TOOLS) nimbus
+all: build-system-checks $(TOOLS) nimbus
 
-#- when the special ".SILENT" target is present, all recipes are silenced as if they all had a "@" prefix
+# must be included after the default target
-#- by setting SILENT_TARGET_PREFIX to a non-empty value, the name of this target becomes meaningless to `make`
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
-#- idea stolen from http://make.mad-scientist.net/managing-recipe-echoing/
+
-$(SILENT_TARGET_PREFIX).SILENT:
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+build-system-checks:
+	@[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \
+		echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo -e "\nYou can now run '$(MAKE)' again."; \
+		exit 1; \
+		}
+
+deps: | deps-common nimbus.nims
+
+#- deletes and recreates "nimbus.nims" which on Windows is a copy instead of a proper symlink
+update: | update-common
+	rm -rf nimbus.nims && \
+		$(MAKE) nimbus.nims
 
 # builds the tools, wherever they are
 $(TOOLS): | build deps nat-libs
@@ -70,44 +52,6 @@ nimbus: | build deps nat-libs
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim nimbus $(NIM_PARAMS) nimbus.nims
 
-# dir
-build:
-	mkdir $@
-
-#- runs only the first time and after `make update`, so have "normal"
-#  (timestamp-checked) prerequisites here
-#- $(NIM_BINARY) is both a proxy for submodules having been initialised
-#  and a check for the actual compiler build
-deps: sanity-checks $(NIM_BINARY) $(NIMBLE_DIR) nimbus.nims
-
-sanity-checks:
-	which $(CC) &>/dev/null || { echo "C compiler ($(CC)) not installed. Aborting."; exit 1; }
-
-nat-libs: | libminiupnpc.a libnatpmp.a
-
-libminiupnpc.a: | deps
-ifeq ($(OS), Windows_NT)
-	+ [ -e vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/$@ ] || \
-		$(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc -f Makefile.mingw CC=gcc init $@ $(HANDLE_OUTPUT)
-else
-	+ $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc $@ $(HANDLE_OUTPUT)
-endif
-
-libnatpmp.a: | deps
-ifeq ($(OS), Windows_NT)
-	+ $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp CC=gcc CFLAGS="-Wall -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR" $@ $(HANDLE_OUTPUT)
-else
-	+ $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp $@ $(HANDLE_OUTPUT)
-endif
-
-#- depends on Git submodules being initialised
-#- fakes a Nimble package repository with the minimum info needed by the Nim compiler
-#  for runtime path (i.e.: the second line in $(NIMBLE_DIR)/pkgs/*/*.nimble-link)
-$(NIMBLE_DIR): | $(NIM_BINARY)
-	mkdir -p $(NIMBLE_DIR)/pkgs
-	NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" PWD_CMD="$(PWD)" \
-		git submodule foreach --quiet 'bash $(CURDIR)/scripts/create_nimble_link.sh "$$sm_path"'
-
 # symlink
 nimbus.nims:
 	ln -s nimbus.nimble $@
@@ -127,78 +71,8 @@ test-reproducibility:
 			{ echo -e "\e[91mFailure: the binary changed between builds.\e[39m"; exit 1; }
 
 # usual cleaning
-clean:
+clean: | clean-common
-	rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc,*.exe,*.so,*.so.0,*_wrapper_test} vendor/go/bin \
+	rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc,*_wrapper_test}
-		$(NIMBLE_DIR) $(NIM_BINARY) $(NIM_DIR)/nimcache nimcache
-	+ $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc clean $(HANDLE_OUTPUT)
-	+ $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp clean $(HANDLE_OUTPUT)
-
-# dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete
-mrproper: clean
-	rm -rf vendor
-
-# for when you want to use SSH keys with GitHub
-github-ssh:
-	git config url."git@github.com:".insteadOf "https://github.com/"
-	git submodule foreach --recursive 'git config url."git@github.com:".insteadOf "https://github.com/"'
-
-#- conditionally re-builds the Nim compiler (not usually needed, because `make update` calls this rule; delete $(NIM_BINARY) to force it)
-#- allows parallel building with the '+' prefix
-#- forces a rebuild of csources, Nimble and a complete compiler rebuild, in case we're called after pulling a new Nim version
-#- uses our Git submodules for csources and Nimble (Git doesn't let us place them in another submodule)
-#- build_all.sh looks at the parent dir to decide whether to copy the resulting csources binary there,
-#  but this is broken when using symlinks, so build csources separately (we get parallel compiling as a bonus)
-#- Windows is a special case, as usual
-#- macOS is also a special case, with its "ln" not supporting "-r"
-#- the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection with ARCH_OVERRIDE
-build-nim: | sanity-checks
-	+ NIM_BUILD_MSG="$(BUILD_MSG) Nim compiler" \
-		V=$(V) \
-		CC=$(CC) \
-		MAKE=$(MAKE) \
-		ARCH_OVERRIDE=$(ARCH_OVERRIDE) \
-		"$(CURDIR)/build_nim.sh" "$(NIM_DIR)" ../Nim-csources ../nimble "$(CI_CACHE)"
-
-#- in case of submodule URL changes, it propagates that change in the parent repo's .git directory
-#- initialises and updates the Git submodules, avoiding automated LFS downloads
-#- manages the AppVeyor cache of Nim compiler binaries
-#- deletes the ".nimble" dir to force the execution of the "deps" target
-#- deletes and recreates "nimbus.nims" which on Windows is a copy instead of a proper symlink
-#- allows parallel building with the '+' prefix
-#- rebuilds the Nim compiler if the corresponding submodule is updated
-$(NIM_BINARY) update: | sanity-checks
-	git submodule sync --quiet --recursive
-	export GIT_LFS_SKIP_SMUDGE=1; git submodule update --init --recursive
-	rm -rf $(NIMBLE_DIR) nimbus.nims && \
-		$(MAKE) nimbus.nims
-	+ $(MAKE) build-nim
-
-# don't use this target, or you risk updating dependency repos that are not ready to be used in Nimbus
-update-remote:
-	git submodule update --remote
-
-# runs `git status` in all Git repos
-status: | $(REPOS)
-	$(eval CMD := $(GIT_STATUS))
-	$(RUN_CMD_IN_ALL_REPOS)
-
-MIN_GO_VER := 1.12.0
-go-checks:
-	which go &>/dev/null || { echo "Go compiler not installed. Aborting."; exit 1; }
-	GO_VER="$$(go version | sed -E 's/^.*go([0-9.]+).*$$/\1/')"; \
-	       [[ $$(echo -e "$${GO_VER}\n$(MIN_GO_VER)" | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -g | head -n 1) == "$(MIN_GO_VER)" ]] || \
-	       { echo "Minimum Go compiler version required: $(MIN_GO_VER). Version available: $$GO_VER. Aborting."; exit 1; }
-
-vendor/go/bin/p2pd: | go-checks
-	echo -e $(BUILD_MSG) "$@" && \
-		cd vendor/go/src/github.com/libp2p/go-libp2p-daemon && \
-		$(ENV_SCRIPT) go get ./... && \
-		$(ENV_SCRIPT) go install ./...
-
-test-libp2p-daemon: | vendor/go/bin/p2pd deps
-	cd vendor/nim-libp2p && \
-		$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) tests/testdaemon.nim && \
-		rm -f tests/testdaemon
 
 libnimbus.so: | build deps nat-libs
 	echo -e $(BUILD_MSG) "build/$@" && \
@@ -223,81 +97,3 @@ wrappers-static: | build deps nat-libs libnimbus.a go-checks
 	echo -e $(BUILD_MSG) "build/go_wrapper_example_static" && \
 		go build -ldflags "-linkmode external -extldflags '-static -ldl'" -o build/go_wrapper_example_static wrappers/wrapper_example.go
 
-# https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory"
-ntags:
-	ntags -R .
-
-#- a few files need to be excluded because they trigger an infinite loop in https://github.com/universal-ctags/ctags
-#- limiting it to Nim files, because there are a lot of C files we don't care about
-ctags:
-	ctags -R --verbose=yes \
-	--langdef=nim \
-	--langmap=nim:.nim \
-	--regex-nim='/(\w+)\*?\s*=\s*object/\1/c,class/' \
-	--regex-nim='/(\w+)\*?\s*=\s*enum/\1/e,enum/' \
-	--regex-nim='/(\w+)\*?\s*=\s*tuple/\1/t,tuple/' \
-	--regex-nim='/(\w+)\*?\s*=\s*range/\1/s,subrange/' \
-	--regex-nim='/(\w+)\*?\s*=\s*proc/\1/p,proctype/' \
-	--regex-nim='/proc\s+(\w+)/\1/f,procedure/' \
-	--regex-nim='/func\s+(\w+)/\1/f,procedure/' \
-	--regex-nim='/method\s+(\w+)/\1/m,method/' \
-	--regex-nim='/proc\s+`([^`]+)`/\1/o,operator/' \
-	--regex-nim='/template\s+(\w+)/\1/u,template/' \
-	--regex-nim='/macro\s+(\w+)/\1/v,macro/' \
-	--languages=nim \
-	--exclude=nimcache \
-	--exclude='*/Nim/tinyc' \
-	--exclude='*/Nim/tests' \
-	--exclude='*/Nim/csources' \
-	--exclude=nimbus/genesis_alloc.nim \
-	--exclude=$(REPOS_DIR)/nim-bncurve/tests/tvectors.nim \
-	.
-
-############################
-# Windows-specific section #
-############################
-
-ifeq ($(OS), Windows_NT)
-  # no tabs allowed for indentation here
-
-  # the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection
-  ifeq ($(ARCH_OVERRIDE), x86)
-    ARCH := x86
-  else
-    ifeq ($(PROCESSOR_ARCHITEW6432), AMD64)
-      ARCH := x64
-    else
-      ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
-        ARCH := x64
-      endif
-      ifeq ($(PROCESSOR_ARCHITECTURE), x86)
-        ARCH := x86
-      endif
-    endif
-  endif
-
-  ifeq ($(ARCH), x86)
-    ROCKSDB_DIR := x86
-  endif
-  ifeq ($(ARCH), x64)
-    ROCKSDB_DIR := x64
-  endif
-
-  ROCKSDB_ARCHIVE := nimbus-deps.zip
-  ROCKSDB_URL := https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/$(ROCKSDB_ARCHIVE)
-  CURL := curl -O -L
-  UNZIP := unzip -o
-
-#- back to tabs
-#- copied from .appveyor.yml
-#- this is why we can't delete the whole "build" dir in the "clean" target
-fetch-dlls: | build deps
-	cd build && \
-		$(CURL) $(ROCKSDB_URL) && \
-		$(CURL) https://nim-lang.org/download/dlls.zip && \
-		$(UNZIP) $(ROCKSDB_ARCHIVE) && \
-		cp -a $(ROCKSDB_DIR)/*.dll . && \
-		$(UNZIP) dlls.zip && \
-		mkdir -p ../vendor/nim-beacon-chain/build && \
-		cp -a *.dll ../vendor/nim-beacon-chain/build/
-endif

README.md

@@ -89,7 +89,7 @@ nix-shell default.nix
 To build Nimbus (in "build/nimbus"), just execute:
 
 ```bash
-make
+make # the first invocation will update all Git submodules and prompt you to run "make" again
 ```
 
 Running `./build/nimbus --help` will provide you with a list of
@@ -133,6 +133,7 @@ Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell
 
 If you don't want to compile RocksDB and SQLite separately, you can fetch pre-compiled DLLs with:
 ```bash
+mingw32-make # this first invocation will update the Git submodules
 mingw32-make fetch-dlls
 ```
 
@@ -258,7 +259,11 @@ latest commits when they are not ready to be used in the superproject.
 Adding the submodule "https://github.com/status-im/foo" to "vendor/foo":
 
 ```bash
-./add_submodule.sh status-im/foo
+vendor/nimbus-build-system/scripts/add_submodule.sh status-im/foo
+# or
+./env.sh add_submodule status-im/foo
+# want to place it in "vendor/bar" instead?
+./env.sh add_submodule status-im/foo vendor/bar
 ```
 
 Removing the submodule "vendor/bar":
@@ -281,8 +286,10 @@ convinced not to run a dependency check, thus clashing with our jury-rigged
 "vendor/.nimble/pkgs"):
 
 ```bash
-cd vendor/nim-blscurve
+cd vendor/nim-rocksdb
-../../nimble.sh test
+../nimbus-build-system/scripts/nimble.sh test
+# or
+../../env.sh nimble test
 ```
 
 ### Troubleshooting

add_submodule.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-[ -z "$1" -o `echo "$1" | tr '/' '\n' | wc -l` != 2 ] && \
-	{ echo "usage: `basename $0` some/repo [destdir] # 'destdir' defaults to 'vendor/repo'"; exit 1; }
-REPO="$1"
-
-DEST="vendor/${REPO#*/}"
-[ -n "$2" ] && DEST="$2"
-
-git submodule add --force https://github.com/${REPO}.git "$DEST"
-git config -f .gitmodules submodule.${DEST}.ignore dirty
-git config -f .gitmodules submodule.${DEST}.branch master
-

build_nim.sh

@@ -1,134 +0,0 @@
-#!/bin/bash
-# used in Travis CI and AppVeyor scripts
-
-set -e
-
-# Git commits
-CSOURCES_COMMIT="b56e49bbedf62db22eb26388f98262e2948b2cbc" # 0.19.0
-NIMBLE_COMMIT="c8d79fc0228682677330a9f57d14389aaa641153" # Mar 26 10:06:06 2019
-
-# script arguments
-[[ $# -ne 4 ]] && { echo "usage: $0 nim_dir csources_dir nimble_dir ci_cache_dir"; exit 1; }
-NIM_DIR="$1"
-CSOURCES_DIR="$2"
-NIMBLE_DIR="$3"
-CI_CACHE="$4"
-
-## env vars
-# verbosity level
-[[ -z "$V" ]] && V=0
-[[ -z "$CC" ]] && CC="gcc"
-# to build csources in parallel, set MAKE="make -jN"
-[[ -z "$MAKE" ]] && MAKE="make"
-# for 32-bit binaries on a 64-bit Windows host
-UCPU=""
-[[ "$ARCH_OVERRIDE" == "x86" ]] && UCPU="ucpu=i686"
-[[ -z "$NIM_BUILD_MSG" ]] && NIM_BUILD_MSG="Building the Nim compiler"
-
-# Windows detection
-if uname | grep -qiE "mingw|msys"; then
-	ON_WINDOWS=1
-	EXE_SUFFIX=".exe"
-else
-	ON_WINDOWS=0
-	EXE_SUFFIX=""
-fi
-# macOS
-if uname | grep -qi "darwin"; then
-	STAT_FORMAT="-f %m"
-else
-	STAT_FORMAT="-c %Y"
-fi
-
-NIM_BINARY="${NIM_DIR}/bin/nim${EXE_SUFFIX}"
-
-nim_needs_rebuilding() {
-	REBUILD=0
-	NO_REBUILD=1
-
-	if [[ ! -e "$NIM_DIR" ]]; then
-		git clone --depth=1 https://github.com/status-im/Nim.git "$NIM_DIR"
-	fi
-
-	if [[ -n "$CI_CACHE" && -d "$CI_CACHE" ]]; then
-		cp -a "$CI_CACHE"/* "$NIM_DIR"/bin/ || true # let this one fail with an empty cache dir
-	fi
-
-	# compare binary mtime to the date of the last commit (keep in mind that Git doesn't preserve file timestamps)
-	if [[ -e "$NIM_BINARY" && $(stat $STAT_FORMAT "$NIM_BINARY") -gt $(cd "$NIM_DIR"; git log --pretty=format:%cd -n 1 --date=unix) ]]; then
-		return $NO_REBUILD
-	else
-		return $REBUILD
-	fi
-}
-
-build_nim() {
-	echo -e "$NIM_BUILD_MSG"
-	[[ "$V" == "0" ]] && exec &>/dev/null
-
-	# working directory
-	pushd "$NIM_DIR"
-
-	# Git repos for csources and Nimble
-	if [[ ! -d "$CSOURCES_DIR" ]]; then
-		mkdir -p "$CSOURCES_DIR"
-		pushd "$CSOURCES_DIR"
-		git clone https://github.com/nim-lang/csources.git .
-		git checkout $CSOURCES_COMMIT
-		popd
-	fi
-	if [[ "$CSOURCES_DIR" != "csources" ]]; then
-		rm -rf csources
-		ln -s "$CSOURCES_DIR" csources
-	fi
-
-	if [[ ! -d "$NIMBLE_DIR" ]]; then
-		mkdir -p "$NIMBLE_DIR"
-		pushd "$NIMBLE_DIR"
-		git clone https://github.com/nim-lang/nimble.git .
-		git checkout $NIMBLE_COMMIT
-		# we have to delete .git or koch.nim will checkout a branch tip, overriding our target commit
-		rm -rf .git
-		popd
-	fi
-	if [[ "$NIMBLE_DIR" != "dist/nimble" ]]; then
-		mkdir -p dist
-		rm -rf dist/nimble
-		ln -s ../"$NIMBLE_DIR" dist/nimble
-	fi
-
-	# bootstrap the Nim compiler and build the tools
-	rm -rf bin/nim_csources
-	pushd csources
-	if [[ "$ON_WINDOWS" == "0" ]]; then
-		$MAKE clean
-		$MAKE LD=$CC
-	else
-		$MAKE myos=windows $UCPU clean
-		$MAKE myos=windows $UCPU CC=gcc LD=gcc
-	fi
-	popd
-	if [[ -e csources/bin ]]; then
-		cp -a csources/bin/nim bin/nim
-		cp -a csources/bin/nim bin/nim_csources
-		rm -rf csources/bin
-	else
-		cp -a bin/nim bin/nim_csources
-	fi
-	sed 's/koch tools/koch --stable tools/' build_all.sh > build_all_custom.sh
-	sh build_all_custom.sh
-	rm build_all_custom.sh
-
-	# update the CI cache
-	popd # we were in $NIM_DIR
-	if [[ -n "$CI_CACHE" ]]; then
-		rm -rf "$CI_CACHE"
-		mkdir "$CI_CACHE"
-		cp -a "$NIM_DIR"/bin/* "$CI_CACHE"/
-	fi
-}
-
-if nim_needs_rebuilding; then
-	build_nim
-fi
-

build_rocksdb.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-# used in Travis CI scripts
-
-set -e
-
-ROCKSDBVER="5.17.2"
-
-# script arguments
-[[ $# -ne 1 ]] && { echo "usage: $0 ci_cache_dir"; exit 1; }
-CI_CACHE="$1" # here we cache the installed files
-
-# install from cache and exit, if the version we want is already there
-if [[ -n "$CI_CACHE" ]] && ls "$CI_CACHE"/lib/librocksdb* 2>/dev/null | grep -q "$ROCKSDBVER"; then
-	sudo cp -a "$CI_CACHE"/* /usr/local/
-	exit 0
-fi
-
-# build it
-echo "Building RocksDB"
-curl -O -L -s -S https://github.com/facebook/rocksdb/archive/v$ROCKSDBVER.tar.gz
-tar xzf v$ROCKSDBVER.tar.gz
-cd rocksdb-$ROCKSDBVER
-make DISABLE_WARNING_AS_ERROR=1 -j2 shared_lib
-
-# install it
-if [[ -n "../$CI_CACHE" ]]; then
-	rm -rf "../$CI_CACHE"
-	mkdir "../$CI_CACHE"
-	make INSTALL_PATH="../$CI_CACHE" install-shared
-	sudo cp -a "../$CI_CACHE"/* /usr/local/
-fi
-

common.mk

@@ -1,39 +0,0 @@
-SHELL := bash # the shell used internally by "make"
-
-CC := gcc
-LD := $(CC)
-
-#- extra parameters for the Nim compiler
-#- NIMFLAGS should come from the environment or make's command line
-NIM_PARAMS := $(NIMFLAGS)
-
-# verbosity level
-V := 0
-NIM_PARAMS := $(NIM_PARAMS) --verbosity:$(V)
-HANDLE_OUTPUT :=
-SILENT_TARGET_PREFIX := disabled
-ifeq ($(V), 0)
-  NIM_PARAMS := $(NIM_PARAMS) --hints:off --warnings:off
-  HANDLE_OUTPUT := &>/dev/null
-  SILENT_TARGET_PREFIX :=
-endif
-
-# Chronicles log level
-LOG_LEVEL :=
-ifdef LOG_LEVEL
-  NIM_PARAMS := $(NIM_PARAMS) -d:chronicles_log_level=$(LOG_LEVEL)
-endif
-
-# avoid a "libpcre.so.3: cannot open shared object file: No such file or directory" message, where possible
-ifneq ($(OS), Windows_NT)
-  NIM_PARAMS := $(NIM_PARAMS) -d:usePcreHeader --passL:\"-lpcre\"
-endif
-
-# guess who does parsing before variable expansion
-COMMA := ,
-EMPTY :=
-SPACE := $(EMPTY) $(EMPTY)
-
-# coloured messages
-BUILD_MSG := "\\e[92mBuilding:\\e[39m"
-

env.sh

@@ -1,32 +1,8 @@
-#!/bin/sh
+#!/bin/bash
 
-PWD_CMD="pwd"
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
-# get native Windows paths on Mingw
+# and we fall back to a Zsh-specific special var to also support Zsh.
-uname | grep -qi mingw && PWD_CMD="pwd -W"
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
-rel_path="$(dirname $0)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
-abs_path="$(cd $rel_path; pwd)"
-# do we still need this?
-abs_path_native="$(cd $rel_path; ${PWD_CMD})"
-
-export NIMBUS_ENV_DIR="${abs_path}"
-
-# used by libp2p/go-libp2p-daemon
-export GOPATH="${abs_path}/vendor/go"
-export GO111MODULE=on
-
-#- make it an absolute path, so we can call this script from other dirs
-#- we can't use native Windows paths in here, because colons can't be escaped in PATH
-export PATH="${abs_path}/vendor/Nim/bin:${GOPATH}/bin:${PATH}"
-
-# Nimble needs this to be an absolute path
-export NIMBLE_DIR="${abs_path}/vendor/.nimble"
-
-# used by nim-beacon-chain/tests/simulation/start.sh
-export BUILD_OUTPUTS_DIR="${abs_path}/build"
-
-# change the prompt in shells that source this file
-export PS1="${PS1%\\\$ } [Nimbus env]\\$ "
-
-exec "$@"
 

nimble.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-######################################################
-# run *.nimble tasks using "nim" instead of "nimble" #
-######################################################
-
-# exit on command error
-set -e
-
-[ -z "$1" ] && { echo "usage: $0 task_name"; exit 1; }
-
-F=""
-for F in *.nimble; do
-	# get the first one
-	break
-done
-[ -z "$F" ] && { echo "No *.nimble file found."; exit 1; }
-
-# "nim" seems to only run custom NimScript files if they have a "nims" extension
-NIMS="${F%.nimble}.nims"
-# delete the temporary symlink on script exit
-cleanup() {
-	rm -rf "$NIMS"
-}
-[ -e "$NIMS" ] || { ln -s "$F" "$NIMS"; trap cleanup EXIT; }
-
-# can't have an "exec" here or the EXIT pseudo-signal won't be triggered
-$(dirname $0)/env.sh nim "$@" "$NIMS"
-

scripts/create_nimble_link.sh

@@ -1,13 +0,0 @@
-set -u
-
-module_name="${1#*/}"
-
-if [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ]; then
-  mkdir -p "${NIMBLE_DIR}/pkgs/${module_name}-#head"
-  PKG_DIR="$(${PWD_CMD})"
-  if [ -d src ]; then
-    PKG_DIR="${PKG_DIR}/src"
-  fi
-  echo -e "${PKG_DIR}\n${PKG_DIR}" > "${NIMBLE_DIR}/pkgs/${module_name}-#head/${module_name}.nimble-link"
-fi
-

vendor/NimYAML

@@ -1 +0,0 @@
-Subproject commit b4936a14dd80a17236a32ff8e30fb55706ded7c9

vendor/go/src/github.com/libp2p/go-libp2p-daemon

@@ -1 +0,0 @@
-Subproject commit 5cd632518bc4aff88ea92e5b8639dc316c776dc3

vendor/nim-beacon-chain

@@ -1 +0,0 @@
-Subproject commit 912563648e7f6c9c20eed0ffe5012f233789a57f

vendor/nim-blscurve

@@ -1 +0,0 @@
-Subproject commit a4c7050f89a42b997ff7c2b398e6c40120196b15

vendor/nim-bncurve

@@ -1 +1 @@
-Subproject commit 09d0f93fb11c9c2c5081a51cc207e067bd962f37
+Subproject commit 9b8aac56325428548a22d8a7ca7f92faf152fb6f

vendor/nim-byteutils

@@ -1 +0,0 @@
-Subproject commit 69ce66ef434b03cf8938be8aa5c658982aeccb40

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit 563a0e81958c0500f5b9a6ff7ab2c709899d0118
+Subproject commit 743f14cf68f213a0cbead5e6dfc00ee3edc7fe1e

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 6d7864aa84010271cb8a4a8614c722bae50702cd
+Subproject commit d3eac1d64341a45547ad298dcd9fcb34523473b8

vendor/nim-confutils

@@ -1 +0,0 @@
-Subproject commit 6fb5ee609974196e2885735258c707c5ccbc5c5a

vendor/nim-eth

@@ -1 +1 @@
-Subproject commit f3aad8f9252166e66c1b2a96a522ea0d1243db61
+Subproject commit bef28a18e58f52f24bcc50f0cadefac0224d4630

vendor/nim-faststreams

@@ -1 +1 @@
-Subproject commit 6bbe1c21b63c1f9b8698a91a3548cbd8aa6b0d25
+Subproject commit 1e85ab9150dd3759c86b2a6c9148f3fad3d1694c

vendor/nim-http-utils

@@ -1 +1 @@
-Subproject commit 7e456be185e42f59b8218cd4f15a6c98d9430bd9
+Subproject commit 21613254e8b2f95c9cb44fafb0ac1b10bbf2e046

vendor/nim-json-rpc

@@ -1 +1 @@
-Subproject commit 07e82fa19499d10012c9d125a61ac565a01e3305
+Subproject commit 9214b095fb0266e5cac44804663343cd8288a84c

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit f4bd22e054b5d5a1419710141ac01bd5f1629e22
+Subproject commit 173c7b4a86e6d75a69577166526b0f5840c45003

vendor/nim-libp2p

@@ -1 +0,0 @@
-Subproject commit 2670cd84331b270313c7348e120dcbbcf9a98988

vendor/nim-metrics

@@ -1 +1 @@
-Subproject commit b77efb8d0d8c0137857d6281a47c230df0ffee1b
+Subproject commit d4492f0d0ae9d2c46cc3dde2d947bc175ae420de

vendor/nim-rocksdb

@@ -1 +1 @@
-Subproject commit 5b1307cb1f4c85bb72ff781d810fb8c0148b1183
+Subproject commit 08fec021c0f28f63d1221d40a655078b5b923d1b

vendor/nim-secp256k1

@@ -1 +1 @@
-Subproject commit a41d4432a17a8d024e5bb47c9bdcc641ef309e97
+Subproject commit 85b2620a6e4c41eaa4caa378bd70902cff5a1c48

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 5ef8130c79d1fb5b3105cd3a294e8ee85c1a5c5b
+Subproject commit e6c59d86be74ef4a7c4ee28cf61ebb6a9e2e5972

vendor/nim-snappy

@@ -1 +1 @@
-Subproject commit 072c5eee4395156df96f73d68bd01bb849f367e2
+Subproject commit d954859d58e56292a3e42630c1ef1257eb26840b

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 43bbe48e5f8181b7ca8923e6edf78dec635e8aea
+Subproject commit a81d1fac850119c2ede9f3997332fa9f0a2ad3d8

vendor/nim-stint

@@ -1 +1 @@
-Subproject commit d0162902b7f5ef2e18ff516e96014196d527cedc
+Subproject commit 9993b9dca4a2dbf76bc432252ad4e8db944ae831

vendor/nim-web3

@@ -1 +0,0 @@
-Subproject commit 6ca87380a616afa2f66de4fc2f8bf3ca05576995

vendor/nimbus-build-system

@@ -0,0 +1 @@
+Subproject commit ed16839098d615c977a139e089e4aa354176a94d