new build system

- isolate dependencies (both Git repos and ".nimble" dir) in "vendor"
- ensure that everything is compiled using our version of Nim with
  something as simple as "make"
- pull the latest changes with "make update"
- re-create Nimble's package dir and re-build the Nim compiler automatically when needed
- "env.sh" script that can be used to prefix any command that needs to use
  "nim" or "nimble" from our Nim repo
- move the top level nim.cfg to nimbus/nim.cfg so it doesn't apply to
  deps
- "fetch-dlls" `make` target for Windows to get precompiled Snappy, RocksDB and SQLite DLLs

.appveyor.yml

@@ -101,8 +101,10 @@ install:
 
 build_script:
   - cd C:\projects\%APPVEYOR_PROJECT_SLUG%
-  - nimble install -y
+  - nimble install -y --depsOnly
+  - nimble nimbus
 test_script:
+  - build\nimbus.exe --help
   - nimble test
 
 deploy: off

.gitignore

@@ -1,7 +1,13 @@
-nimcache/
+/nimcache
 
 # Executables shall be put in an ignored build/ directory
-build/
+/build
+
+# external packages
+/vendor
+
+# ntags/ctags output
+/tags
 
 # Ignore dynamic, static libs and libtool archive files
 *.so

.travis.yml

@@ -51,5 +51,9 @@ install:
   - cd ../..
 
 script:
-  - nimble install -y
+  # fail fast
+  - set -e
+  - nimble install -y --depsOnly
+  - nimble nimbus
+  - ./build/nimbus --help
   - nimble test

Makefile

@@ -0,0 +1,210 @@
+SHELL := bash # the shell used internally by "make"
+GIT_CLONE := git clone --quiet --recurse-submodules
+GIT_PULL := git pull --recurse-submodules
+GIT_STATUS := git status
+#- the Nimble dir can't be "[...]/vendor", or Nimble will start looking for
+#  version numbers in repo dirs (because those would be in its subdirectories)
+#- duplicated in "env.sh" for the env var with the same name
+NIMBLE_DIR := vendor/.nimble
+NIMBLE := nimble -y
+REPOS_DIR := vendor/repos
+# we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes
+RUN_CMD_IN_ALL_REPOS = for D in . vendor/Nim $(REPOS); do echo -e "\n\e[32m$${D}:\e[39m"; cd "$$D"; $(CMD); cd - >/dev/null; done
+# absolute path, since it will be run at various subdirectory depths
+ENV_SCRIPT := "$(CURDIR)/env.sh"
+# duplicated in "env.sh" to prepend NIM_DIR/bin to PATH
+NIM_DIR := vendor/Nim
+#- forces an update of csources and Nimble repos and a complete rebuild, in case we're called after pulling a new Nim version
+#- recompiles Nimble with -d:release until we upgrade to nim-0.20 where koch does it by default
+BUILD_NIM := cd $(NIM_DIR) && \
+	rm -rf bin/nim_csources csources dist/nimble && \
+	sh build_all.sh && \
+	$(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim
+
+#- Git repositories for those dependencies that a Nimbus developer might want to
+#  modify and test locally
+#- their order ensures that `nimble develop` will run in a certain package's
+#  repo before Nimble tries to install it as a (direct or indirect) dependency, in
+#  order to avoid duplicate dirs in ".nimble/pgks/"
+#- dependencies not listed here are handled entirely by Nimble with "install -y --depsOnly"
+REPOS := $(addprefix $(REPOS_DIR)/, \
+	status-im/nim-chronicles \
+	cheatfate/nimcrypto \
+	status-im/nim-ranges \
+	status-im/nim-rlp \
+	status-im/nim-stint \
+	status-im/nim-rocksdb \
+	status-im/nim-eth-trie \
+	status-im/nim-byteutils \
+	status-im/nim-eth-common \
+	status-im/nim-http-utils \
+	status-im/nim-asyncdispatch2 \
+	status-im/nim-json-rpc \
+	status-im/nim-faststreams \
+	status-im/nim-std-shims \
+	status-im/nim-serialization \
+	status-im/nim-json-serialization \
+	zah/nim-package-visible-types \
+	status-im/nim-secp256k1 \
+	jangko/snappy \
+	status-im/nim-eth-keys \
+	status-im/nim-eth-p2p \
+	status-im/nim-eth-keyfile \
+	status-im/nim-eth-bloom \
+	status-im/nim-bncurve \
+	)
+
+.PHONY: all deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls
+
+# default target, because it's the first one that doesn't start with '.'
+all: nimbus
+
+#- "--nimbleDir" is ignored for custom tasks: https://github.com/nim-lang/nimble/issues/495
+#  so we can't run `nimble ... nimbus` or `nimble ... test`. We have to duplicate those custom tasks here.
+#- we could use a way to convince Nimble not to check the dependencies each and every time - https://github.com/nim-lang/nimble/issues/589
+#- we don't want "-y" here, because the user should be reminded to run `make update`
+#  after a manual `git pull` that adds to $(REPOS)
+#- a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated
+nimbus: | build deps
+	$(ENV_SCRIPT) $(NIMBLE) c -o:build/nimbus nimbus/nimbus.nim && echo -e "\nThe binary is in './build/nimbus'."
+
+# dir
+build:
+	mkdir $@
+
+#- runs only the first time and after `make update` actually updates some repo,
+#  so have a "normal" (timestamp-checked) prerequisite here
+deps: $(NIMBLE_DIR)
+
+#- depends on Git repos being fetched and our Nim and Nimble being built
+#- runs `nimble develop` in those repos (but not in the Nimbus repo) - not
+#  parallelizable, because package order matters
+#- installs any remaining Nimbus dependency (those not in $(REPOS))
+$(NIMBLE_DIR): | $(REPOS) $(NIM_DIR)
+	$(eval CMD := [ "$$$$D" = "." ] && continue; $(ENV_SCRIPT) $(NIMBLE) develop)
+	$(RUN_CMD_IN_ALL_REPOS)
+	$(ENV_SCRIPT) $(NIMBLE) install --depsOnly
+
+#- clones the Git repos
+#- can run in parallel with `make -jN`
+$(REPOS):
+	$(GIT_CLONE) https://github.com/$(subst $(REPOS_DIR)/,,$@) $@
+
+#- clones and builds the Nim compiler and Nimble
+$(NIM_DIR):
+	$(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@
+	$(BUILD_NIM)
+
+# builds and runs all tests
+test: | build deps
+	$(ENV_SCRIPT) $(NIMBLE) c -r -d:chronicles_log_level=ERROR -o:build/all_tests tests/all_tests.nim
+
+# usual cleaning
+clean:
+	rm -rf build/{nimbus,all_tests,*.exe} $(NIMBLE_DIR)
+
+# dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete
+mrproper: clean
+	rm -rf vendor
+
+# for when you have write access to a repo and you want to use SSH keys
+github-ssh:
+	sed -i 's#https://github.com/#git@github.com:#' .git/config $(NIM_DIR)/.git/config $(REPOS_DIR)/*/*/.git/config
+
+#- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary)
+build-nim: | $(NIM_DIR)
+	$(BUILD_NIM)
+
+#- runs `git pull` in all Git repos, if there are new commits in the remote branch
+#- rebuilds the Nim compiler after pulling new commits
+#- deletes the ".nimble" dir to force the execution of the "deps" target if at least one repo was updated
+#- ignores non-zero exit codes from [...] tests
+update: | $(REPOS)
+	$(eval CMD := \
+		git remote update && \
+		[ -n "$$$$(git rev-parse @{u})" -a "$$$$(git rev-parse @)" != "$$$$(git rev-parse @{u})" ] && \
+		REPO_UPDATED=1 && \
+		$(GIT_PULL) && \
+		{ [ "$$$$D" = "$(NIM_DIR)" ] && { cd - >/dev/null; $(BUILD_NIM); }; } \
+		|| true \
+	)
+	REPO_UPDATED=0; $(RUN_CMD_IN_ALL_REPOS); [ $$REPO_UPDATED = 1 ] && echo -e "\nAt least one repo updated. Deleting '$(NIMBLE_DIR)'." && rm -rf $(NIMBLE_DIR) || true
+
+# runs `git status` in all Git repos
+status: | $(REPOS)
+	$(eval CMD := $(GIT_STATUS))
+	$(RUN_CMD_IN_ALL_REPOS)
+
+# https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory"
+ntags:
+	ntags -R .
+
+#- a few files need to be excluded because they trigger an infinite loop in https://github.com/universal-ctags/ctags
+#- limiting it to Nim files, because there are a lot of C files we don't care about
+ctags:
+	ctags -R --verbose=yes \
+	--langdef=nim \
+	--langmap=nim:.nim \
+	--regex-nim='/(\w+)\*?\s*=\s*object/\1/c,class/' \
+	--regex-nim='/(\w+)\*?\s*=\s*enum/\1/e,enum/' \
+	--regex-nim='/(\w+)\*?\s*=\s*tuple/\1/t,tuple/' \
+	--regex-nim='/(\w+)\*?\s*=\s*range/\1/s,subrange/' \
+	--regex-nim='/(\w+)\*?\s*=\s*proc/\1/p,proctype/' \
+	--regex-nim='/proc\s+(\w+)/\1/f,procedure/' \
+	--regex-nim='/method\s+(\w+)/\1/m,method/' \
+	--regex-nim='/proc\s+`([^`]+)`/\1/o,operator/' \
+	--regex-nim='/template\s+(\w+)/\1/u,template/' \
+	--regex-nim='/macro\s+(\w+)/\1/v,macro/' \
+	--languages=nim \
+	--exclude=nimcache \
+	--exclude='*/Nim/tinyc' \
+	--exclude='*/Nim/tests' \
+	--exclude='*/Nim/csources' \
+	--exclude=nimbus/genesis_alloc.nim \
+	--exclude=$(REPOS_DIR)/status-im/nim-bncurve/tests/tvectors.nim \
+	.
+
+############################
+# Windows-specific section #
+############################
+
+ifeq ($(OS), Windows_NT)
+  # no tabs allowed for indentation here
+  SQLITE_ARCHIVE_32 := sqlite-dll-win32-x86-3240000.zip
+  SQLITE_ARCHIVE_64 := sqlite-dll-win64-x64-3240000.zip
+
+  ifeq ($(PROCESSOR_ARCHITEW6432), AMD64)
+    SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64)
+    SQLITE_SUFFIX := _64
+    ROCKSDB_DIR := x64
+  else
+    ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
+      SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64)
+      SQLITE_SUFFIX := _64
+      ROCKSDB_DIR := x64
+    endif
+    ifeq ($(PROCESSOR_ARCHITECTURE), x86)
+      SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_32)
+      SQLITE_SUFFIX := _32
+      ROCKSDB_DIR := x86
+    endif
+  endif
+
+  SQLITE_URL := https://www.sqlite.org/2018/$(SQLITE_ARCHIVE)
+  ROCKSDB_ARCHIVE := nimbus-deps.zip
+  ROCKSDB_URL := https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/$(ROCKSDB_ARCHIVE)
+  CURL := curl -O -L
+  UNZIP := unzip -o
+
+#- back to tabs
+#- copied from .appveyor.yml
+#- this is why we can't delete the whole "build" dir in the "clean" target
+fetch-dlls: | build
+	cd build && \
+		$(CURL) $(SQLITE_URL) && \
+		$(CURL) $(ROCKSDB_URL) && \
+		$(UNZIP) $(SQLITE_ARCHIVE) && \
+		cp -a sqlite3.dll sqlite3$(SQLITE_SUFFIX).dll && \
+		$(UNZIP) $(ROCKSDB_ARCHIVE) && \
+		cp -a $(ROCKSDB_DIR)/*.dll .
+endif

README.md

@@ -23,11 +23,11 @@ To keep up to date with changes and development progress, follow the [Nimbus blo
 
 ### Prerequisites
 
-* A recent version of Nim
-  * We use the version in the [Status fork](https://github.com/status-im/Nim)
-  * Follow the Nim installation instructions or use [choosenim](https://github.com/dom96/choosenim) to manage your Nim versions
 * A recent version of Facebook's [RocksDB](https://github.com/facebook/rocksdb/)
   * Compile [from source](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) or use the package manager of your OS; for example, [Debian](https://packages.debian.org/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all&section=all), [Ubuntu](https://packages.ubuntu.com/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all&section=all), and [Fedora](https://apps.fedoraproject.org/packages/rocksdb) have working RocksDB packages
+  * on Windows, you can [download pre-compiled DLLs](#windows)
+
+* GNU make, Bash and the usual POSIX utilities
 
 #### Obtaining the prerequisites through the Nix package manager
 
@@ -37,39 +37,59 @@ Users of the [Nix package manager](https://nixos.org/nix/download.html) can inst
 nix-shell nimbus.nix
 ```
 
-### Build & Install
+### Build & Develop
 
-We use [Nimble](https://github.com/nim-lang/nimble) to manage dependencies and run tests.
+#### POSIX-compatible OS
 
-To build and install Nimbus in your home folder, just execute:
+To build Nimbus (in "build/nimbus"), just execute:
 
 ```bash
-nimble install
+make
 ```
 
-After a succesful installation, running `nimbus --help` will provide you with a list of
+Running `./build/nimbus --help` will provide you with a list of
-the available command-line options. To start syncing with mainnet, just execute `nimbus`
+the available command-line options. To start syncing with mainnet, just execute `./build/nimbus`
 without any parameters.
 
 To execute all tests:
 ```bash
-nimble test
+make test
+```
+
+To pull the latest changes in all the Git repositories involved:
+```bash
+make update
+```
+
+To run a command that might use binaries from the Status Nim fork:
+```bash
+./env.sh vim
 ```
 
 Our Wiki provides additional helpful information for [debugging individual test cases][1]
 and for [pairing Nimbus with a locally running copy of Geth][2].
 
-[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
+#### Windows
-[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction
 
-### Build and Run Nimbus
+Install Mingw-w64 for your architecture using the "[MinGW-W64 Online
+Installer](https://sourceforge.net/projects/mingw-w64/files/)" (first link
+under the directory listing). Run it and select your architecture in the setup
+menu ("i686" on 32-bit, "x86\_64" on 64-bit), set the threads to "win32" and
+the exceptions to "dwarf" on 32-bit and "seh" on 64-bit. Change the
+installation directory to "C:\mingw-w64" and add it to your system PATH in "My
+Computer"/"This PC" -> Properties -> Advanced system settings -> Environment
+Variables -> Path -> Edit -> New -> C:\mingw-w64\mingw64\bin (it's "C:\mingw-w64\mingw32\bin" on 32-bit)
 
+Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build Nimbus.
+
+If you don't want to compile RocksDB and SQLite separately, you can fetch pre-compiled DLLs with:
 ```bash
-nimble nimbus
+mingw32-make fetch-dlls
-./build/nimbus
 ```
 
-Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus)!
+This will place the right DLLs for your architecture in the "build/" directory.
+
+You can now follow those instructions in the previous section by replacing `make` with `mingw32-make` (regardless of your 32-bit or 64-bit architecture).
 
 ### Development tips
 
@@ -77,10 +97,22 @@ Report any errors you encounter, please, if not [already documented](https://git
   `-d:nimbus_db_backend=...` where the (case-insensitive) value is one of
   "rocksdb" (the default), "sqlite", "lmdb".
 
-#### Troubleshooting
+### Troubleshooting
+
+Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus/issues)!
+
 
 Sometimes, the build will fail even though the latest CI is green - here are a few tips to handle this:
 
+#### Using the Makefile
+
+* Turn it off and on again:
+```bash
+make clean update
+```
+
+#### Using Nimble directly
+
 * Wrong Nim version
   * We depend on many bleeding-edge features - Nim regressions often happen
   * Use the [Status fork](https://github.com/status-im/Nim) of Nim
@@ -102,3 +134,6 @@ or
 * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
 
 at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
+[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction

env.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+rel_path="$(dirname $0)"
+abs_path="$(cd $rel_path; pwd)"
+
+#- make it an absolute path, so we can call this script from other dirs
+export PATH="${abs_path}/vendor/Nim/bin:$PATH"
+
+#- Nimble needs this to be an absolute path
+export NIMBLE_DIR="${abs_path}/vendor/.nimble"
+
+exec "$@"

nimbus.nimble

@@ -6,9 +6,11 @@ author        = "Status Research & Development GmbH"
 description   = "An Ethereum 2.0 Sharding Client for Resource-Restricted Devices"
 license       = "Apache License 2.0"
 skipDirs      = @["tests", "examples"]
-bin           = @["nimbus/nimbus"]
+# we can't have the result of a custom task in the "bin" var - https://github.com/nim-lang/nimble/issues/542
+# bin           = @["build/nimbus"]
 
 requires "nim >= 0.18.1",
+         "chronicles",
          "nimcrypto",
          "rlp",
          "stint",
@@ -21,7 +23,7 @@ requires "nim >= 0.18.1",
          "eth_keyfile",
          "eth_keys",
          "eth_bloom",
-         "https://github.com/status-im/nim-bncurve >= 1.0.1"
+         "bncurve"
 
 proc buildBinary(name: string, srcDir = ".", lang = "c") =
   if not dirExists "build": mkDir "build"
@@ -38,3 +40,4 @@ task test, "Run tests":
 
 task nimbus, "Build Nimbus":
   buildBinary "nimbus", "nimbus/"
+

nimbus/nim.cfg

@@ -1,3 +1,4 @@
 -d:chronicles_line_numbers
 -d:"chronicles_sinks=textblocks"
+-d:nimDebugDlOpen