diff --git a/.appveyor.yml b/.appveyor.yml index b83063620..89f1418d2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -101,8 +101,10 @@ install: build_script: - cd C:\projects\%APPVEYOR_PROJECT_SLUG% - - nimble install -y + - nimble install -y --depsOnly + - nimble nimbus test_script: + - build\nimbus.exe --help - nimble test deploy: off diff --git a/.gitignore b/.gitignore index fe0f76e8b..f73b9bfce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ -nimcache/ +/nimcache # Executables shall be put in an ignored build/ directory -build/ +/build + +# external packages +/vendor + +# ntags/ctags output +/tags # Ignore dynamic, static libs and libtool archive files *.so diff --git a/.travis.yml b/.travis.yml index 2eeaef799..9737fd00a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,5 +51,9 @@ install: - cd ../.. script: - - nimble install -y + # fail fast + - set -e + - nimble install -y --depsOnly + - nimble nimbus + - ./build/nimbus --help - nimble test diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..def9fcd9a --- /dev/null +++ b/Makefile @@ -0,0 +1,210 @@ +SHELL := bash # the shell used internally by "make" +GIT_CLONE := git clone --quiet --recurse-submodules +GIT_PULL := git pull --recurse-submodules +GIT_STATUS := git status +#- the Nimble dir can't be "[...]/vendor", or Nimble will start looking for +# version numbers in repo dirs (because those would be in its subdirectories) +#- duplicated in "env.sh" for the env var with the same name +NIMBLE_DIR := vendor/.nimble +NIMBLE := nimble -y +REPOS_DIR := vendor/repos +# we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes +RUN_CMD_IN_ALL_REPOS = for D in . vendor/Nim $(REPOS); do echo -e "\n\e[32m$${D}:\e[39m"; cd "$$D"; $(CMD); cd - >/dev/null; done +# absolute path, since it will be run at various subdirectory depths +ENV_SCRIPT := "$(CURDIR)/env.sh" +# duplicated in "env.sh" to prepend NIM_DIR/bin to PATH +NIM_DIR := vendor/Nim +#- forces an update of csources and Nimble repos and a complete rebuild, in case we're called after pulling a new Nim version +#- recompiles Nimble with -d:release until we upgrade to nim-0.20 where koch does it by default +BUILD_NIM := cd $(NIM_DIR) && \ + rm -rf bin/nim_csources csources dist/nimble && \ + sh build_all.sh && \ + $(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim + +#- Git repositories for those dependencies that a Nimbus developer might want to +# modify and test locally +#- their order ensures that `nimble develop` will run in a certain package's +# repo before Nimble tries to install it as a (direct or indirect) dependency, in +# order to avoid duplicate dirs in ".nimble/pgks/" +#- dependencies not listed here are handled entirely by Nimble with "install -y --depsOnly" +REPOS := $(addprefix $(REPOS_DIR)/, \ + status-im/nim-chronicles \ + cheatfate/nimcrypto \ + status-im/nim-ranges \ + status-im/nim-rlp \ + status-im/nim-stint \ + status-im/nim-rocksdb \ + status-im/nim-eth-trie \ + status-im/nim-byteutils \ + status-im/nim-eth-common \ + status-im/nim-http-utils \ + status-im/nim-asyncdispatch2 \ + status-im/nim-json-rpc \ + status-im/nim-faststreams \ + status-im/nim-std-shims \ + status-im/nim-serialization \ + status-im/nim-json-serialization \ + zah/nim-package-visible-types \ + status-im/nim-secp256k1 \ + jangko/snappy \ + status-im/nim-eth-keys \ + status-im/nim-eth-p2p \ + status-im/nim-eth-keyfile \ + status-im/nim-eth-bloom \ + status-im/nim-bncurve \ + ) + +.PHONY: all deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls + +# default target, because it's the first one that doesn't start with '.' +all: nimbus + +#- "--nimbleDir" is ignored for custom tasks: https://github.com/nim-lang/nimble/issues/495 +# so we can't run `nimble ... nimbus` or `nimble ... test`. We have to duplicate those custom tasks here. +#- we could use a way to convince Nimble not to check the dependencies each and every time - https://github.com/nim-lang/nimble/issues/589 +#- we don't want "-y" here, because the user should be reminded to run `make update` +# after a manual `git pull` that adds to $(REPOS) +#- a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated +nimbus: | build deps + $(ENV_SCRIPT) $(NIMBLE) c -o:build/nimbus nimbus/nimbus.nim && echo -e "\nThe binary is in './build/nimbus'." + +# dir +build: + mkdir $@ + +#- runs only the first time and after `make update` actually updates some repo, +# so have a "normal" (timestamp-checked) prerequisite here +deps: $(NIMBLE_DIR) + +#- depends on Git repos being fetched and our Nim and Nimble being built +#- runs `nimble develop` in those repos (but not in the Nimbus repo) - not +# parallelizable, because package order matters +#- installs any remaining Nimbus dependency (those not in $(REPOS)) +$(NIMBLE_DIR): | $(REPOS) $(NIM_DIR) + $(eval CMD := [ "$$$$D" = "." ] && continue; $(ENV_SCRIPT) $(NIMBLE) develop) + $(RUN_CMD_IN_ALL_REPOS) + $(ENV_SCRIPT) $(NIMBLE) install --depsOnly + +#- clones the Git repos +#- can run in parallel with `make -jN` +$(REPOS): + $(GIT_CLONE) https://github.com/$(subst $(REPOS_DIR)/,,$@) $@ + +#- clones and builds the Nim compiler and Nimble +$(NIM_DIR): + $(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@ + $(BUILD_NIM) + +# builds and runs all tests +test: | build deps + $(ENV_SCRIPT) $(NIMBLE) c -r -d:chronicles_log_level=ERROR -o:build/all_tests tests/all_tests.nim + +# usual cleaning +clean: + rm -rf build/{nimbus,all_tests,*.exe} $(NIMBLE_DIR) + +# dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete +mrproper: clean + rm -rf vendor + +# for when you have write access to a repo and you want to use SSH keys +github-ssh: + sed -i 's#https://github.com/#git@github.com:#' .git/config $(NIM_DIR)/.git/config $(REPOS_DIR)/*/*/.git/config + +#- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary) +build-nim: | $(NIM_DIR) + $(BUILD_NIM) + +#- runs `git pull` in all Git repos, if there are new commits in the remote branch +#- rebuilds the Nim compiler after pulling new commits +#- deletes the ".nimble" dir to force the execution of the "deps" target if at least one repo was updated +#- ignores non-zero exit codes from [...] tests +update: | $(REPOS) + $(eval CMD := \ + git remote update && \ + [ -n "$$$$(git rev-parse @{u})" -a "$$$$(git rev-parse @)" != "$$$$(git rev-parse @{u})" ] && \ + REPO_UPDATED=1 && \ + $(GIT_PULL) && \ + { [ "$$$$D" = "$(NIM_DIR)" ] && { cd - >/dev/null; $(BUILD_NIM); }; } \ + || true \ + ) + REPO_UPDATED=0; $(RUN_CMD_IN_ALL_REPOS); [ $$REPO_UPDATED = 1 ] && echo -e "\nAt least one repo updated. Deleting '$(NIMBLE_DIR)'." && rm -rf $(NIMBLE_DIR) || true + +# runs `git status` in all Git repos +status: | $(REPOS) + $(eval CMD := $(GIT_STATUS)) + $(RUN_CMD_IN_ALL_REPOS) + +# https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory" +ntags: + ntags -R . + +#- a few files need to be excluded because they trigger an infinite loop in https://github.com/universal-ctags/ctags +#- limiting it to Nim files, because there are a lot of C files we don't care about +ctags: + ctags -R --verbose=yes \ + --langdef=nim \ + --langmap=nim:.nim \ + --regex-nim='/(\w+)\*?\s*=\s*object/\1/c,class/' \ + --regex-nim='/(\w+)\*?\s*=\s*enum/\1/e,enum/' \ + --regex-nim='/(\w+)\*?\s*=\s*tuple/\1/t,tuple/' \ + --regex-nim='/(\w+)\*?\s*=\s*range/\1/s,subrange/' \ + --regex-nim='/(\w+)\*?\s*=\s*proc/\1/p,proctype/' \ + --regex-nim='/proc\s+(\w+)/\1/f,procedure/' \ + --regex-nim='/method\s+(\w+)/\1/m,method/' \ + --regex-nim='/proc\s+`([^`]+)`/\1/o,operator/' \ + --regex-nim='/template\s+(\w+)/\1/u,template/' \ + --regex-nim='/macro\s+(\w+)/\1/v,macro/' \ + --languages=nim \ + --exclude=nimcache \ + --exclude='*/Nim/tinyc' \ + --exclude='*/Nim/tests' \ + --exclude='*/Nim/csources' \ + --exclude=nimbus/genesis_alloc.nim \ + --exclude=$(REPOS_DIR)/status-im/nim-bncurve/tests/tvectors.nim \ + . + +############################ +# Windows-specific section # +############################ + +ifeq ($(OS), Windows_NT) + # no tabs allowed for indentation here + SQLITE_ARCHIVE_32 := sqlite-dll-win32-x86-3240000.zip + SQLITE_ARCHIVE_64 := sqlite-dll-win64-x64-3240000.zip + + ifeq ($(PROCESSOR_ARCHITEW6432), AMD64) + SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64) + SQLITE_SUFFIX := _64 + ROCKSDB_DIR := x64 + else + ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) + SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64) + SQLITE_SUFFIX := _64 + ROCKSDB_DIR := x64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE), x86) + SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_32) + SQLITE_SUFFIX := _32 + ROCKSDB_DIR := x86 + endif + endif + + SQLITE_URL := https://www.sqlite.org/2018/$(SQLITE_ARCHIVE) + ROCKSDB_ARCHIVE := nimbus-deps.zip + ROCKSDB_URL := https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/$(ROCKSDB_ARCHIVE) + CURL := curl -O -L + UNZIP := unzip -o + +#- back to tabs +#- copied from .appveyor.yml +#- this is why we can't delete the whole "build" dir in the "clean" target +fetch-dlls: | build + cd build && \ + $(CURL) $(SQLITE_URL) && \ + $(CURL) $(ROCKSDB_URL) && \ + $(UNZIP) $(SQLITE_ARCHIVE) && \ + cp -a sqlite3.dll sqlite3$(SQLITE_SUFFIX).dll && \ + $(UNZIP) $(ROCKSDB_ARCHIVE) && \ + cp -a $(ROCKSDB_DIR)/*.dll . +endif diff --git a/README.md b/README.md index 8782c65bf..24b0ff3fc 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ To keep up to date with changes and development progress, follow the [Nimbus blo ### Prerequisites -* A recent version of Nim - * We use the version in the [Status fork](https://github.com/status-im/Nim) - * Follow the Nim installation instructions or use [choosenim](https://github.com/dom96/choosenim) to manage your Nim versions * A recent version of Facebook's [RocksDB](https://github.com/facebook/rocksdb/) * Compile [from source](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) or use the package manager of your OS; for example, [Debian](https://packages.debian.org/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all§ion=all), [Ubuntu](https://packages.ubuntu.com/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all§ion=all), and [Fedora](https://apps.fedoraproject.org/packages/rocksdb) have working RocksDB packages + * on Windows, you can [download pre-compiled DLLs](#windows) + +* GNU make, Bash and the usual POSIX utilities #### Obtaining the prerequisites through the Nix package manager @@ -37,39 +37,59 @@ Users of the [Nix package manager](https://nixos.org/nix/download.html) can inst nix-shell nimbus.nix ``` -### Build & Install +### Build & Develop -We use [Nimble](https://github.com/nim-lang/nimble) to manage dependencies and run tests. +#### POSIX-compatible OS -To build and install Nimbus in your home folder, just execute: +To build Nimbus (in "build/nimbus"), just execute: ```bash -nimble install +make ``` -After a succesful installation, running `nimbus --help` will provide you with a list of -the available command-line options. To start syncing with mainnet, just execute `nimbus` +Running `./build/nimbus --help` will provide you with a list of +the available command-line options. To start syncing with mainnet, just execute `./build/nimbus` without any parameters. To execute all tests: ```bash -nimble test +make test +``` + +To pull the latest changes in all the Git repositories involved: +```bash +make update +``` + +To run a command that might use binaries from the Status Nim fork: +```bash +./env.sh vim ``` Our Wiki provides additional helpful information for [debugging individual test cases][1] and for [pairing Nimbus with a locally running copy of Geth][2]. -[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests -[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction +#### Windows -### Build and Run Nimbus +Install Mingw-w64 for your architecture using the "[MinGW-W64 Online +Installer](https://sourceforge.net/projects/mingw-w64/files/)" (first link +under the directory listing). Run it and select your architecture in the setup +menu ("i686" on 32-bit, "x86\_64" on 64-bit), set the threads to "win32" and +the exceptions to "dwarf" on 32-bit and "seh" on 64-bit. Change the +installation directory to "C:\mingw-w64" and add it to your system PATH in "My +Computer"/"This PC" -> Properties -> Advanced system settings -> Environment +Variables -> Path -> Edit -> New -> C:\mingw-w64\mingw64\bin (it's "C:\mingw-w64\mingw32\bin" on 32-bit) +Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build Nimbus. + +If you don't want to compile RocksDB and SQLite separately, you can fetch pre-compiled DLLs with: ```bash -nimble nimbus -./build/nimbus +mingw32-make fetch-dlls ``` -Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus)! +This will place the right DLLs for your architecture in the "build/" directory. + +You can now follow those instructions in the previous section by replacing `make` with `mingw32-make` (regardless of your 32-bit or 64-bit architecture). ### Development tips @@ -77,10 +97,22 @@ Report any errors you encounter, please, if not [already documented](https://git `-d:nimbus_db_backend=...` where the (case-insensitive) value is one of "rocksdb" (the default), "sqlite", "lmdb". -#### Troubleshooting +### Troubleshooting + +Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus/issues)! + Sometimes, the build will fail even though the latest CI is green - here are a few tips to handle this: +#### Using the Makefile + +* Turn it off and on again: +```bash +make clean update +``` + +#### Using Nimble directly + * Wrong Nim version * We depend on many bleeding-edge features - Nim regressions often happen * Use the [Status fork](https://github.com/status-im/Nim) of Nim @@ -102,3 +134,6 @@ or * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) at your option. This file may not be copied, modified, or distributed except according to those terms. + +[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests +[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction diff --git a/env.sh b/env.sh new file mode 100755 index 000000000..7000c1a7e --- /dev/null +++ b/env.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +rel_path="$(dirname $0)" +abs_path="$(cd $rel_path; pwd)" + +#- make it an absolute path, so we can call this script from other dirs +export PATH="${abs_path}/vendor/Nim/bin:$PATH" + +#- Nimble needs this to be an absolute path +export NIMBLE_DIR="${abs_path}/vendor/.nimble" + +exec "$@" diff --git a/nimbus.nimble b/nimbus.nimble index 0f267d275..88d4ab055 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -6,9 +6,11 @@ author = "Status Research & Development GmbH" description = "An Ethereum 2.0 Sharding Client for Resource-Restricted Devices" license = "Apache License 2.0" skipDirs = @["tests", "examples"] -bin = @["nimbus/nimbus"] +# we can't have the result of a custom task in the "bin" var - https://github.com/nim-lang/nimble/issues/542 +# bin = @["build/nimbus"] requires "nim >= 0.18.1", + "chronicles", "nimcrypto", "rlp", "stint", @@ -21,7 +23,7 @@ requires "nim >= 0.18.1", "eth_keyfile", "eth_keys", "eth_bloom", - "https://github.com/status-im/nim-bncurve >= 1.0.1" + "bncurve" proc buildBinary(name: string, srcDir = ".", lang = "c") = if not dirExists "build": mkDir "build" @@ -29,7 +31,7 @@ proc buildBinary(name: string, srcDir = ".", lang = "c") = setCommand lang, srcDir & name & ".nim" proc test(name: string, lang = "c") = - --define: "chronicles_log_level=ERROR" + --define:"chronicles_log_level=ERROR" --run buildBinary name, "tests/" @@ -38,3 +40,4 @@ task test, "Run tests": task nimbus, "Build Nimbus": buildBinary "nimbus", "nimbus/" + diff --git a/nim.cfg b/nimbus/nim.cfg similarity index 77% rename from nim.cfg rename to nimbus/nim.cfg index ee64291c7..adc57d915 100644 --- a/nim.cfg +++ b/nimbus/nim.cfg @@ -1,3 +1,4 @@ -d:chronicles_line_numbers -d:"chronicles_sinks=textblocks" +-d:nimDebugDlOpen