new build system
- isolate dependencies (both Git repos and ".nimble" dir) in "vendor" - ensure that everything is compiled using our version of Nim with something as simple as "make" - pull the latest changes with "make update" - re-create Nimble's package dir and re-build the Nim compiler automatically when needed - "env.sh" script that can be used to prefix any command that needs to use "nim" or "nimble" from our Nim repo - move the top level nim.cfg to nimbus/nim.cfg so it doesn't apply to deps - "fetch-dlls" `make` target for Windows to get precompiled Snappy, RocksDB and SQLite DLLs
This commit is contained in:
Ștefan Talpalaru
zah
parent
74add30c5f
commit
bd9d15d947
|
|
@ -101,8 +101,10 @@ install:
|
|||
|
||||
build_script:
|
||||
- cd C:\projects\%APPVEYOR_PROJECT_SLUG%
|
||||
- nimble install -y
|
||||
- nimble install -y --depsOnly
|
||||
- nimble nimbus
|
||||
test_script:
|
||||
- build\nimbus.exe --help
|
||||
- nimble test
|
||||
|
||||
deploy: off
|
||||
|
|
|
|||
|
|
@ -1,7 +1,13 @@
|
|||
nimcache/
|
||||
/nimcache
|
||||
|
||||
# Executables shall be put in an ignored build/ directory
|
||||
build/
|
||||
/build
|
||||
|
||||
# external packages
|
||||
/vendor
|
||||
|
||||
# ntags/ctags output
|
||||
/tags
|
||||
|
||||
# Ignore dynamic, static libs and libtool archive files
|
||||
*.so
|
||||
|
|
|
|||
|
|
@ -51,5 +51,9 @@ install:
|
|||
- cd ../..
|
||||
|
||||
script:
|
||||
- nimble install -y
|
||||
# fail fast
|
||||
- set -e
|
||||
- nimble install -y --depsOnly
|
||||
- nimble nimbus
|
||||
- ./build/nimbus --help
|
||||
- nimble test
|
||||
|
|
|
|||
|
|
@ -0,0 +1,210 @@
|
|||
SHELL := bash # the shell used internally by "make"
|
||||
GIT_CLONE := git clone --quiet --recurse-submodules
|
||||
GIT_PULL := git pull --recurse-submodules
|
||||
GIT_STATUS := git status
|
||||
#- the Nimble dir can't be "[...]/vendor", or Nimble will start looking for
|
||||
# version numbers in repo dirs (because those would be in its subdirectories)
|
||||
#- duplicated in "env.sh" for the env var with the same name
|
||||
NIMBLE_DIR := vendor/.nimble
|
||||
NIMBLE := nimble -y
|
||||
REPOS_DIR := vendor/repos
|
||||
# we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes
|
||||
RUN_CMD_IN_ALL_REPOS = for D in . vendor/Nim $(REPOS); do echo -e "\n\e[32m$${D}:\e[39m"; cd "$$D"; $(CMD); cd - >/dev/null; done
|
||||
# absolute path, since it will be run at various subdirectory depths
|
||||
ENV_SCRIPT := "$(CURDIR)/env.sh"
|
||||
# duplicated in "env.sh" to prepend NIM_DIR/bin to PATH
|
||||
NIM_DIR := vendor/Nim
|
||||
#- forces an update of csources and Nimble repos and a complete rebuild, in case we're called after pulling a new Nim version
|
||||
#- recompiles Nimble with -d:release until we upgrade to nim-0.20 where koch does it by default
|
||||
BUILD_NIM := cd $(NIM_DIR) && \
|
||||
rm -rf bin/nim_csources csources dist/nimble && \
|
||||
sh build_all.sh && \
|
||||
$(ENV_SCRIPT) nim c -d:release --noNimblePath -p:compiler --nilseqs:on -o:bin/nimble dist/nimble/src/nimble.nim
|
||||
|
||||
#- Git repositories for those dependencies that a Nimbus developer might want to
|
||||
# modify and test locally
|
||||
#- their order ensures that `nimble develop` will run in a certain package's
|
||||
# repo before Nimble tries to install it as a (direct or indirect) dependency, in
|
||||
# order to avoid duplicate dirs in ".nimble/pgks/"
|
||||
#- dependencies not listed here are handled entirely by Nimble with "install -y --depsOnly"
|
||||
REPOS := $(addprefix $(REPOS_DIR)/, \
|
||||
status-im/nim-chronicles \
|
||||
cheatfate/nimcrypto \
|
||||
status-im/nim-ranges \
|
||||
status-im/nim-rlp \
|
||||
status-im/nim-stint \
|
||||
status-im/nim-rocksdb \
|
||||
status-im/nim-eth-trie \
|
||||
status-im/nim-byteutils \
|
||||
status-im/nim-eth-common \
|
||||
status-im/nim-http-utils \
|
||||
status-im/nim-asyncdispatch2 \
|
||||
status-im/nim-json-rpc \
|
||||
status-im/nim-faststreams \
|
||||
status-im/nim-std-shims \
|
||||
status-im/nim-serialization \
|
||||
status-im/nim-json-serialization \
|
||||
zah/nim-package-visible-types \
|
||||
status-im/nim-secp256k1 \
|
||||
jangko/snappy \
|
||||
status-im/nim-eth-keys \
|
||||
status-im/nim-eth-p2p \
|
||||
status-im/nim-eth-keyfile \
|
||||
status-im/nim-eth-bloom \
|
||||
status-im/nim-bncurve \
|
||||
)
|
||||
|
||||
.PHONY: all deps github-ssh build-nim update status ntags ctags nimbus test clean mrproper fetch-dlls
|
||||
|
||||
# default target, because it's the first one that doesn't start with '.'
|
||||
all: nimbus
|
||||
|
||||
#- "--nimbleDir" is ignored for custom tasks: https://github.com/nim-lang/nimble/issues/495
|
||||
# so we can't run `nimble ... nimbus` or `nimble ... test`. We have to duplicate those custom tasks here.
|
||||
#- we could use a way to convince Nimble not to check the dependencies each and every time - https://github.com/nim-lang/nimble/issues/589
|
||||
#- we don't want "-y" here, because the user should be reminded to run `make update`
|
||||
# after a manual `git pull` that adds to $(REPOS)
|
||||
#- a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated
|
||||
nimbus: | build deps
|
||||
$(ENV_SCRIPT) $(NIMBLE) c -o:build/nimbus nimbus/nimbus.nim && echo -e "\nThe binary is in './build/nimbus'."
|
||||
|
||||
# dir
|
||||
build:
|
||||
mkdir $@
|
||||
|
||||
#- runs only the first time and after `make update` actually updates some repo,
|
||||
# so have a "normal" (timestamp-checked) prerequisite here
|
||||
deps: $(NIMBLE_DIR)
|
||||
|
||||
#- depends on Git repos being fetched and our Nim and Nimble being built
|
||||
#- runs `nimble develop` in those repos (but not in the Nimbus repo) - not
|
||||
# parallelizable, because package order matters
|
||||
#- installs any remaining Nimbus dependency (those not in $(REPOS))
|
||||
$(NIMBLE_DIR): | $(REPOS) $(NIM_DIR)
|
||||
$(eval CMD := [ "$$$$D" = "." ] && continue; $(ENV_SCRIPT) $(NIMBLE) develop)
|
||||
$(RUN_CMD_IN_ALL_REPOS)
|
||||
$(ENV_SCRIPT) $(NIMBLE) install --depsOnly
|
||||
|
||||
#- clones the Git repos
|
||||
#- can run in parallel with `make -jN`
|
||||
$(REPOS):
|
||||
$(GIT_CLONE) https://github.com/$(subst $(REPOS_DIR)/,,$@) $@
|
||||
|
||||
#- clones and builds the Nim compiler and Nimble
|
||||
$(NIM_DIR):
|
||||
$(GIT_CLONE) --depth 1 https://github.com/status-im/Nim $@
|
||||
$(BUILD_NIM)
|
||||
|
||||
# builds and runs all tests
|
||||
test: | build deps
|
||||
$(ENV_SCRIPT) $(NIMBLE) c -r -d:chronicles_log_level=ERROR -o:build/all_tests tests/all_tests.nim
|
||||
|
||||
# usual cleaning
|
||||
clean:
|
||||
rm -rf build/{nimbus,all_tests,*.exe} $(NIMBLE_DIR)
|
||||
|
||||
# dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete
|
||||
mrproper: clean
|
||||
rm -rf vendor
|
||||
|
||||
# for when you have write access to a repo and you want to use SSH keys
|
||||
github-ssh:
|
||||
sed -i 's#https://github.com/#git@github.com:#' .git/config $(NIM_DIR)/.git/config $(REPOS_DIR)/*/*/.git/config
|
||||
|
||||
#- re-builds the Nim compiler (not usually needed, because `make update` does it when necessary)
|
||||
build-nim: | $(NIM_DIR)
|
||||
$(BUILD_NIM)
|
||||
|
||||
#- runs `git pull` in all Git repos, if there are new commits in the remote branch
|
||||
#- rebuilds the Nim compiler after pulling new commits
|
||||
#- deletes the ".nimble" dir to force the execution of the "deps" target if at least one repo was updated
|
||||
#- ignores non-zero exit codes from [...] tests
|
||||
update: | $(REPOS)
|
||||
$(eval CMD := \
|
||||
git remote update && \
|
||||
[ -n "$$$$(git rev-parse @{u})" -a "$$$$(git rev-parse @)" != "$$$$(git rev-parse @{u})" ] && \
|
||||
REPO_UPDATED=1 && \
|
||||
$(GIT_PULL) && \
|
||||
{ [ "$$$$D" = "$(NIM_DIR)" ] && { cd - >/dev/null; $(BUILD_NIM); }; } \
|
||||
|| true \
|
||||
)
|
||||
REPO_UPDATED=0; $(RUN_CMD_IN_ALL_REPOS); [ $$REPO_UPDATED = 1 ] && echo -e "\nAt least one repo updated. Deleting '$(NIMBLE_DIR)'." && rm -rf $(NIMBLE_DIR) || true
|
||||
|
||||
# runs `git status` in all Git repos
|
||||
status: | $(REPOS)
|
||||
$(eval CMD := $(GIT_STATUS))
|
||||
$(RUN_CMD_IN_ALL_REPOS)
|
||||
|
||||
# https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory"
|
||||
ntags:
|
||||
ntags -R .
|
||||
|
||||
#- a few files need to be excluded because they trigger an infinite loop in https://github.com/universal-ctags/ctags
|
||||
#- limiting it to Nim files, because there are a lot of C files we don't care about
|
||||
ctags:
|
||||
ctags -R --verbose=yes \
|
||||
--langdef=nim \
|
||||
--langmap=nim:.nim \
|
||||
--regex-nim='/(\w+)\*?\s*=\s*object/\1/c,class/' \
|
||||
--regex-nim='/(\w+)\*?\s*=\s*enum/\1/e,enum/' \
|
||||
--regex-nim='/(\w+)\*?\s*=\s*tuple/\1/t,tuple/' \
|
||||
--regex-nim='/(\w+)\*?\s*=\s*range/\1/s,subrange/' \
|
||||
--regex-nim='/(\w+)\*?\s*=\s*proc/\1/p,proctype/' \
|
||||
--regex-nim='/proc\s+(\w+)/\1/f,procedure/' \
|
||||
--regex-nim='/method\s+(\w+)/\1/m,method/' \
|
||||
--regex-nim='/proc\s+`([^`]+)`/\1/o,operator/' \
|
||||
--regex-nim='/template\s+(\w+)/\1/u,template/' \
|
||||
--regex-nim='/macro\s+(\w+)/\1/v,macro/' \
|
||||
--languages=nim \
|
||||
--exclude=nimcache \
|
||||
--exclude='*/Nim/tinyc' \
|
||||
--exclude='*/Nim/tests' \
|
||||
--exclude='*/Nim/csources' \
|
||||
--exclude=nimbus/genesis_alloc.nim \
|
||||
--exclude=$(REPOS_DIR)/status-im/nim-bncurve/tests/tvectors.nim \
|
||||
.
|
||||
|
||||
############################
|
||||
# Windows-specific section #
|
||||
############################
|
||||
|
||||
ifeq ($(OS), Windows_NT)
|
||||
# no tabs allowed for indentation here
|
||||
SQLITE_ARCHIVE_32 := sqlite-dll-win32-x86-3240000.zip
|
||||
SQLITE_ARCHIVE_64 := sqlite-dll-win64-x64-3240000.zip
|
||||
|
||||
ifeq ($(PROCESSOR_ARCHITEW6432), AMD64)
|
||||
SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64)
|
||||
SQLITE_SUFFIX := _64
|
||||
ROCKSDB_DIR := x64
|
||||
else
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||
SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_64)
|
||||
SQLITE_SUFFIX := _64
|
||||
ROCKSDB_DIR := x64
|
||||
endif
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), x86)
|
||||
SQLITE_ARCHIVE := $(SQLITE_ARCHIVE_32)
|
||||
SQLITE_SUFFIX := _32
|
||||
ROCKSDB_DIR := x86
|
||||
endif
|
||||
endif
|
||||
|
||||
SQLITE_URL := https://www.sqlite.org/2018/$(SQLITE_ARCHIVE)
|
||||
ROCKSDB_ARCHIVE := nimbus-deps.zip
|
||||
ROCKSDB_URL := https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/$(ROCKSDB_ARCHIVE)
|
||||
CURL := curl -O -L
|
||||
UNZIP := unzip -o
|
||||
|
||||
#- back to tabs
|
||||
#- copied from .appveyor.yml
|
||||
#- this is why we can't delete the whole "build" dir in the "clean" target
|
||||
fetch-dlls: | build
|
||||
cd build && \
|
||||
$(CURL) $(SQLITE_URL) && \
|
||||
$(CURL) $(ROCKSDB_URL) && \
|
||||
$(UNZIP) $(SQLITE_ARCHIVE) && \
|
||||
cp -a sqlite3.dll sqlite3$(SQLITE_SUFFIX).dll && \
|
||||
$(UNZIP) $(ROCKSDB_ARCHIVE) && \
|
||||
cp -a $(ROCKSDB_DIR)/*.dll .
|
||||
endif
|
||||
69
README.md
69
README.md
|
|
@ -23,11 +23,11 @@ To keep up to date with changes and development progress, follow the [Nimbus blo
|
|||
|
||||
### Prerequisites
|
||||
|
||||
* A recent version of Nim
|
||||
* We use the version in the [Status fork](https://github.com/status-im/Nim)
|
||||
* Follow the Nim installation instructions or use [choosenim](https://github.com/dom96/choosenim) to manage your Nim versions
|
||||
* A recent version of Facebook's [RocksDB](https://github.com/facebook/rocksdb/)
|
||||
* Compile [from source](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) or use the package manager of your OS; for example, [Debian](https://packages.debian.org/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all§ion=all), [Ubuntu](https://packages.ubuntu.com/search?keywords=librocksdb-dev&searchon=names&exact=1&suite=all§ion=all), and [Fedora](https://apps.fedoraproject.org/packages/rocksdb) have working RocksDB packages
|
||||
* on Windows, you can [download pre-compiled DLLs](#windows)
|
||||
|
||||
* GNU make, Bash and the usual POSIX utilities
|
||||
|
||||
#### Obtaining the prerequisites through the Nix package manager
|
||||
|
||||
|
|
@ -37,39 +37,59 @@ Users of the [Nix package manager](https://nixos.org/nix/download.html) can inst
|
|||
nix-shell nimbus.nix
|
||||
```
|
||||
|
||||
### Build & Install
|
||||
### Build & Develop
|
||||
|
||||
We use [Nimble](https://github.com/nim-lang/nimble) to manage dependencies and run tests.
|
||||
#### POSIX-compatible OS
|
||||
|
||||
To build and install Nimbus in your home folder, just execute:
|
||||
To build Nimbus (in "build/nimbus"), just execute:
|
||||
|
||||
```bash
|
||||
nimble install
|
||||
make
|
||||
```
|
||||
|
||||
After a succesful installation, running `nimbus --help` will provide you with a list of
|
||||
the available command-line options. To start syncing with mainnet, just execute `nimbus`
|
||||
Running `./build/nimbus --help` will provide you with a list of
|
||||
the available command-line options. To start syncing with mainnet, just execute `./build/nimbus`
|
||||
without any parameters.
|
||||
|
||||
To execute all tests:
|
||||
```bash
|
||||
nimble test
|
||||
make test
|
||||
```
|
||||
|
||||
To pull the latest changes in all the Git repositories involved:
|
||||
```bash
|
||||
make update
|
||||
```
|
||||
|
||||
To run a command that might use binaries from the Status Nim fork:
|
||||
```bash
|
||||
./env.sh vim
|
||||
```
|
||||
|
||||
Our Wiki provides additional helpful information for [debugging individual test cases][1]
|
||||
and for [pairing Nimbus with a locally running copy of Geth][2].
|
||||
|
||||
[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
|
||||
[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction
|
||||
#### Windows
|
||||
|
||||
### Build and Run Nimbus
|
||||
Install Mingw-w64 for your architecture using the "[MinGW-W64 Online
|
||||
Installer](https://sourceforge.net/projects/mingw-w64/files/)" (first link
|
||||
under the directory listing). Run it and select your architecture in the setup
|
||||
menu ("i686" on 32-bit, "x86\_64" on 64-bit), set the threads to "win32" and
|
||||
the exceptions to "dwarf" on 32-bit and "seh" on 64-bit. Change the
|
||||
installation directory to "C:\mingw-w64" and add it to your system PATH in "My
|
||||
Computer"/"This PC" -> Properties -> Advanced system settings -> Environment
|
||||
Variables -> Path -> Edit -> New -> C:\mingw-w64\mingw64\bin (it's "C:\mingw-w64\mingw32\bin" on 32-bit)
|
||||
|
||||
Install [Git for Windows](https://gitforwindows.org/) and use a "Git Bash" shell to clone and build Nimbus.
|
||||
|
||||
If you don't want to compile RocksDB and SQLite separately, you can fetch pre-compiled DLLs with:
|
||||
```bash
|
||||
nimble nimbus
|
||||
./build/nimbus
|
||||
mingw32-make fetch-dlls
|
||||
```
|
||||
|
||||
Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus)!
|
||||
This will place the right DLLs for your architecture in the "build/" directory.
|
||||
|
||||
You can now follow those instructions in the previous section by replacing `make` with `mingw32-make` (regardless of your 32-bit or 64-bit architecture).
|
||||
|
||||
### Development tips
|
||||
|
||||
|
|
@ -77,10 +97,22 @@ Report any errors you encounter, please, if not [already documented](https://git
|
|||
`-d:nimbus_db_backend=...` where the (case-insensitive) value is one of
|
||||
"rocksdb" (the default), "sqlite", "lmdb".
|
||||
|
||||
#### Troubleshooting
|
||||
### Troubleshooting
|
||||
|
||||
Report any errors you encounter, please, if not [already documented](https://github.com/status-im/nimbus/issues)!
|
||||
|
||||
|
||||
Sometimes, the build will fail even though the latest CI is green - here are a few tips to handle this:
|
||||
|
||||
#### Using the Makefile
|
||||
|
||||
* Turn it off and on again:
|
||||
```bash
|
||||
make clean update
|
||||
```
|
||||
|
||||
#### Using Nimble directly
|
||||
|
||||
* Wrong Nim version
|
||||
* We depend on many bleeding-edge features - Nim regressions often happen
|
||||
* Use the [Status fork](https://github.com/status-im/Nim) of Nim
|
||||
|
|
@ -102,3 +134,6 @@ or
|
|||
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
|
||||
|
||||
at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
[1]: https://github.com/status-im/nimbus/wiki/Understanding-and-debugging-Nimbus-EVM-JSON-tests
|
||||
[2]: https://github.com/status-im/nimbus/wiki/Debugging-state-reconstruction
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
#!/bin/sh
|
||||
|
||||
rel_path="$(dirname $0)"
|
||||
abs_path="$(cd $rel_path; pwd)"
|
||||
|
||||
#- make it an absolute path, so we can call this script from other dirs
|
||||
export PATH="${abs_path}/vendor/Nim/bin:$PATH"
|
||||
|
||||
#- Nimble needs this to be an absolute path
|
||||
export NIMBLE_DIR="${abs_path}/vendor/.nimble"
|
||||
|
||||
exec "$@"
|
||||
|
|
@ -6,9 +6,11 @@ author = "Status Research & Development GmbH"
|
|||
description = "An Ethereum 2.0 Sharding Client for Resource-Restricted Devices"
|
||||
license = "Apache License 2.0"
|
||||
skipDirs = @["tests", "examples"]
|
||||
bin = @["nimbus/nimbus"]
|
||||
# we can't have the result of a custom task in the "bin" var - https://github.com/nim-lang/nimble/issues/542
|
||||
# bin = @["build/nimbus"]
|
||||
|
||||
requires "nim >= 0.18.1",
|
||||
"chronicles",
|
||||
"nimcrypto",
|
||||
"rlp",
|
||||
"stint",
|
||||
|
|
@ -21,7 +23,7 @@ requires "nim >= 0.18.1",
|
|||
"eth_keyfile",
|
||||
"eth_keys",
|
||||
"eth_bloom",
|
||||
"https://github.com/status-im/nim-bncurve >= 1.0.1"
|
||||
"bncurve"
|
||||
|
||||
proc buildBinary(name: string, srcDir = ".", lang = "c") =
|
||||
if not dirExists "build": mkDir "build"
|
||||
|
|
@ -29,7 +31,7 @@ proc buildBinary(name: string, srcDir = ".", lang = "c") =
|
|||
setCommand lang, srcDir & name & ".nim"
|
||||
|
||||
proc test(name: string, lang = "c") =
|
||||
--define: "chronicles_log_level=ERROR"
|
||||
--define:"chronicles_log_level=ERROR"
|
||||
--run
|
||||
buildBinary name, "tests/"
|
||||
|
||||
|
|
@ -38,3 +40,4 @@ task test, "Run tests":
|
|||
|
||||
task nimbus, "Build Nimbus":
|
||||
buildBinary "nimbus", "nimbus/"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
-d:chronicles_line_numbers
|
||||
-d:"chronicles_sinks=textblocks"
|
||||
-d:nimDebugDlOpen
|
||||
|
||||
Loading…
Reference in New Issue