diff --git a/.appveyor.yml b/.appveyor.yml index 1b34df9d4..327f7266d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,10 +7,8 @@ environment: GIT_LFS_SKIP_SMUDGE: 1 cache: - - sqlite-dll-win32-x86-3240000.zip -> .appveyor.yml - - sqlite-dll-win64-x64-3240000.zip -> .appveyor.yml - - nimbus-deps.zip -> .appveyor.yml - NimBinaries + - p2pdCache - jsonTestsCache matrix: @@ -23,51 +21,22 @@ platform: install: - git submodule update --init --recursive - - setlocal EnableExtensions EnableDelayedExpansion - - - IF "%PLATFORM%" == "x86" ( - SET "SQLITE_URL=https://www.sqlite.org/2018/sqlite-dll-win32-x86-3240000.zip" & - SET "SQLITE_ARCHIVE=sqlite-dll-win32-x86-3240000.zip" & - SET "ROCKSDB_URL=https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" & - SET "ROCKSDB_ARCHIVE=nimbus-deps.zip" - ) - - IF "%PLATFORM%" == "x64" ( - SET "SQLITE_URL=https://www.sqlite.org/2018/sqlite-dll-win64-x64-3240000.zip" & - SET "SQLITE_ARCHIVE=sqlite-dll-win64-x64-3240000.zip" & - SET "ROCKSDB_URL=https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" & - SET "ROCKSDB_ARCHIVE=nimbus-deps.zip" - ) - # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - MKDIR %CD%\bin - - SET PATH=%CD%\bin;%CD%\Nim\bin;%PATH% - - # Unpack sqlite - - IF not exist "%SQLITE_ARCHIVE%" appveyor DownloadFile "%SQLITE_URL%" -FileName "%SQLITE_ARCHIVE%" - - 7z x -y "%SQLITE_ARCHIVE%" > nul - - IF "%PLATFORM%" == "x64" ( copy %CD%\sqlite3.dll %CD%\bin\sqlite3_64.dll ) ELSE ( copy %CD%\sqlite3.dll %CD%\bin\sqlite3_32.dll ) - - # Unpack rocksdb - - IF not exist "%ROCKSDB_ARCHIVE%" appveyor DownloadFile "%ROCKSDB_URL%" -FileName "%ROCKSDB_ARCHIVE%" - - 7z x -y "%ROCKSDB_ARCHIVE%" > nul - - IF "%PLATFORM%" == "x64" ( copy %CD%\x64\librocksdb.dll %CD%\bin\librocksdb.dll ) ELSE ( copy %CD%\x86\librocksdb.dll %CD%\bin\librocksdb.dll ) - - # build nim from our own branch - this to avoid the day-to-day churn and - # regressions of the fast-paced Nim development while maintaining the - # flexibility to apply patches - - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus/devel/build_nim.sh - - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries - # LFS test fixtures - bash scripts\process_lfs.sh jsonTestsCache build_script: - - bash -c "nimble install -y --depsOnly" + # the 32-bit build is done on a 64-bit image, so we need to override the architecture + - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% CI_CACHE=NimBinaries update + - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% fetch-dlls test_script: - - nimble test + # the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug + - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache + - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_LFS_SCRIPT=1 DISABLE_GO_CHECKS=1 test deploy: off + diff --git a/.gitignore b/.gitignore index 6cedf950f..4382ac221 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,19 @@ nimcache/ # Executables shall be put in an ignored build/ directory -# Ignore dynamic, static libs and libtool archive files build/ + +# Nimble packages +/vendor/.nimble + +# Go packages +/vendor/go +!/vendor/go/src/github.com/libp2p/go-libp2p-daemon + +# ntags/ctags output +/tags + +# Ignore dynamic, static libs and libtool archive files *.so *.dylib *.a @@ -10,6 +21,9 @@ build/ *.exe *.dll +# a symlink that can't be added to the repo because of Windows +/*.nims + /scripts/testnet*.sh # State sim # TODO - move in another folder diff --git a/.gitmodules b/.gitmodules index 44a95a110..cd86bf12d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,143 @@ [submodule "tests/official/fixtures"] path = tests/official/fixtures url = https://github.com/status-im/nim-eth2-official-tests +[submodule "vendor/go/src/github.com/libp2p/go-libp2p-daemon"] + path = vendor/go/src/github.com/libp2p/go-libp2p-daemon + url = https://github.com/libp2p/go-libp2p-daemon.git + ignore = dirty + branch = master +[submodule "vendor/nim-blscurve"] + path = vendor/nim-blscurve + url = https://github.com/status-im/nim-blscurve.git + ignore = dirty + branch = master +[submodule "vendor/nim-confutils"] + path = vendor/nim-confutils + url = https://github.com/status-im/nim-confutils.git + ignore = dirty + branch = master +[submodule "vendor/nim-libp2p"] + path = vendor/nim-libp2p + url = https://github.com/status-im/nim-libp2p.git + ignore = dirty + branch = master +[submodule "vendor/nimbus-build-system"] + path = vendor/nimbus-build-system + url = https://github.com/status-im/nimbus-build-system.git + ignore = dirty + branch = master +[submodule "vendor/NimYAML"] + path = vendor/NimYAML + url = https://github.com/flyx/NimYAML.git + ignore = dirty + branch = master +[submodule "vendor/nim-web3"] + path = vendor/nim-web3 + url = https://github.com/status-im/nim-web3.git + ignore = dirty + branch = master +[submodule "vendor/Nim"] + path = vendor/Nim + url = https://github.com/status-im/Nim.git + ignore = dirty + branch = master +[submodule "vendor/Nim-csources"] + ignore = dirty + branch = master + path = vendor/Nim-csources + url = https://github.com/nim-lang/csources.git +[submodule "vendor/nimble"] + path = vendor/nimble + url = https://github.com/nim-lang/nimble.git + ignore = dirty + branch = master +[submodule "vendor/nim-nat-traversal"] + path = vendor/nim-nat-traversal + url = https://github.com/status-im/nim-nat-traversal.git + ignore = dirty + branch = master +[submodule "vendor/nimcrypto"] + path = vendor/nimcrypto + url = https://github.com/cheatfate/nimcrypto.git + ignore = dirty + branch = master +[submodule "vendor/nim-stew"] + path = vendor/nim-stew + url = https://github.com/status-im/nim-stew.git + ignore = dirty + branch = master +[submodule "vendor/nim-chronos"] + path = vendor/nim-chronos + url = https://github.com/status-im/nim-chronos.git + ignore = dirty + branch = master +[submodule "vendor/nim-chronicles"] + path = vendor/nim-chronicles + url = https://github.com/status-im/nim-chronicles.git + ignore = dirty + branch = master +[submodule "vendor/nim-faststreams"] + path = vendor/nim-faststreams + url = https://github.com/status-im/nim-faststreams.git + ignore = dirty + branch = master +[submodule "vendor/nim-serialization"] + path = vendor/nim-serialization + url = https://github.com/status-im/nim-serialization.git + ignore = dirty + branch = master +[submodule "vendor/nim-json-serialization"] + path = vendor/nim-json-serialization + url = https://github.com/status-im/nim-json-serialization.git + ignore = dirty + branch = master +[submodule "vendor/nim-eth"] + path = vendor/nim-eth + url = https://github.com/status-im/nim-eth.git + ignore = dirty + branch = master +[submodule "vendor/nim-stint"] + path = vendor/nim-stint + url = https://github.com/status-im/nim-stint.git + ignore = dirty + branch = master +[submodule "vendor/nim-metrics"] + path = vendor/nim-metrics + url = https://github.com/status-im/nim-metrics.git + ignore = dirty + branch = master +[submodule "vendor/nim-rocksdb"] + path = vendor/nim-rocksdb + url = https://github.com/status-im/nim-rocksdb.git + ignore = dirty + branch = master +[submodule "vendor/nim-secp256k1"] + path = vendor/nim-secp256k1 + url = https://github.com/status-im/nim-secp256k1.git + ignore = dirty + branch = master +[submodule "vendor/nim-http-utils"] + path = vendor/nim-http-utils + url = https://github.com/status-im/nim-http-utils.git + ignore = dirty + branch = master +[submodule "vendor/nim-json-rpc"] + path = vendor/nim-json-rpc + url = https://github.com/status-im/nim-json-rpc.git + ignore = dirty + branch = master +[submodule "vendor/news"] + path = vendor/news + url = https://github.com/tormund/news.git + ignore = dirty + branch = master +[submodule "vendor/nim-result"] + path = vendor/nim-result + url = https://github.com/arnetheduck/nim-result.git + ignore = dirty + branch = master +[submodule "vendor/nim-byteutils"] + path = vendor/nim-byteutils + url = https://github.com/status-im/nim-byteutils.git + ignore = dirty + branch = master diff --git a/.travis.yml b/.travis.yml index 8d42bf05e..414b86522 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: go # https://docs.travis-ci.com/user/caching/ cache: directories: - - NimBinaries + - vendor/Nim/bin + - vendor/go/bin - rocksdbCache - jsonTestsCache @@ -27,21 +28,14 @@ matrix: - launchctl setenv LIBRARY_PATH /usr/local/lib # for RocksDB install: - # build nim from our own branch - this to avoid the day-to-day churn and - # regressions of the fast-paced Nim development while maintaining the - # flexibility to apply patches - - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus/devel/build_nim.sh - - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries - - export PATH=$PWD/Nim/bin:$PATH - # build our own rocksdb to test with a fixed version that we think works - - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus/devel/build_rocksdb.sh - - bash build_rocksdb.sh rocksdbCache + - vendor/nimbus-build-system/scripts/build_rocksdb.sh rocksdbCache # LFS test fixtures - scripts/process_lfs.sh jsonTestsCache script: - - nimble install -y --depsOnly - - nimble test + - set -e # fail fast + - make -j2 + - make -j2 DISABLE_LFS_SCRIPT=1 test diff --git a/Makefile b/Makefile index 7036379b1..46b1e4edf 100644 --- a/Makefile +++ b/Makefile @@ -5,47 +5,67 @@ # at your option. This file may not be copied, modified, or distributed except # according to those terms. -# we don't want an error here, so we can explain things later, in the sanity-checks target --include ../../common.mk +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system -ENV_SCRIPT := "../../env.sh" +# we don't want an error here, so we can handle things later, in the build-system-checks target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk TOOLS := beacon_node validator_keygen bench_bls_sig_agggregation state_sim TOOLS_DIRS := beacon_chain benchmarks research TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) -DISABLE_LFS_SCRIPT := 0 +.PHONY: all build-system-checks deps update go-checks p2pd test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0 clean-testnet1 testnet1 clean -.PHONY: all sanity-checks deps nat-libs p2pd test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0-nocleaning testnet0 clean-testnet1 testnet1-nocleaning testnet1 clean +all: | build-system-checks $(TOOLS) -all: | $(TOOLS) +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk -$(SILENT_TARGET_PREFIX).SILENT: +GIT_SUBMODULE_UPDATE := export GIT_LFS_SKIP_SMUDGE=1; git submodule update --init --recursive +build-system-checks: + @[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \ + echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo -e "\nYou can now run '$(MAKE)' again."; \ + exit 1; \ + } -sanity-checks: - @ [[ "$$PWD" =~ /vendor/nim-beacon-chain$ && -e ../../Makefile && -e ../../common.mk ]] || \ - { echo -e "This Makefile can only be used from the corresponding Git submodule in the Nimbus repository.\nDetailed instructions available in README.md or online at https://github.com/status-im/nim-beacon-chain/#building-and-testing"; exit 1; } - @+ $(MAKE) --silent -C ../../ sanity-checks +deps: | deps-common beacon_chain.nims -deps: | sanity-checks - @+ $(MAKE) --silent -C ../../ deps +#- deletes and recreates "beacon_chain.nims" which on Windows is a copy instead of a proper symlink +update: | update-common + rm -rf beacon_chain.nims && \ + $(MAKE) beacon_chain.nims -build: - mkdir $@ +# symlink +beacon_chain.nims: + ln -s beacon_chain.nimble $@ -nat-libs: | deps - + $(MAKE) --silent -C ../../ nat-libs +MIN_GO_VER := 1.12 +DISABLE_GO_CHECKS := 0 +go-checks: +ifeq ($(DISABLE_GO_CHECKS), 0) + which go &>/dev/null || { echo "Go compiler not installed. Aborting."; exit 1; } + GO_VER="$$(go version | sed -E 's/^.*go([0-9.]+).*$$/\1/')"; \ + [[ $$(echo -e "$${GO_VER}\n$(MIN_GO_VER)" | sort -t '.' -k 1,1 -k 2,2 -g | head -n 1) == "$(MIN_GO_VER)" ]] || \ + { echo "Minimum Go compiler version required: $(MIN_GO_VER). Version available: $$GO_VER. Aborting."; exit 1; } +endif -p2pd: | deps - + $(MAKE) --silent -C ../../ vendor/go/bin/p2pd +P2PD_CACHE := +p2pd: | go-checks + BUILD_MSG="$(BUILD_MSG) $@" \ + V=$(V) \ + $(ENV_SCRIPT) scripts/build_p2pd.sh "$(P2PD_CACHE)" # Windows 10 with WSL enabled, but no distro installed, fails if "../../nimble.sh" is executed directly # in a Makefile recipe but works when prefixing it with `bash`. No idea how the PATH is overridden. -test: | build deps nat-libs +DISABLE_LFS_SCRIPT := 0 +test: | build deps nat-libs p2pd ifeq ($(DISABLE_LFS_SCRIPT), 0) - bash scripts/process_lfs.sh $(HANDLE_OUTPUT) + scripts/process_lfs.sh $(HANDLE_OUTPUT) endif - bash ../../nimble.sh test $(NIM_PARAMS) && rm -f 0000-*.json + $(ENV_SCRIPT) nim test $(NIM_PARAMS) beacon_chain.nims && rm -f 0000-*.json $(TOOLS): | build deps nat-libs p2pd for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \ @@ -56,10 +76,10 @@ clean_eth2_network_simulation_files: rm -rf tests/simulation/{data,validators} eth2_network_simulation: | beacon_node validator_keygen clean_eth2_network_simulation_files - SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" tests/simulation/start.sh + SKIP_BUILDS=1 GIT_ROOT="$$PWD" BUILD_OUTPUTS_DIR="./build" $(ENV_SCRIPT) tests/simulation/start.sh testnet0 testnet1: | build deps nat-libs p2pd - NIM_PARAMS="$(NIM_PARAMS)" ../../env.sh scripts/build_testnet_node.sh $@ + NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) scripts/build_testnet_node.sh $@ clean-testnet0: rm -rf ~/.cache/nimbus/BeaconNode/testnet0 @@ -67,6 +87,6 @@ clean-testnet0: clean-testnet1: rm -rf ~/.cache/nimbus/BeaconNode/testnet1 -clean: - rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*.exe} nimcache +clean: | clean-common + rm -rf build/{$(TOOLS_CSV),all_tests,*_node} diff --git a/README.md b/README.md index 1d564111b..f3b15d3dd 100644 --- a/README.md +++ b/README.md @@ -11,27 +11,15 @@ Nimbus beacon chain is a research implementation of the beacon chain component o * [status-im/nimbus](https://github.com/status-im/nimbus/): main Nimbus repository - start here to learn more about the Nimbus eco-system * [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md): Serenity specification that this project implements -* [ethereum/beacon_chain](https://github.com/ethereum/beacon_chain): reference implementation from the Ethereum foundation +* [ethereum/beacon\_chain](https://github.com/ethereum/beacon_chain): reference implementation from the Ethereum foundation You can check where the beacon chain fits in the Ethereum research ecosystem in the [Status Athenaeum](https://github.com/status-im/athenaeum/blob/b465626cc551e361492e56d32517b2cdadd7493f/ethereum_research_records.json#L38). ## Building and Testing -The beacon chain components need to be built with the Nim compiler - the easiest way to get started is to head over to the main [Nimbus](https://github.com/status-im/nimbus/) repository and follow the build instructions there or just execute the commands below in order. - -_Note: This is because this repository is actually pulled in as a dependency of Nimbus - the Ethereum 1.0 + 2.0 client - so it makes sense to start from there even if you are only interested in testing the Ethereum 2.0 side of things (contained almost entirely in this repository)._ - ```bash -# Clone main nimbus repository: -git clone https://github.com/status-im/nimbus.git -cd nimbus - -# Prep environment (assuming you have 4 CPU cores and want to take advantage of them): -make update -make -j4 deps - -# Head over to the vendor repo where you should have a checkout of this project: -cd vendor/nim-beacon-chain +# this first "make" invocation will update the Git submodules +make # You can now run the test suite: make test @@ -47,7 +35,7 @@ The beacon node simulation will create a full peer-to-peer network of beacon nod make eth2_network_simulation # In another terminal, get a shell with the right environment variables set: -../../env.sh bash +./env.sh bash # Run an extra node - by default the network will launch with 9 nodes, each # hosting 10 validators. The last 10 validators are lazy bums that hid from the @@ -72,7 +60,8 @@ _Alternatively, fire up our [experimental Vagrant instance with Nim pre-installe - build all those tools known to the Makefile: ```bash -make +# (assuming you have 4 CPU cores and want to take advantage of them): +make -j4 ``` - build a specific tool: @@ -115,7 +104,7 @@ The state transition simulator can quickly run the Beacon chain state transition # build and run the state simulator, then display its help ("-d:release" speeds it # up substantially, allowing the simulation of longer runs in reasonable time) make NIMFLAGS="-d:release" state_sim -./build/state_sim --help +build/state_sim --help ``` ## Testnet @@ -124,7 +113,7 @@ The beacon chain now has a public testnet available. Connect to it with: ```bash make testnet0 -./scripts/testnet0.sh # this launches the testnet0-specific node you just built +scripts/testnet0.sh # this launches the testnet0-specific node you just built ``` For more information about the testnet and to find out how to launch your own, see [this announcement](https://our.status.im/the-nimbus-mvp-testnet-is-here/) and the [official docs on launching the testnets](https://nimbus.status.im/docs/t0.html). diff --git a/env.sh b/env.sh new file mode 100755 index 000000000..43ab3d77f --- /dev/null +++ b/env.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +exec vendor/nimbus-build-system/scripts/env.sh "$@" + diff --git a/scripts/build_p2pd.sh b/scripts/build_p2pd.sh new file mode 100755 index 000000000..31c5db547 --- /dev/null +++ b/scripts/build_p2pd.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -e + +SUBREPO_DIR="vendor/go/src/github.com/libp2p/go-libp2p-daemon" +CACHE_DIR="$1" # optional parameter pointing to a CI cache dir. + +## env vars +# verbosity level +[[ -z "$V" ]] && V=0 +[[ -z "$BUILD_MSG" ]] && BUILD_MSG="Building p2pd" + +# Windows detection +if uname | grep -qiE "mingw|msys"; then + EXE_SUFFIX=".exe" +else + EXE_SUFFIX="" +fi + +# macOS +if uname | grep -qi "darwin"; then + STAT_FORMAT="-f %m" +else + STAT_FORMAT="-c %Y" +fi + +TARGET_DIR="vendor/go/bin" +TARGET_BINARY="${TARGET_DIR}/p2pd${EXE_SUFFIX}" + +target_needs_rebuilding() { + REBUILD=0 + NO_REBUILD=1 + + if [[ -n "$CACHE_DIR" && -e "${CACHE_DIR}/p2pd${EXE_SUFFIX}" ]]; then + mkdir -p "${TARGET_DIR}" + cp -a "$CACHE_DIR"/* "${TARGET_DIR}/" + fi + + # compare binary mtime to the date of the last commit (keep in mind that Git doesn't preserve file timestamps) + if [[ -e "$TARGET_BINARY" && $(stat $STAT_FORMAT "$TARGET_BINARY") -gt $(cd "$SUBREPO_DIR"; git log --pretty=format:%cd -n 1 --date=unix) ]]; then + return $NO_REBUILD + else + return $REBUILD + fi +} + +build_target() { + echo -e "$BUILD_MSG" + [[ "$V" == "0" ]] && exec &>/dev/null + + pushd "$SUBREPO_DIR" + go get ./... + go install ./... + popd + + # update the CI cache + if [[ -n "$CACHE_DIR" ]]; then + rm -rf "$CACHE_DIR" + mkdir "$CACHE_DIR" + cp -a "$TARGET_DIR"/* "$CACHE_DIR"/ + fi +} + +if target_needs_rebuilding; then + build_target +fi + diff --git a/scripts/build_testnet_node.sh b/scripts/build_testnet_node.sh index c176e0de2..06036d602 100755 --- a/scripts/build_testnet_node.sh +++ b/scripts/build_testnet_node.sh @@ -32,7 +32,7 @@ cat > "$BEACON_NODE_SCRIPT" <