From a7618a6491b76fdcde72d50d2e49dbd3b0fd4014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Tue, 20 Aug 2019 23:14:45 +0200 Subject: [PATCH] initial commit --- .gitignore | 3 + LICENSE-APACHEv2 | 201 ++++++++++++++++++++++++++++++++++ LICENSE-MIT | 21 ++++ README.md | 18 +++ makefiles/targets.mk | 196 +++++++++++++++++++++++++++++++++ makefiles/variables.mk | 84 ++++++++++++++ scripts/add_submodule.sh | 20 ++++ scripts/build_nim.sh | 141 ++++++++++++++++++++++++ scripts/build_rocksdb.sh | 39 +++++++ scripts/create_nimble_link.sh | 22 ++++ scripts/env.sh | 39 +++++++ scripts/nimble.sh | 36 ++++++ 12 files changed, 820 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE-APACHEv2 create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 makefiles/targets.mk create mode 100644 makefiles/variables.mk create mode 100755 scripts/add_submodule.sh create mode 100755 scripts/build_nim.sh create mode 100755 scripts/build_rocksdb.sh create mode 100755 scripts/create_nimble_link.sh create mode 100755 scripts/env.sh create mode 100755 scripts/nimble.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8181e4a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.swp +*~ + diff --git a/LICENSE-APACHEv2 b/LICENSE-APACHEv2 new file mode 100644 index 0000000..782d1bf --- /dev/null +++ b/LICENSE-APACHEv2 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Status Research & Development GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..4d2c763 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019 Status Research & Development GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..452fa58 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Build system used by Nimbus and related projects + +## Prerequisites + +GNU make, Bash and the usual POSIX utilities + +## License + +Licensed and distributed under either of + +* MIT license: [LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT + +or + +* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) + +at your option. These files may not be copied, modified, or distributed except according to those terms. + diff --git a/makefiles/targets.mk b/makefiles/targets.mk new file mode 100644 index 0000000..4d69d55 --- /dev/null +++ b/makefiles/targets.mk @@ -0,0 +1,196 @@ +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +.PHONY: deps-common sanity-checks nat-libs libminiupnpc.a libnatpmp.a clean-common mrproper github-ssh build-nim update-common update-remote status go-checks ntags ctags fetch-dlls + +#- when the special ".SILENT" target is present, all recipes are silenced as if they all had a "@" prefix +#- by setting SILENT_TARGET_PREFIX to a non-empty value, the name of this target becomes meaningless to `make` +#- idea stolen from http://make.mad-scientist.net/managing-recipe-echoing/ +$(SILENT_TARGET_PREFIX).SILENT: + +# dir +build: + mkdir $@ + +sanity-checks: + which $(CC) &>/dev/null || { echo "C compiler ($(CC)) not installed. Aborting."; exit 1; } + +#- runs only the first time and after `make update`, so have "normal" +# (timestamp-checked) prerequisites here +#- $(NIM_BINARY) is both a proxy for submodules having been initialised +# and a check for the actual compiler build +deps-common: sanity-checks $(NIM_BINARY) $(NIMBLE_DIR) + +#- conditionally re-builds the Nim compiler (not usually needed, because `make update` calls this rule; delete $(NIM_BINARY) to force it) +#- allows parallel building with the '+' prefix +#- forces a rebuild of csources, Nimble and a complete compiler rebuild, in case we're called after pulling a new Nim version +#- uses our Git submodules for csources and Nimble (Git doesn't let us place them in another submodule) +#- build_all.sh looks at the parent dir to decide whether to copy the resulting csources binary there, +# but this is broken when using symlinks, so build csources separately (we get parallel compiling as a bonus) +#- Windows is a special case, as usual +#- macOS is also a special case, with its "ln" not supporting "-r" +#- the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection with ARCH_OVERRIDE +build-nim: | sanity-checks + + NIM_BUILD_MSG="$(BUILD_MSG) Nim compiler" \ + V=$(V) \ + CC=$(CC) \ + MAKE=$(MAKE) \ + ARCH_OVERRIDE=$(ARCH_OVERRIDE) \ + "$(CURDIR)/$(BUILD_SYSTEM_DIR)/scripts/build_nim.sh" "$(NIM_DIR)" ../Nim-csources ../nimble "$(CI_CACHE)" + +#- in case of submodule URL changes, it propagates that change in the parent repo's .git directory +#- initialises and updates the Git submodules, avoiding automated LFS downloads +#- manages the AppVeyor cache of Nim compiler binaries +#- deletes the ".nimble" dir to force the execution of the "deps" target +#- allows parallel building with the '+' prefix +#- rebuilds the Nim compiler if the corresponding submodule is updated +$(NIM_BINARY) update-common: | sanity-checks + git submodule sync --quiet --recursive + export GIT_LFS_SKIP_SMUDGE=1; git submodule update --init --recursive + rm -rf $(NIMBLE_DIR) + + $(MAKE) build-nim + +nat-libs: | libminiupnpc.a libnatpmp.a + +libminiupnpc.a: | deps +ifeq ($(OS), Windows_NT) + + [ -e vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/$@ ] || \ + $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc -f Makefile.mingw CC=gcc init $@ $(HANDLE_OUTPUT) +else + + $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc $@ $(HANDLE_OUTPUT) +endif + +libnatpmp.a: | deps +ifeq ($(OS), Windows_NT) + + $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp CC=gcc CFLAGS="-Wall -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR" $@ $(HANDLE_OUTPUT) +else + + $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp $@ $(HANDLE_OUTPUT) +endif + +#- depends on Git submodules being initialised +#- fakes a Nimble package repository with the minimum info needed by the Nim compiler +# for runtime path (i.e.: the second line in $(NIMBLE_DIR)/pkgs/*/*.nimble-link) +$(NIMBLE_DIR): | $(NIM_BINARY) + mkdir -p $(NIMBLE_DIR)/pkgs + NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" PWD_CMD="$(PWD)" \ + git submodule foreach --quiet '$(CURDIR)/$(BUILD_SYSTEM_DIR)/scripts/create_nimble_link.sh "$$sm_path"' + +clean-common: + rm -rf build/{*.exe,*.so,*.so.0} vendor/go/bin $(NIMBLE_DIR) $(NIM_BINARY) $(NIM_DIR)/nimcache nimcache + + $(MAKE) -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc clean $(HANDLE_OUTPUT) + + $(MAKE) -C vendor/nim-nat-traversal/vendor/libnatpmp clean $(HANDLE_OUTPUT) + +# dangerous cleaning, because you may have not-yet-pushed branches and commits in those vendor repos you're about to delete +mrproper: clean + rm -rf vendor + +# for when you want to use SSH keys with GitHub +github-ssh: + git config url."git@github.com:".insteadOf "https://github.com/" + git submodule foreach --recursive 'git config url."git@github.com:".insteadOf "https://github.com/"' + +# don't use this target, or you risk updating dependency repos that are not ready to be used in Nimbus +update-remote: + git submodule update --remote + +# runs `git status` in all Git repos +status: | $(REPOS) + $(eval CMD := $(GIT_STATUS)) + $(RUN_CMD_IN_ALL_REPOS) + +MIN_GO_VER := 1.12 +go-checks: + which go &>/dev/null || { echo "Go compiler not installed. Aborting."; exit 1; } + GO_VER="$$(go version | sed -E 's/^.*go([0-9.]+).*$$/\1/')"; \ + [[ $$(echo -e "$${GO_VER}\n$(MIN_GO_VER)" | sort -t '.' -k 1,1 -k 2,2 -g | head -n 1) == "$(MIN_GO_VER)" ]] || \ + { echo "Minimum Go compiler version required: $(MIN_GO_VER). Version available: $$GO_VER. Aborting."; exit 1; } + +vendor/go/bin/p2pd: | go-checks + echo -e $(BUILD_MSG) "$@" && \ + cd vendor/go/src/github.com/libp2p/go-libp2p-daemon && \ + $(ENV_SCRIPT) go get ./... $(HANDLE_OUTPUT) && \ + $(ENV_SCRIPT) go install ./... $(HANDLE_OUTPUT) + +# https://bitbucket.org/nimcontrib/ntags/ - currently fails with "out of memory" +ntags: + ntags -R . + +#- a few files need to be excluded because they trigger an infinite loop in https://github.com/universal-ctags/ctags +#- limiting it to Nim files, because there are a lot of C files we don't care about +ctags: + ctags -R --verbose=yes \ + --langdef=nim \ + --langmap=nim:.nim \ + --regex-nim='/(\w+)\*?\s*=\s*object/\1/c,class/' \ + --regex-nim='/(\w+)\*?\s*=\s*enum/\1/e,enum/' \ + --regex-nim='/(\w+)\*?\s*=\s*tuple/\1/t,tuple/' \ + --regex-nim='/(\w+)\*?\s*=\s*range/\1/s,subrange/' \ + --regex-nim='/(\w+)\*?\s*=\s*proc/\1/p,proctype/' \ + --regex-nim='/proc\s+(\w+)/\1/f,procedure/' \ + --regex-nim='/func\s+(\w+)/\1/f,procedure/' \ + --regex-nim='/method\s+(\w+)/\1/m,method/' \ + --regex-nim='/proc\s+`([^`]+)`/\1/o,operator/' \ + --regex-nim='/template\s+(\w+)/\1/u,template/' \ + --regex-nim='/macro\s+(\w+)/\1/v,macro/' \ + --languages=nim \ + --exclude=nimcache \ + --exclude='*/Nim/tinyc' \ + --exclude='*/Nim/tests' \ + --exclude='*/Nim/csources' \ + --exclude=nimbus/genesis_alloc.nim \ + --exclude=$(REPOS_DIR)/nim-bncurve/tests/tvectors.nim \ + . + +############################ +# Windows-specific section # +############################ + +ifeq ($(OS), Windows_NT) + # no tabs allowed for indentation here + + # the AppVeyor 32-build is done on a 64-bit image, so we need to override the architecture detection + ifeq ($(ARCH_OVERRIDE), x86) + ARCH := x86 + else + ifeq ($(PROCESSOR_ARCHITEW6432), AMD64) + ARCH := x64 + else + ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) + ARCH := x64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE), x86) + ARCH := x86 + endif + endif + endif + + ifeq ($(ARCH), x86) + ROCKSDB_DIR := x86 + endif + ifeq ($(ARCH), x64) + ROCKSDB_DIR := x64 + endif + + ROCKSDB_ARCHIVE := nimbus-deps.zip + ROCKSDB_URL := https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/$(ROCKSDB_ARCHIVE) + CURL := curl -O -L + UNZIP := unzip -o + +#- back to tabs +#- copied from .appveyor.yml +#- this is why we can't delete the whole "build" dir in the "clean" target +fetch-dlls: | build deps + cd build && \ + $(CURL) $(ROCKSDB_URL) && \ + $(CURL) https://nim-lang.org/download/dlls.zip && \ + $(UNZIP) $(ROCKSDB_ARCHIVE) && \ + cp -a $(ROCKSDB_DIR)/*.dll . && \ + $(UNZIP) dlls.zip && \ + mkdir -p ../vendor/nim-beacon-chain/build && \ + cp -a *.dll ../vendor/nim-beacon-chain/build/ +endif + diff --git a/makefiles/variables.mk b/makefiles/variables.mk new file mode 100644 index 0000000..f06861d --- /dev/null +++ b/makefiles/variables.mk @@ -0,0 +1,84 @@ +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +SHELL := bash # the shell used internally by "make" + +CC := gcc +LD := $(CC) + +#- extra parameters for the Nim compiler +#- NIMFLAGS should come from the environment or make's command line +NIM_PARAMS := $(NIMFLAGS) + +# verbosity level +V := 0 +NIM_PARAMS := $(NIM_PARAMS) --verbosity:$(V) +HANDLE_OUTPUT := +SILENT_TARGET_PREFIX := disabled +ifeq ($(V), 0) + NIM_PARAMS := $(NIM_PARAMS) --hints:off --warnings:off + HANDLE_OUTPUT := &>/dev/null + SILENT_TARGET_PREFIX := +endif + +# Chronicles log level +LOG_LEVEL := +ifdef LOG_LEVEL + NIM_PARAMS := $(NIM_PARAMS) -d:chronicles_log_level=$(LOG_LEVEL) +endif + +# avoid a "libpcre.so.3: cannot open shared object file: No such file or directory" message, where possible +ifneq ($(OS), Windows_NT) + NIM_PARAMS := $(NIM_PARAMS) -d:usePcreHeader --passL:\"-lpcre\" +endif + +# guess who does parsing before variable expansion +COMMA := , +EMPTY := +SPACE := $(EMPTY) $(EMPTY) + +# coloured messages +BUILD_MSG := "\\e[92mBuilding:\\e[39m" + +GIT_CLONE := git clone --quiet --recurse-submodules +GIT_PULL := git pull --recurse-submodules +GIT_STATUS := git status +#- the Nimble dir can't be "[...]/vendor", or Nimble will start looking for +# version numbers in repo dirs (because those would be in its subdirectories) +#- duplicated in "env.sh" for the env var with the same name +NIMBLE_DIR := vendor/.nimble +REPOS_DIR := vendor +ifeq ($(OS), Windows_NT) + PWD := pwd -W +else + PWD := pwd +endif +ifeq ($(BUILD_SYSTEM_DIR),) + $(error You need to define BUILD_SYSTEM_DIR before including this file) +endif +# we want a "recursively expanded" (delayed interpolation) variable here, so we can set CMD in rule recipes +RUN_CMD_IN_ALL_REPOS = git submodule foreach --recursive --quiet 'echo -e "\n\e[32m$$name:\e[39m"; $(CMD)'; echo -e "\n\e[32m$$($(PWD)):\e[39m"; $(CMD) +# absolute path, since it will be run at various subdirectory depths +ENV_SCRIPT := "$(CURDIR)/$(BUILD_SYSTEM_DIR)/scripts/env.sh" +# duplicated in "env.sh" to prepend NIM_DIR/bin to PATH +NIM_DIR := vendor/Nim + +ifeq ($(OS), Windows_NT) + EXE_SUFFIX := .exe +else + EXE_SUFFIX := +endif +NIM_BINARY := $(NIM_DIR)/bin/nim$(EXE_SUFFIX) +# md5sum - macOS is a special case +ifeq ($(shell uname), Darwin) + MD5SUM := md5 -r +else + MD5SUM := md5sum +endif +# AppVeyor/Travis cache of $(NIM_DIR)/bin +CI_CACHE := + diff --git a/scripts/add_submodule.sh b/scripts/add_submodule.sh new file mode 100755 index 0000000..189b18f --- /dev/null +++ b/scripts/add_submodule.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +[ -z "$1" -o `echo "$1" | tr '/' '\n' | wc -l` != 2 ] && \ + { echo "usage: `basename $0` some/repo [destdir] # 'destdir' defaults to 'vendor/repo'"; exit 1; } +REPO="$1" + +DEST="vendor/${REPO#*/}" +[ -n "$2" ] && DEST="$2" + +git submodule add --force https://github.com/${REPO}.git "$DEST" +git config -f .gitmodules submodule.${DEST}.ignore dirty +git config -f .gitmodules submodule.${DEST}.branch master + diff --git a/scripts/build_nim.sh b/scripts/build_nim.sh new file mode 100755 index 0000000..4bf376a --- /dev/null +++ b/scripts/build_nim.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# used in Travis CI and AppVeyor scripts + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -e + +# Git commits +CSOURCES_COMMIT="b56e49bbedf62db22eb26388f98262e2948b2cbc" # 0.19.0 +NIMBLE_COMMIT="c8d79fc0228682677330a9f57d14389aaa641153" # Mar 26 10:06:06 2019 + +# script arguments +[[ $# -ne 4 ]] && { echo "usage: $0 nim_dir csources_dir nimble_dir ci_cache_dir"; exit 1; } +NIM_DIR="$1" +CSOURCES_DIR="$2" +NIMBLE_DIR="$3" +CI_CACHE="$4" + +## env vars +# verbosity level +[[ -z "$V" ]] && V=0 +[[ -z "$CC" ]] && CC="gcc" +# to build csources in parallel, set MAKE="make -jN" +[[ -z "$MAKE" ]] && MAKE="make" +# for 32-bit binaries on a 64-bit Windows host +UCPU="" +[[ "$ARCH_OVERRIDE" == "x86" ]] && UCPU="ucpu=i686" +[[ -z "$NIM_BUILD_MSG" ]] && NIM_BUILD_MSG="Building the Nim compiler" + +# Windows detection +if uname | grep -qiE "mingw|msys"; then + ON_WINDOWS=1 + EXE_SUFFIX=".exe" +else + ON_WINDOWS=0 + EXE_SUFFIX="" +fi +# macOS +if uname | grep -qi "darwin"; then + STAT_FORMAT="-f %m" +else + STAT_FORMAT="-c %Y" +fi + +NIM_BINARY="${NIM_DIR}/bin/nim${EXE_SUFFIX}" + +nim_needs_rebuilding() { + REBUILD=0 + NO_REBUILD=1 + + if [[ ! -e "$NIM_DIR" ]]; then + git clone --depth=1 https://github.com/status-im/Nim.git "$NIM_DIR" + fi + + if [[ -n "$CI_CACHE" && -d "$CI_CACHE" ]]; then + cp -a "$CI_CACHE"/* "$NIM_DIR"/bin/ || true # let this one fail with an empty cache dir + fi + + # compare binary mtime to the date of the last commit (keep in mind that Git doesn't preserve file timestamps) + if [[ -e "$NIM_BINARY" && $(stat $STAT_FORMAT "$NIM_BINARY") -gt $(cd "$NIM_DIR"; git log --pretty=format:%cd -n 1 --date=unix) ]]; then + return $NO_REBUILD + else + return $REBUILD + fi +} + +build_nim() { + echo -e "$NIM_BUILD_MSG" + [[ "$V" == "0" ]] && exec &>/dev/null + + # working directory + pushd "$NIM_DIR" + + # Git repos for csources and Nimble + if [[ ! -d "$CSOURCES_DIR" ]]; then + mkdir -p "$CSOURCES_DIR" + pushd "$CSOURCES_DIR" + git clone https://github.com/nim-lang/csources.git . + git checkout $CSOURCES_COMMIT + popd + fi + if [[ "$CSOURCES_DIR" != "csources" ]]; then + rm -rf csources + ln -s "$CSOURCES_DIR" csources + fi + + if [[ ! -d "$NIMBLE_DIR" ]]; then + mkdir -p "$NIMBLE_DIR" + pushd "$NIMBLE_DIR" + git clone https://github.com/nim-lang/nimble.git . + git checkout $NIMBLE_COMMIT + # we have to delete .git or koch.nim will checkout a branch tip, overriding our target commit + rm -rf .git + popd + fi + if [[ "$NIMBLE_DIR" != "dist/nimble" ]]; then + mkdir -p dist + rm -rf dist/nimble + ln -s ../"$NIMBLE_DIR" dist/nimble + fi + + # bootstrap the Nim compiler and build the tools + rm -rf bin/nim_csources + pushd csources + if [[ "$ON_WINDOWS" == "0" ]]; then + $MAKE clean + $MAKE LD=$CC + else + $MAKE myos=windows $UCPU clean + $MAKE myos=windows $UCPU CC=gcc LD=gcc + fi + popd + if [[ -e csources/bin ]]; then + cp -a csources/bin/nim bin/nim + cp -a csources/bin/nim bin/nim_csources + rm -rf csources/bin + else + cp -a bin/nim bin/nim_csources + fi + sed 's/koch tools/koch --stable tools/' build_all.sh > build_all_custom.sh + sh build_all_custom.sh + rm build_all_custom.sh + + # update the CI cache + popd # we were in $NIM_DIR + if [[ -n "$CI_CACHE" ]]; then + rm -rf "$CI_CACHE" + mkdir "$CI_CACHE" + cp -a "$NIM_DIR"/bin/* "$CI_CACHE"/ + fi +} + +if nim_needs_rebuilding; then + build_nim +fi + diff --git a/scripts/build_rocksdb.sh b/scripts/build_rocksdb.sh new file mode 100755 index 0000000..73704a3 --- /dev/null +++ b/scripts/build_rocksdb.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# used in Travis CI scripts + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -e + +ROCKSDBVER="5.17.2" + +# script arguments +[[ $# -ne 1 ]] && { echo "usage: $0 ci_cache_dir"; exit 1; } +CI_CACHE="$1" # here we cache the installed files + +# install from cache and exit, if the version we want is already there +if [[ -n "$CI_CACHE" ]] && ls "$CI_CACHE"/lib/librocksdb* 2>/dev/null | grep -q "$ROCKSDBVER"; then + sudo cp -a "$CI_CACHE"/* /usr/local/ + exit 0 +fi + +# build it +echo "Building RocksDB" +curl -O -L -s -S https://github.com/facebook/rocksdb/archive/v$ROCKSDBVER.tar.gz +tar xzf v$ROCKSDBVER.tar.gz +cd rocksdb-$ROCKSDBVER +make DISABLE_WARNING_AS_ERROR=1 -j2 shared_lib + +# install it +if [[ -n "../$CI_CACHE" ]]; then + rm -rf "../$CI_CACHE" + mkdir "../$CI_CACHE" + make INSTALL_PATH="../$CI_CACHE" install-shared + sudo cp -a "../$CI_CACHE"/* /usr/local/ +fi + diff --git a/scripts/create_nimble_link.sh b/scripts/create_nimble_link.sh new file mode 100755 index 0000000..a0da6ee --- /dev/null +++ b/scripts/create_nimble_link.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -u + +module_name="${1#*/}" + +if [ `ls -1 *.nimble 2>/dev/null | wc -l ` -gt 0 ]; then + mkdir -p "${NIMBLE_DIR}/pkgs/${module_name}-#head" + PKG_DIR="$(${PWD_CMD})" + if [ -d src ]; then + PKG_DIR="${PKG_DIR}/src" + fi + echo -e "${PKG_DIR}\n${PKG_DIR}" > "${NIMBLE_DIR}/pkgs/${module_name}-#head/${module_name}.nimble-link" +fi + diff --git a/scripts/env.sh b/scripts/env.sh new file mode 100755 index 0000000..ec65c38 --- /dev/null +++ b/scripts/env.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +#PWD_CMD="pwd" +## get native Windows paths on Mingw +#uname | grep -qi mingw && PWD_CMD="pwd -W" + +REL_PATH="$(dirname $0)" +ABS_PATH="$(cd ${REL_PATH}; pwd)" +# do we still need this? +#ABS_PATH_NATIVE="$(cd ${REL_PATH}; ${PWD_CMD})" + +export NIMBUS_ENV_DIR="${ABS_PATH}" + +# used by libp2p/go-libp2p-daemon +export GOPATH="${ABS_PATH}/../../go" +export GO111MODULE=on + +#- make it an absolute path, so we can call this script from other dirs +#- we can't use native Windows paths in here, because colons can't be escaped in PATH +export PATH="${ABS_PATH}/../../Nim/bin:${GOPATH}/bin:${PATH}" + +# Nimble needs this to be an absolute path +export NIMBLE_DIR="${ABS_PATH}/../../.nimble" + +# used by nim-beacon-chain/tests/simulation/start.sh +export BUILD_OUTPUTS_DIR="${ABS_PATH}/../../../build" + +# change the prompt in shells that source this file +export PS1="${PS1%\\\$ } [Nimbus env]\\$ " + +exec "$@" + diff --git a/scripts/nimble.sh b/scripts/nimble.sh new file mode 100755 index 0000000..e206296 --- /dev/null +++ b/scripts/nimble.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2018-2019 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +###################################################### +# run *.nimble tasks using "nim" instead of "nimble" # +###################################################### + +# exit on command error +set -e + +[ -z "$1" ] && { echo "usage: $0 task_name"; exit 1; } + +F="" +for F in *.nimble; do + # get the first one + break +done +[ -z "$F" ] && { echo "No *.nimble file found."; exit 1; } + +# "nim" seems to only run custom NimScript files if they have a "nims" extension +NIMS="${F%.nimble}.nims" +# delete the temporary symlink on script exit +cleanup() { + rm -rf "$NIMS" +} +[ -e "$NIMS" ] || { ln -s "$F" "$NIMS"; trap cleanup EXIT; } + +# can't have an "exec" here or the EXIT pseudo-signal won't be triggered +$(dirname $0)/env.sh nim "$@" "$NIMS" +