From 623cfe2e02cc64d829d221c014bb849c84ff9ae0 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 23 Nov 2021 10:55:16 -0600 Subject: [PATCH] adding nimbus build system --- .gitignore | 4 +- .gitmodules | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 102 ++++++++++++++++++++++++++++++++++++++ config.nims | 66 +++++++++++++++++++++++++ dagger.nimble | 19 ++++++++ 5 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 100644 Makefile create mode 100644 config.nims diff --git a/.gitignore b/.gitignore index fedb828b..f8626293 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ * !*/ -!*.* \ No newline at end of file +!*.* +nimcache/* +vendor/* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..2f92d4a4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,132 @@ +[submodule "vendor/nimbus-build-system"] + path = vendor/nimbus-build-system + url = https://github.com/status-im/nimbus-build-system.git + ignore = untracked + branch = master +[submodule "vendor/nim-libp2p"] + path = vendor/nim-libp2p + url = https://github.com/status-im/nim-libp2p.git + ignore = untracked + branch = master +[submodule "vendor/nimcrypto"] + path = vendor/nimcrypto + url = https://github.com/cheatfate/nimcrypto.git + ignore = untracked + branch = master +[submodule "vendor/bearssl"] + path = vendor/bearssl + url = https://github.com/status-im/bearssl.git + ignore = untracked + branch = master +[submodule "vendor/nim-chronicles"] + path = vendor/nim-chronicles + url = https://github.com/status-im/nim-chronicles.git + ignore = untracked + branch = master +[submodule "vendor/nim-metrics"] + path = vendor/nim-metrics + url = https://github.com/status-im/nim-metrics.git + ignore = untracked + branch = master +[submodule "vendor/nim-secp256k1"] + path = vendor/nim-secp256k1 + url = https://github.com/status-im/nim-secp256k1.git + ignore = untracked + branch = master +[submodule "vendor/nim-stew"] + path = vendor/nim-stew + url = https://github.com/status-im/nim-stew.git + ignore = untracked + branch = master +[submodule "vendor/nim-protobufserialization"] + ignore = untracked + branch = master +[submodule "vendor/nim-protobuf-serialization"] + path = vendor/nim-protobuf-serialization + url = https://github.com/status-im/nim-protobuf-serialization.git + ignore = untracked + branch = master +[submodule "vendor/nim-nitro"] + path = vendor/nim-nitro + url = https://github.com/status-im/nim-nitro.git + ignore = untracked + branch = master +[submodule "vendor/nim-questionable"] + ignore = untracked + branch = master +[submodule "vendor/questionable"] + path = vendor/questionable + url = https://github.com/status-im/questionable.git + ignore = untracked + branch = master +[submodule "vendor/upraises"] + ignore = untracked + branch = master + path = vendor/upraises + url = https://github.com/markspanbroek/upraises.git +[submodule "vendor/asynctest"] + path = vendor/asynctest + url = https://github.com/status-im/asynctest.git + ignore = untracked + branch = master +[submodule "vendor/nim-presto"] + path = vendor/nim-presto + url = https://github.com/status-im/nim-presto.git + ignore = untracked + branch = master +[submodule "vendor/nim-confutils"] + path = vendor/nim-confutils + url = https://github.com/status-im/nim-confutils.git + ignore = untracked + branch = master +[submodule "vendor/nim-nat-traversal"] + path = vendor/nim-nat-traversal + url = https://github.com/status-im/nim-nat-traversal.git + ignore = untracked + branch = master +[submodule "vendor/nim-libbacktrace"] + path = vendor/nim-libbacktrace + url = https://github.com/status-im/nim-libbacktrace.git + ignore = untracked + branch = master +[submodule "vendor/nim-chronos"] + path = vendor/nim-chronos + url = https://github.com/status-im/nim-chronos.git + ignore = untracked + branch = master +[submodule "vendor/nim-faststreams"] + path = vendor/nim-faststreams + url = https://github.com/status-im/nim-faststreams.git + ignore = untracked + branch = master +[submodule "vendor/nim-son_serialization"] + ignore = untracked + branch = master +[submodule "vendor/nim-json_serialization"] + ignore = untracked + branch = master +[submodule "vendor/nim-json-serialization"] + path = vendor/nim-json-serialization + url = https://github.com/status-im/nim-json-serialization.git + ignore = untracked + branch = master +[submodule "vendor/nim-serialization"] + path = vendor/nim-serialization + url = https://github.com/status-im/nim-serialization.git + ignore = untracked + branch = master +[submodule "vendor/nim-bearssl"] + path = vendor/nim-bearssl + url = https://github.com/status-im/nim-bearssl.git + ignore = untracked + branch = master +[submodule "vendor/combparser"] + path = vendor/combparser + url = https://github.com/PMunch/combparser.git + ignore = untracked + branch = master +[submodule "vendor/stint"] + path = vendor/stint + url = https://github.com/status-im/stint.git + ignore = untracked + branch = master diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e00f84e9 --- /dev/null +++ b/Makefile @@ -0,0 +1,102 @@ +# Copyright (c) 2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +SHELL := bash # the shell used internally by Make + +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system + +# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics +# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker +DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure + +LINK_PCRE := 0 + +# we don't want an error here, so we can handle things later, in the ".DEFAULT" target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk + +.PHONY: \ + all \ + deps \ + update \ + test \ + clean \ + libbacktrace + +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included, so we update the submodules. +GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +.DEFAULT: + +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo +# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: +# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +# +# After restarting, it will execute its original goal, so we don't have to start a child Make here +# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? + +else # "variables.mk" was included. Business as usual until the end of this file. + +# default target, because it's the first one that doesn't start with '.' +all: | test + +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + +# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims +ifeq ($(USE_LIBBACKTRACE), 0) +NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace +else +NIM_PARAMS := $(NIM_PARAMS) -d:release +endif + +deps: | deps-common nat-libs dagger.nims +ifneq ($(USE_LIBBACKTRACE), 0) +deps: | libbacktrace +endif + +#- deletes and recreates "dagger.nims" which on Windows is a copy instead of a proper symlink +update: | update-common + rm -rf dagger.nims && \ + $(MAKE) dagger.nims $(HANDLE_OUTPUT) + +# a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated + +testAll: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) dagger.nims + +# detecting the os +ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... + detected_OS := Windows +else ifeq ($(strip $(shell uname)),Darwin) + detected_OS := macOS +else + # e.g. Linux + detected_OS := $(strip $(shell uname)) +endif + +# Builds and run the test suite (Waku v1 + v2) +test: | testAll + +# symlink +dagger.nims: + ln -s dagger.nimble $@ + +# nim-libbacktrace +libbacktrace: + + $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 + +# usual cleaning +clean: | clean-common + rm -rf build +ifneq ($(USE_LIBBACKTRACE), 0) + + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) +endif + +endif # "variables.mk" was not included diff --git a/config.nims b/config.nims new file mode 100644 index 00000000..0abfe838 --- /dev/null +++ b/config.nims @@ -0,0 +1,66 @@ +if defined(release): + switch("nimcache", "nimcache/release/$projectName") +else: + switch("nimcache", "nimcache/debug/$projectName") + +if defined(windows): + # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries + switch("passL", "-Wl,--no-insert-timestamp") + # increase stack size + switch("passL", "-Wl,--stack,8388608") + # https://github.com/nim-lang/Nim/issues/4057 + --tlsEmulation:off + if defined(i386): + # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM + switch("passL", "-Wl,--large-address-aware") + + # The dynamic Chronicles output currently prevents us from using colors on Windows + # because these require direct manipulations of the stdout File object. + switch("define", "chronicles_colors=off") + +# This helps especially for 32-bit x86, which sans SSE2 and newer instructions +# requires quite roundabout code generation for cryptography, and other 64-bit +# and larger arithmetic use cases, along with register starvation issues. When +# engineering a more portable binary release, this should be tweaked but still +# use at least -msse2 or -msse3. +if defined(disableMarchNative): + switch("passC", "-msse3") +else: + switch("passC", "-march=native") + if defined(windows): + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782 + # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes) + switch("passC", "-mno-avx512vl") + +--threads:on +--opt:speed +--excessiveStackTrace:on +# enable metric collection +--define:metrics +# for heap-usage-by-instance-type metrics and object base-type strings +--define:nimTypeNames + +switch("define", "withoutPCRE") + +# the default open files limit is too low on macOS (512), breaking the +# "--debugger:native" build. It can be increased with `ulimit -n 1024`. +if not defined(macosx): + # add debugging symbols and original files and line numbers + --debugger:native + if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace): + # light-weight stack traces using libbacktrace and libunwind + --define:nimStackTraceOverride + switch("import", "libbacktrace") + +--define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9 + +# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'" +switch("warning", "CaseTransition:off") + +# The compiler doth protest too much, methinks, about all these cases where it can't +# do its (N)RVO pass: https://github.com/nim-lang/RFCs/issues/230 +switch("warning", "ObservableStores:off") + +# Too many false positives for "Warning: method has lock level , but another method has 0 [LockLevel]" +switch("warning", "LockLevel:off") + diff --git a/dagger.nimble b/dagger.nimble index c61eb536..d12ceb5a 100644 --- a/dagger.nimble +++ b/dagger.nimble @@ -1,3 +1,5 @@ +mode = ScriptMode.Verbose + version = "0.1.0" author = "Dagger Team" description = "p2p data durability engine" @@ -16,3 +18,20 @@ requires "libp2p#unstable", "questionable >= 0.9.1 & < 0.10.0", "upraises >= 0.1.0 & < 0.2.0", "asynctest >= 0.3.0 & < 0.4.0" + +### Helper functions +proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = + if not dirExists "build": + mkDir "build" + # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" + var extra_params = params + for i in 2..