adding nimbus build system

.gitignore

@@ -1,3 +1,5 @@
 *
 !*/
 !*.*
+nimcache/*
+vendor/*

.gitmodules

@@ -0,0 +1,132 @@
+[submodule "vendor/nimbus-build-system"]
+	path = vendor/nimbus-build-system
+	url = https://github.com/status-im/nimbus-build-system.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-libp2p"]
+	path = vendor/nim-libp2p
+	url = https://github.com/status-im/nim-libp2p.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nimcrypto"]
+	path = vendor/nimcrypto
+	url = https://github.com/cheatfate/nimcrypto.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/bearssl"]
+	path = vendor/bearssl
+	url = https://github.com/status-im/bearssl.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-chronicles"]
+	path = vendor/nim-chronicles
+	url = https://github.com/status-im/nim-chronicles.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-metrics"]
+	path = vendor/nim-metrics
+	url = https://github.com/status-im/nim-metrics.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-secp256k1"]
+	path = vendor/nim-secp256k1
+	url = https://github.com/status-im/nim-secp256k1.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-stew"]
+	path = vendor/nim-stew
+	url = https://github.com/status-im/nim-stew.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-protobufserialization"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-protobuf-serialization"]
+	path = vendor/nim-protobuf-serialization
+	url = https://github.com/status-im/nim-protobuf-serialization.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-nitro"]
+	path = vendor/nim-nitro
+	url = https://github.com/status-im/nim-nitro.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-questionable"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/questionable"]
+	path = vendor/questionable
+	url = https://github.com/status-im/questionable.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/upraises"]
+	ignore = untracked
+	branch = master
+	path = vendor/upraises
+	url = https://github.com/markspanbroek/upraises.git
+[submodule "vendor/asynctest"]
+	path = vendor/asynctest
+	url = https://github.com/status-im/asynctest.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-presto"]
+	path = vendor/nim-presto
+	url = https://github.com/status-im/nim-presto.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-confutils"]
+	path = vendor/nim-confutils
+	url = https://github.com/status-im/nim-confutils.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-nat-traversal"]
+	path = vendor/nim-nat-traversal
+	url = https://github.com/status-im/nim-nat-traversal.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-libbacktrace"]
+	path = vendor/nim-libbacktrace
+	url = https://github.com/status-im/nim-libbacktrace.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-chronos"]
+	path = vendor/nim-chronos
+	url = https://github.com/status-im/nim-chronos.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-faststreams"]
+	path = vendor/nim-faststreams
+	url = https://github.com/status-im/nim-faststreams.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-son_serialization"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-json_serialization"]
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-json-serialization"]
+	path = vendor/nim-json-serialization
+	url = https://github.com/status-im/nim-json-serialization.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-serialization"]
+	path = vendor/nim-serialization
+	url = https://github.com/status-im/nim-serialization.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-bearssl"]
+	path = vendor/nim-bearssl
+	url = https://github.com/status-im/nim-bearssl.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/combparser"]
+	path = vendor/combparser
+	url = https://github.com/PMunch/combparser.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/stint"]
+	path = vendor/stint
+	url = https://github.com/status-im/stint.git
+	ignore = untracked
+	branch = master

Makefile

@@ -0,0 +1,102 @@
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+SHELL := bash # the shell used internally by Make
+
+# used inside the included makefiles
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
+
+LINK_PCRE := 0
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+.PHONY: \
+	all \
+	deps \
+	update \
+	test \
+	clean \
+	libbacktrace
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# default target, because it's the first one that doesn't start with '.'
+all: | test
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
+ifeq ($(USE_LIBBACKTRACE), 0)
+NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
+else
+NIM_PARAMS := $(NIM_PARAMS) -d:release
+endif
+
+deps: | deps-common nat-libs dagger.nims
+ifneq ($(USE_LIBBACKTRACE), 0)
+deps: | libbacktrace
+endif
+
+#- deletes and recreates "dagger.nims" which on Windows is a copy instead of a proper symlink
+update: | update-common
+	rm -rf dagger.nims && \
+		$(MAKE) dagger.nims $(HANDLE_OUTPUT)
+
+# a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated
+
+testAll: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) dagger.nims
+
+# detecting the os
+ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
+ detected_OS := Windows
+else ifeq ($(strip $(shell uname)),Darwin)
+ detected_OS := macOS
+else
+ # e.g. Linux
+ detected_OS := $(strip $(shell uname))
+endif
+
+# Builds and run the test suite (Waku v1 + v2)
+test: | testAll
+
+# symlink
+dagger.nims:
+	ln -s dagger.nimble $@
+
+# nim-libbacktrace
+libbacktrace:
+	+ $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
+
+# usual cleaning
+clean: | clean-common
+	rm -rf build
+ifneq ($(USE_LIBBACKTRACE), 0)
+	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
+endif
+
+endif # "variables.mk" was not included

config.nims

@@ -0,0 +1,66 @@
+if defined(release):
+  switch("nimcache", "nimcache/release/$projectName")
+else:
+  switch("nimcache", "nimcache/debug/$projectName")
+
+if defined(windows):
+  # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries
+  switch("passL", "-Wl,--no-insert-timestamp")
+  # increase stack size
+  switch("passL", "-Wl,--stack,8388608")
+  # https://github.com/nim-lang/Nim/issues/4057
+  --tlsEmulation:off
+  if defined(i386):
+    # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
+    switch("passL", "-Wl,--large-address-aware")
+
+  # The dynamic Chronicles output currently prevents us from using colors on Windows
+  # because these require direct manipulations of the stdout File object.
+  switch("define", "chronicles_colors=off")
+
+# This helps especially for 32-bit x86, which sans SSE2 and newer instructions
+# requires quite roundabout code generation for cryptography, and other 64-bit
+# and larger arithmetic use cases, along with register starvation issues. When
+# engineering a more portable binary release, this should be tweaked but still
+# use at least -msse2 or -msse3.
+if defined(disableMarchNative):
+  switch("passC", "-msse3")
+else:
+  switch("passC", "-march=native")
+  if defined(windows):
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
+    # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
+    switch("passC", "-mno-avx512vl")
+
+--threads:on
+--opt:speed
+--excessiveStackTrace:on
+# enable metric collection
+--define:metrics
+# for heap-usage-by-instance-type metrics and object base-type strings
+--define:nimTypeNames
+
+switch("define", "withoutPCRE")
+
+# the default open files limit is too low on macOS (512), breaking the
+# "--debugger:native" build. It can be increased with `ulimit -n 1024`.
+if not defined(macosx):
+  # add debugging symbols and original files and line numbers
+  --debugger:native
+  if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace):
+    # light-weight stack traces using libbacktrace and libunwind
+    --define:nimStackTraceOverride
+    switch("import", "libbacktrace")
+
+--define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9
+
+# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'"
+switch("warning", "CaseTransition:off")
+
+# The compiler doth protest too much, methinks, about all these cases where it can't
+# do its (N)RVO pass: https://github.com/nim-lang/RFCs/issues/230
+switch("warning", "ObservableStores:off")
+
+# Too many false positives for "Warning: method has lock level <unknown>, but another method has 0 [LockLevel]"
+switch("warning", "LockLevel:off")
+

dagger.nimble

@@ -1,3 +1,5 @@
+mode = ScriptMode.Verbose
+
 version = "0.1.0"
 author = "Dagger Team"
 description = "p2p data durability engine"
@@ -16,3 +18,20 @@ requires "libp2p#unstable",
          "questionable >= 0.9.1 & < 0.10.0",
          "upraises >= 0.1.0 & < 0.2.0",
          "asynctest >= 0.3.0 & < 0.4.0"
+
+### Helper functions
+proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
+  if not dirExists "build":
+    mkDir "build"
+  # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
+  var extra_params = params
+  for i in 2..<paramCount():
+    extra_params &= " " & paramStr(i)
+  exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
+
+proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") =
+  buildBinary name, "tests/", params
+  exec "build/" & name
+
+task testAll, "Build & run Waku v1 tests":
+  test "testAll", "-d:chronicles_log_level=WARN"