Merge branch 'libbacktrace' into devel

.appveyor.yml

@@ -36,8 +36,8 @@ build_script:
 
 test_script:
   # the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug
-  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache
+  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE
-  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
+  - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
   - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 DISABLE_GO_CHECKS=1 test
 
 deploy: off

.gitignore

@@ -22,7 +22,7 @@ build/
 *.dll
 
 # a symlink that can't be added to the repo because of Windows
-/*.nims
+/beacon_chain.nims
 
 /scripts/testnet*.sh
 

.gitmodules

@@ -134,3 +134,8 @@
 [submodule "vendor/lmdb"]
 	path = vendor/lmdb
 	url = https://github.com/status-im/lmdb.git
+[submodule "vendor/nim-libbacktrace"]
+	path = vendor/nim-libbacktrace
+	url = https://github.com/status-im/nim-libbacktrace.git
+	ignore = dirty
+	branch = master

.travis.yml

@@ -49,7 +49,7 @@ script:
   - set -e # fail fast
   # Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected
-  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
+  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" LOG_LEVEL=TRACE
-  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
+  - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:NETWORK_TYPE=libp2p -d:testnet_servers_image" LOG_LEVEL=TRACE
   - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_TEST_FIXTURES_SCRIPT=1 test
 

Jenkinsfile

@@ -21,8 +21,8 @@ def runStages() {
 			parallel(
 				"tools": {
 					stage("Tools") {
-						sh "make -j${env.NPROC}"
+						sh "make -j${env.NPROC} LOG_LEVEL=TRACE"
-						sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
+						sh "make -j${env.NPROC} LOG_LEVEL=TRACE NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
 					}
 				},
 				"test suite": {

Makefile

@@ -45,7 +45,8 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	testnet0 \
 	clean-testnet1 \
 	testnet1 \
-	clean
+	clean \
+	libbacktrace
 
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included. We can only execute one target in this state.
@@ -57,6 +58,10 @@ endif
 # must be included after the default target
 -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
 
+# "--import" can't be added to config.nims, for some reason
+# "--define:release" implies "--stacktrace:off" and it cannot be added to config.nims either
+NIM_PARAMS := $(NIM_PARAMS) -d:release --import:libbacktrace
+
 #- the Windows build fails on Azure Pipelines if we have Unicode symbols copy/pasted here,
 #  so we encode them in ASCII
 GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
@@ -70,7 +75,7 @@ build-system-checks:
 		}; \
 		exit 0
 
-deps: | deps-common beacon_chain.nims p2pd
+deps: | deps-common beacon_chain.nims p2pd libbacktrace
 
 #- deletes and recreates "beacon_chain.nims" which on Windows is a copy instead of a proper symlink
 update: | update-common
@@ -81,6 +86,10 @@ update: | update-common
 beacon_chain.nims:
 	ln -s beacon_chain.nimble $@
 
+# nim-libbacktrace
+libbacktrace:
+	+ $(MAKE) -C vendor/nim-libbacktrace
+
 P2PD_CACHE :=
 p2pd: | go-checks
 	BUILD_MSG="$(BUILD_MSG) $@" \
@@ -121,6 +130,7 @@ testnet1: | build deps
 
 clean: | clean-common
 	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}
+	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 
 libnfuzz.so: | build deps-common beacon_chain.nims
 	echo -e $(BUILD_MSG) "build/$@" && \

azure-pipelines.yml

@@ -71,8 +71,8 @@ jobs:
           scripts/setup_official_tests.sh jsonTestsCache
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE
-          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
+          mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
           file build/beacon_node
           mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
         displayName: 'build and test'

beacon_chain.nimble

@@ -24,6 +24,7 @@ requires "nim >= 0.19.0",
   "eth",
   "json_rpc",
   "json_serialization",
+  "libbacktrace",
   "libp2p",
   "metrics",
   "nimcrypto",
@@ -41,26 +42,30 @@ proc buildBinary(name: string, srcDir = "./", params = "", cmdParams = "", lang
   var extra_params = params
   for i in 2..<paramCount():
     extra_params &= " " & paramStr(i)
-  exec "nim " & lang & " --out:./build/" & name & " " & extra_params & " " & srcDir & name & ".nim" & " " & cmdParams
+  exec "nim " & lang & " --out:./build/" & name & " -r -d:release --import:libbacktrace " & extra_params & " " & srcDir & name & ".nim" & " " & cmdParams
 
 ### tasks
 task test, "Run all tests":
-  # Mainnet config
+  # We're enabling the TRACE log level so we're sure that those rarely used
-  buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR -d:const_preset=mainnet"
+  # pieces of code get tested regularly. Increased test output verbosity is the
+  # price we pay for that.
+
   # Minimal config
-  buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR -d:const_preset=minimal"
+  buildBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
+  # Mainnet config
+  buildBinary "all_tests", "tests/", "-d:const_preset=mainnet"
 
   # Generic SSZ test, doesn't use consensus objects minimal/mainnet presets
-  buildBinary "test_fixture_ssz_generic_types", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG"
+  buildBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE"
 
   # Consensus object SSZ tests
-  buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-r -d:release -d:chronicles_log_level=TRACE -d:const_preset=minimal"
+  buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
-  buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG -d:const_preset=mainnet"
+  buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:const_preset=mainnet"
 
-  buildBinary "all_fixtures_require_ssz", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG -d:const_preset=minimal"
+  buildBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
-  buildBinary "all_fixtures_require_ssz", "tests/official/", "-r -d:release -d:chronicles_log_level=TRACE -d:const_preset=mainnet"
+  buildBinary "all_fixtures_require_ssz", "tests/official/", "-d:const_preset=mainnet"
 
   # State sim; getting into 4th epoch useful to trigger consensus checks
-  buildBinary "state_sim", "research/", "-r -d:release", "--validators=1024 --slots=32"
+  buildBinary "state_sim", "research/", "", "--validators=1024 --slots=32"
-  buildBinary "state_sim", "research/", "-r -d:release -d:const_preset=mainnet", "--validators=1024 --slots=128"
+  buildBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=1024 --slots=128"
 

config.nims

@@ -1,54 +1,56 @@
-@if release:
+if defined(release):
-  nimcache = "nimcache/release/$projectName"
+  switch("nimcache", "nimcache/release/$projectName")
-@else:
+else:
-  nimcache = "nimcache/debug/$projectName"
+  switch("nimcache", "nimcache/debug/$projectName")
-@end
 
-@if windows:
+if defined(windows):
   # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries
-  --passL:"-Wl,--no-insert-timestamp"
+  switch("passL", "-Wl,--no-insert-timestamp")
   # increase stack size
-  --passL:"-Wl,--stack,8388608"
+  switch("passL", "-Wl,--stack,8388608")
   # https://github.com/nim-lang/Nim/issues/4057
   --tlsEmulation:off
-  @if i386:
+  if defined(i386):
     # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
-    --passL:"-Wl,--large-address-aware"
+    switch("passL", "-Wl,--large-address-aware")
-  @end
 
   # The dynamic Chronicles output currently prevents us from using colors on Windows
   # because these require direct manipulations of the stdout File object.
-  -d:"chronicles_colors=off"
+  switch("define", "chronicles_colors=off")
-@end
 
 # This helps especially for 32-bit x86, which sans SSE2 and newer instructions
 # requires quite roundabout code generation for cryptography, and other 64-bit
 # and larger arithmetic use cases, along with register starvation issues. When
 # engineering a more portable binary release, this should be tweaked but still
 # use at least -msse2 or -msse3.
-@if disableMarchNative:
+if defined(disableMarchNative):
-  --passC:"-msse3"
+  switch("passC", "-msse3")
-@else:
+else:
-  --passC:"-march=native"
+  switch("passC", "-march=native")
-  @if windows:
+  if defined(windows):
     # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
     # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
-    --passC:"-mno-avx512vl"
+    switch("passC", "-mno-avx512vl")
-  @end
-@end
 
 --threads:on
 --opt:speed
 --excessiveStackTrace:on
 # enable metric collection
--d:metrics
+--define:metrics
 
 # the default open files limit is too low on macOS (512), breaking the
 # "--debugger:native" build. It can be increased with `ulimit -n 1024`.
-@if not macosx:
+if not defined(macosx):
   # add debugging symbols and original files and line numbers
   --debugger:native
-@end
+  if not (defined(windows) and defined(i386)):
+    # light-weight stack traces using libbacktrace and libunwind
+    --define:nimStackTraceOverride
+    # "--import:libbacktrace" is added to NIM_PARAMS inside the Makefile,
+    # because it doesn't work in here ("Error: undeclared identifier: 'copyMem'", like it kicks in in some other NimScript file)
+
+--define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9
+
+# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'"
+switch("warning", "CaseTransition:off")
 
--d:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9
---warning[CaseTransition]:off

docker/Dockerfile

@@ -27,7 +27,7 @@ RUN cd /root/nim-beacon-chain \
  && git fetch \
  && git reset --hard ${GIT_REVISION} \
  && make -j$(nproc) update \
- && make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
+ && make LOG_LEVEL=TRACE NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
 
 # --------------------------------- #
 # Starting new image to reduce size #

scripts/launch_local_testnet.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Mostly a duplication of "tests/simulation/{start.sh,run_node.sh}", but with a focus on
-# replicating testnets as close as possible, which means following the Docker execution labyrinth.
+# replicating testnets as closely as possible, which means following the Docker execution labyrinth.
 
 set -e
 
@@ -22,7 +22,7 @@ source "scripts/${NETWORK}.env"
 set +a
 
 NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
-make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
+make LOG_LEVEL=DEBUG NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
 
 rm -rf "${DEPOSITS_DIR}"
 ./build/beacon_node makeDeposits \

vendor/nim-json-rpc

@@ -1 +1 @@
-Subproject commit fc0665f88a4f24e9f51fe059ad4e943f3eed927e
+Subproject commit 09f6fd63556a22312af7c5a7e3fa105b26897316

vendor/nim-libbacktrace

@@ -0,0 +1 @@
+Subproject commit 61f29dcfb4ec395bf1c0752e3ee61653c5b3aad3