From 06e0eb80e6a1b9e303b2abab9b36f6fafd70b88a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Sun, 9 Feb 2020 17:31:37 +0100 Subject: [PATCH] lightweight stack traces Also converted the top-level nim.cfg into a config.nims, in the process. --- .appveyor.yml | 6 ++--- .gitmodules | 5 ++++ .travis.yml | 2 +- Makefile | 38 +++++++++++++++++++++++------ config.nims | 37 ++++++++++++++++++++++++++++ nim.cfg | 34 -------------------------- nimbus.nimble | 2 +- vendor/nim-libbacktrace | 1 + wrappers/wrapper_example.go | 4 +-- wrappers/wrapper_whisper_example.go | 19 ++++++++------- 10 files changed, 90 insertions(+), 58 deletions(-) create mode 100644 config.nims delete mode 100644 nim.cfg create mode 160000 vendor/nim-libbacktrace diff --git a/.appveyor.yml b/.appveyor.yml index 8bcb0a424..0372b7684 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -42,9 +42,9 @@ test_script: - build\nimbus.exe --help - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% test - IF "%PLATFORM%" == "x64" mingw32-make -j2 test-reproducibility - # disable libnimbus builds until https://github.com/nim-lang/Nim/issues/12759 is fixed - # wrapper builds could then also get activated instead but need some rework for Windows - # the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug + # Disable libnimbus builds until https://github.com/nim-lang/Nim/issues/12759 is fixed. + # Wrapper builds could then also get activated instead but need some rework for Windows. + # The "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug. # - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 libnimbus.so # - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 libnimbus.a - mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% wakusim diff --git a/.gitmodules b/.gitmodules index 967cdab0a..63dafad21 100644 --- a/.gitmodules +++ b/.gitmodules @@ -158,3 +158,8 @@ url = https://github.com/status-im/nim-evmc ignore = dirty branch = master +[submodule "vendor/nim-libbacktrace"] + path = vendor/nim-libbacktrace + url = https://github.com/status-im/nim-libbacktrace.git + ignore = dirty + branch = master diff --git a/.travis.yml b/.travis.yml index 660dfc5a8..c1b40384a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ script: - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" - build/nimbus --help - # -static option will not work for osx unless static system libraries are provided + # "-static" option will not work for osx unless static system libraries are provided - if [ "$TRAVIS_OS_NAME" = "osx" ]; then make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" test test-reproducibility wrappers wakusim; else diff --git a/Makefile b/Makefile index 5fbf81ee9..fc37691f4 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,11 @@ TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTe TOOLS_DIRS := premix tests waku # comma-separated values for the "clean" target TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) +# "--import" can't be added to config.nims, for some reason +# "--define:release" implies "--stacktrace:off" and it cannot be added to config.nims either +NIM_PARAMS := $(NIM_PARAMS) -d:release --import:libbacktrace -.PHONY: all $(TOOLS) build-system-checks deps update nimbus test test-reproducibility clean libnimbus.so libnimbus.a wrappers wrappers-static +.PHONY: all $(TOOLS) build-system-checks deps update nimbus test test-reproducibility clean libnimbus.so libnimbus.a wrappers wrappers-static libbacktrace # default target, because it's the first one that doesn't start with '.' all: build-system-checks $(TOOLS) nimbus @@ -36,7 +39,7 @@ build-system-checks: exit 1; \ } -deps: | deps-common nimbus.nims +deps: | deps-common nimbus.nims libbacktrace #- deletes and recreates "nimbus.nims" which on Windows is a copy instead of a proper symlink update: | update-common @@ -58,6 +61,10 @@ nimbus: | build deps nimbus.nims: ln -s nimbus.nimble $@ +# nim-libbacktrace +libbacktrace: + + $(MAKE) -C vendor/nim-libbacktrace + # builds and runs the test suite test: | build deps $(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims @@ -75,6 +82,7 @@ test-reproducibility: # usual cleaning clean: | clean-common rm -rf build/{nimbus,$(TOOLS_CSV),all_tests,test_rpc,*_wrapper_test} + + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) libnimbus.so: | build deps echo -e $(BUILD_MSG) "build/$@" && \ @@ -82,13 +90,15 @@ libnimbus.so: | build deps rm -f build/$@ && \ ln -s $@.0 build/$@ +# libraries for dynamic linking of non-Nim objects +EXTRA_LIBS_DYNAMIC := -L"$(CURDIR)/build" -lnimbus -lm wrappers: | build deps libnimbus.so go-checks echo -e $(BUILD_MSG) "build/C_wrapper_example" && \ - $(CC) wrappers/wrapper_example.c -Wl,-rpath,'$$ORIGIN' -Lbuild -lnimbus -lm -g -o build/C_wrapper_example + $(CC) wrappers/wrapper_example.c -Wl,-rpath,'$$ORIGIN' $(EXTRA_LIBS_DYNAMIC) -g -o build/C_wrapper_example echo -e $(BUILD_MSG) "build/go_wrapper_example" && \ - go build -o build/go_wrapper_example wrappers/wrapper_example.go wrappers/cfuncs.go + go build -ldflags "-linkmode external -extldflags '$(EXTRA_LIBS_DYNAMIC)'" -o build/go_wrapper_example wrappers/wrapper_example.go wrappers/cfuncs.go echo -e $(BUILD_MSG) "build/go_wrapper_whisper_example" && \ - go build -o build/go_wrapper_whisper_example wrappers/wrapper_whisper_example.go wrappers/cfuncs.go + go build -ldflags "-linkmode external -extldflags '$(EXTRA_LIBS_DYNAMIC)'" -o build/go_wrapper_whisper_example wrappers/wrapper_whisper_example.go wrappers/cfuncs.go libnimbus.a: | build deps echo -e $(BUILD_MSG) "build/$@" && \ @@ -96,13 +106,25 @@ libnimbus.a: | build deps $(ENV_SCRIPT) nim c --app:staticlib --noMain --nimcache:nimcache/libnimbus_static $(NIM_PARAMS) -o:build/$@ wrappers/libnimbus.nim && \ [[ -e "$@" ]] && mv "$@" build/ # workaround for https://github.com/nim-lang/Nim/issues/12745 +# These libraries are for statically linking non-Nim objects to libnimbus.a +# (where "vendor/nim-libbacktrace/libbacktrace.nim" doesn't get to set its LDFLAGS) +EXTRA_LIBS_STATIC := -L"$(CURDIR)/build" -lnimbus -L"$(CURDIR)/vendor/nim-libbacktrace/install/usr/lib" -lbacktracenim -lbacktrace -lm -ldl -lpcre +ifeq ($(shell uname), Darwin) +USE_VENDORED_LIBUNWIND := 1 +endif # macOS +ifeq ($(OS), Windows_NT) +USE_VENDORED_LIBUNWIND := 1 +endif # Windows +ifeq ($(USE_VENDORED_LIBUNWIND), 1) +EXTRA_LIBS_STATIC := $(EXTRA_LIBS_STATIC) -lunwind +endif # USE_VENDORED_LIBUNWIND wrappers-static: | build deps libnimbus.a go-checks echo -e $(BUILD_MSG) "build/C_wrapper_example_static" && \ - $(CC) wrappers/wrapper_example.c -static -pthread -Lbuild -lnimbus -lm -ldl -lpcre -g -o build/C_wrapper_example_static + $(CC) wrappers/wrapper_example.c -static -pthread $(EXTRA_LIBS_STATIC) -g -o build/C_wrapper_example_static echo -e $(BUILD_MSG) "build/go_wrapper_example_static" && \ - go build -ldflags "-linkmode external -extldflags '-static -ldl -lpcre'" -o build/go_wrapper_example_static wrappers/wrapper_example.go wrappers/cfuncs.go + go build -ldflags "-linkmode external -extldflags '-static $(EXTRA_LIBS_STATIC)'" -o build/go_wrapper_example_static wrappers/wrapper_example.go wrappers/cfuncs.go echo -e $(BUILD_MSG) "build/go_wrapper_whisper_example_static" && \ - go build -ldflags "-linkmode external -extldflags '-static -ldl -lpcre'" -o build/go_wrapper_whisper_example_static wrappers/wrapper_example.go wrappers/cfuncs.go + go build -ldflags "-linkmode external -extldflags '-static $(EXTRA_LIBS_STATIC)'" -o build/go_wrapper_whisper_example_static wrappers/wrapper_whisper_example.go wrappers/cfuncs.go wakunode: | build deps echo -e $(BUILD_MSG) "build/$@" && \ diff --git a/config.nims b/config.nims new file mode 100644 index 000000000..62617f21d --- /dev/null +++ b/config.nims @@ -0,0 +1,37 @@ +if defined(release): + switch("nimcache", "nimcache/release/$projectName") +else: + switch("nimcache", "nimcache/debug/$projectName") + +if defined(windows): + # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries + switch("passL", "-Wl,--no-insert-timestamp") + # increase stack size + switch("passL", "-Wl,--stack,8388608") + # https://github.com/nim-lang/Nim/issues/4057 + --tlsEmulation:off + if defined(i386): + # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM + switch("passL", "-Wl,--large-address-aware") + +--threads:on +--opt:speed +--excessiveStackTrace:on +# enable metric collection +--define:metrics + +# the default open files limit is too low on macOS (512), breaking the +# "--debugger:native" build. It can be increased with `ulimit -n 1024`. +if not defined(macosx): + # add debugging symbols and original files and line numbers + --debugger:native + if not (defined(windows) and defined(i386)): + # light-weight stack traces using libbacktrace and libunwind + --define:nimStackTraceOverride + # "--import:libbacktrace" is added to NIM_PARAMS inside the Makefile, + # because it doesn't work in here ("Error: undeclared identifier: 'copyMem'", like it kicks in in some other NimScript file) + +--define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9 +# libnimbus.so needs position-independent code +switch("passC", "-fPIC") + diff --git a/nim.cfg b/nim.cfg deleted file mode 100644 index a4082ac65..000000000 --- a/nim.cfg +++ /dev/null @@ -1,34 +0,0 @@ -@if release: - nimcache = "nimcache/release/$projectName" -@else: - nimcache = "nimcache/debug/$projectName" -@end - -@if windows: - # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries - --passL:"-Wl,--no-insert-timestamp" - # increase stack size - --passL:"-Wl,--stack,8388608" - # https://github.com/nim-lang/Nim/issues/4057 - --tlsEmulation:off - @if i386: - # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM - --passL:"-Wl,--large-address-aware" - @end -@end - ---threads:on ---opt:speed ---excessiveStackTrace:on -# enable metric collection --d:metrics - -# the default open files limit is too low on macOS (512), breaking the -# "--debugger:native" build. It can be increased with `ulimit -n 1024`. -@if not macosx: - # add debugging symbols and original files and line numbers - --debugger:native -@end - --d:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9 - diff --git a/nimbus.nimble b/nimbus.nimble index 868f1d0bd..2e090d9e2 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -29,7 +29,7 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim" proc test(name: string, lang = "c") = - buildBinary name, "tests/", "-d:release -d:chronicles_log_level=ERROR" + buildBinary name, "tests/", "-d:chronicles_log_level=ERROR" exec "build/" & name task test, "Run tests": diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace new file mode 160000 index 000000000..465cb67f2 --- /dev/null +++ b/vendor/nim-libbacktrace @@ -0,0 +1 @@ +Subproject commit 465cb67f234e66064f146e79063a1bd82db11d3c diff --git a/wrappers/wrapper_example.go b/wrappers/wrapper_example.go index 65d22d886..a7b680009 100644 --- a/wrappers/wrapper_example.go +++ b/wrappers/wrapper_example.go @@ -7,11 +7,11 @@ import ( "unsafe" ) - /* #include -#cgo LDFLAGS: -Wl,-rpath,'$ORIGIN' -L${SRCDIR}/../build -lnimbus -lm +// Passing "-lnimbus" to the Go linker through "-extldflags" is not enough. We need it in here, for some reason. +#cgo LDFLAGS: -Wl,-rpath,'$ORIGIN' -L${SRCDIR}/../build -lnimbus #include "libnimbus.h" void receiveHandler_cgo(received_message * msg); // Forward declaration. diff --git a/wrappers/wrapper_whisper_example.go b/wrappers/wrapper_whisper_example.go index 013da9319..333cbf4ed 100644 --- a/wrappers/wrapper_whisper_example.go +++ b/wrappers/wrapper_whisper_example.go @@ -1,18 +1,19 @@ package main import ( + "encoding/hex" "fmt" "runtime" "time" "unsafe" - "encoding/hex" ) /* #include #include -#cgo LDFLAGS: -Wl,-rpath,'$ORIGIN' -L${SRCDIR}/../build -lnimbus -lm +// Passing "-lnimbus" to the Go linker through "-extldflags" is not enough. We need it in here, for some reason. +#cgo LDFLAGS: -Wl,-rpath,'$ORIGIN' -L${SRCDIR}/../build -lnimbus #include "libnimbus.h" void receiveHandler_cgo(received_message * msg, void* udata); // Forward declaration. @@ -88,12 +89,12 @@ func StatusListenAndPost(channel string) { options := C.filter_options{symKeyID: symKeyIdC, minPow: 0.002, - topic: C.nimbus_channel_to_topic(channelC).topic} + topic: C.nimbus_channel_to_topic(channelC).topic} tmp = C.malloc(C.size_t(C.ID_LEN)) if C.nimbus_subscribe_filter(&options, - (C.received_msg_handler)(unsafe.Pointer(C.receiveHandler_cgo)), - unsafe.Pointer(&msgCount), (*C.uint8_t)(tmp)) == false { + (C.received_msg_handler)(unsafe.Pointer(C.receiveHandler_cgo)), + unsafe.Pointer(&msgCount), (*C.uint8_t)(tmp)) == false { C.free(unsafe.Pointer(tmp)) panic("Cannot subscribe filter") } @@ -103,11 +104,11 @@ func StatusListenAndPost(channel string) { hex.EncodeToString(filterId)) postMessage := C.post_message{symKeyID: symKeyIdC, - sourceID: asymKeyIdC, - ttl: 20, - topic: C.nimbus_channel_to_topic(channelC).topic, + sourceID: asymKeyIdC, + ttl: 20, + topic: C.nimbus_channel_to_topic(channelC).topic, powTarget: 0.002, - powTime: 1.0} + powTime: 1.0} i := 0 for {