lightweight stack traces

- plus some light test runner refactoring and some Makefile cosmetic changes
- compile tools with LOG_LEVEL=TRACE in CI
- bump a couple of submodules
This commit is contained in:
Ștefan Talpalaru 2020-02-13 13:19:12 +01:00
parent 812386edd2
commit 06322385d9
No known key found for this signature in database
GPG Key ID: CBF7934204F1B6F9
13 changed files with 75 additions and 52 deletions

View File

@ -36,8 +36,8 @@ build_script:
test_script:
# the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 DISABLE_GO_CHECKS=1 test
deploy: off

2
.gitignore vendored
View File

@ -22,7 +22,7 @@ build/
*.dll
# a symlink that can't be added to the repo because of Windows
/*.nims
/beacon_chain.nims
/scripts/testnet*.sh

5
.gitmodules vendored
View File

@ -134,3 +134,8 @@
[submodule "vendor/lmdb"]
path = vendor/lmdb
url = https://github.com/status-im/lmdb.git
[submodule "vendor/nim-libbacktrace"]
path = vendor/nim-libbacktrace
url = https://github.com/status-im/nim-libbacktrace.git
ignore = dirty
branch = master

View File

@ -49,7 +49,7 @@ script:
- set -e # fail fast
# Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" LOG_LEVEL=TRACE
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:NETWORK_TYPE=libp2p -d:testnet_servers_image" LOG_LEVEL=TRACE
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_TEST_FIXTURES_SCRIPT=1 test

4
Jenkinsfile vendored
View File

@ -21,8 +21,8 @@ def runStages() {
parallel(
"tools": {
stage("Tools") {
sh "make -j${env.NPROC}"
sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
sh "make -j${env.NPROC} LOG_LEVEL=TRACE"
sh "make -j${env.NPROC} LOG_LEVEL=TRACE NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
}
},
"test suite": {

View File

@ -45,7 +45,8 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
testnet0 \
clean-testnet1 \
testnet1 \
clean
clean \
libbacktrace
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included. We can only execute one target in this state.
@ -57,6 +58,10 @@ endif
# must be included after the default target
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
# "--import" can't be added to config.nims, for some reason
# "--define:release" implies "--stacktrace:off" and it cannot be added to config.nims either
NIM_PARAMS := $(NIM_PARAMS) -d:release --import:libbacktrace
#- the Windows build fails on Azure Pipelines if we have Unicode symbols copy/pasted here,
# so we encode them in ASCII
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
@ -70,7 +75,7 @@ build-system-checks:
}; \
exit 0
deps: | deps-common beacon_chain.nims p2pd
deps: | deps-common beacon_chain.nims p2pd libbacktrace
#- deletes and recreates "beacon_chain.nims" which on Windows is a copy instead of a proper symlink
update: | update-common
@ -81,6 +86,10 @@ update: | update-common
beacon_chain.nims:
ln -s beacon_chain.nimble $@
# nim-libbacktrace
libbacktrace:
+ $(MAKE) -C vendor/nim-libbacktrace
P2PD_CACHE :=
p2pd: | go-checks
BUILD_MSG="$(BUILD_MSG) $@" \
@ -121,6 +130,7 @@ testnet1: | build deps
clean: | clean-common
rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
libnfuzz.so: | build deps-common beacon_chain.nims
echo -e $(BUILD_MSG) "build/$@" && \

View File

@ -71,8 +71,8 @@ jobs:
scripts/setup_official_tests.sh jsonTestsCache
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache LOG_LEVEL=TRACE NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
file build/beacon_node
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
displayName: 'build and test'

View File

@ -24,6 +24,7 @@ requires "nim >= 0.19.0",
"eth",
"json_rpc",
"json_serialization",
"libbacktrace",
"libp2p",
"metrics",
"nimcrypto",
@ -41,26 +42,30 @@ proc buildBinary(name: string, srcDir = "./", params = "", cmdParams = "", lang
var extra_params = params
for i in 2..<paramCount():
extra_params &= " " & paramStr(i)
exec "nim " & lang & " --out:./build/" & name & " " & extra_params & " " & srcDir & name & ".nim" & " " & cmdParams
exec "nim " & lang & " --out:./build/" & name & " -r -d:release --import:libbacktrace " & extra_params & " " & srcDir & name & ".nim" & " " & cmdParams
### tasks
task test, "Run all tests":
# Mainnet config
buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR -d:const_preset=mainnet"
# We're enabling the TRACE log level so we're sure that those rarely used
# pieces of code get tested regularly. Increased test output verbosity is the
# price we pay for that.
# Minimal config
buildBinary "all_tests", "tests/", "-r -d:release -d:chronicles_log_level=ERROR -d:const_preset=minimal"
buildBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
# Mainnet config
buildBinary "all_tests", "tests/", "-d:const_preset=mainnet"
# Generic SSZ test, doesn't use consensus objects minimal/mainnet presets
buildBinary "test_fixture_ssz_generic_types", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG"
buildBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE"
# Consensus object SSZ tests
buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-r -d:release -d:chronicles_log_level=TRACE -d:const_preset=minimal"
buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG -d:const_preset=mainnet"
buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
buildBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:const_preset=mainnet"
buildBinary "all_fixtures_require_ssz", "tests/official/", "-r -d:release -d:chronicles_log_level=DEBUG -d:const_preset=minimal"
buildBinary "all_fixtures_require_ssz", "tests/official/", "-r -d:release -d:chronicles_log_level=TRACE -d:const_preset=mainnet"
buildBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=minimal"
buildBinary "all_fixtures_require_ssz", "tests/official/", "-d:const_preset=mainnet"
# State sim; getting into 4th epoch useful to trigger consensus checks
buildBinary "state_sim", "research/", "-r -d:release", "--validators=1024 --slots=32"
buildBinary "state_sim", "research/", "-r -d:release -d:const_preset=mainnet", "--validators=1024 --slots=128"
buildBinary "state_sim", "research/", "", "--validators=1024 --slots=32"
buildBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=1024 --slots=128"

View File

@ -1,54 +1,56 @@
@if release:
nimcache = "nimcache/release/$projectName"
@else:
nimcache = "nimcache/debug/$projectName"
@end
if defined(release):
switch("nimcache", "nimcache/release/$projectName")
else:
switch("nimcache", "nimcache/debug/$projectName")
@if windows:
if defined(windows):
# disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries
--passL:"-Wl,--no-insert-timestamp"
switch("passL", "-Wl,--no-insert-timestamp")
# increase stack size
--passL:"-Wl,--stack,8388608"
switch("passL", "-Wl,--stack,8388608")
# https://github.com/nim-lang/Nim/issues/4057
--tlsEmulation:off
@if i386:
if defined(i386):
# set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM
--passL:"-Wl,--large-address-aware"
@end
switch("passL", "-Wl,--large-address-aware")
# The dynamic Chronicles output currently prevents us from using colors on Windows
# because these require direct manipulations of the stdout File object.
-d:"chronicles_colors=off"
@end
switch("define", "chronicles_colors=off")
# This helps especially for 32-bit x86, which sans SSE2 and newer instructions
# requires quite roundabout code generation for cryptography, and other 64-bit
# and larger arithmetic use cases, along with register starvation issues. When
# engineering a more portable binary release, this should be tweaked but still
# use at least -msse2 or -msse3.
@if disableMarchNative:
--passC:"-msse3"
@else:
--passC:"-march=native"
@if windows:
if defined(disableMarchNative):
switch("passC", "-msse3")
else:
switch("passC", "-march=native")
if defined(windows):
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
# ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
--passC:"-mno-avx512vl"
@end
@end
switch("passC", "-mno-avx512vl")
--threads:on
--opt:speed
--excessiveStackTrace:on
# enable metric collection
-d:metrics
--define:metrics
# the default open files limit is too low on macOS (512), breaking the
# "--debugger:native" build. It can be increased with `ulimit -n 1024`.
@if not macosx:
if not defined(macosx):
# add debugging symbols and original files and line numbers
--debugger:native
@end
if not (defined(windows) and defined(i386)):
# light-weight stack traces using libbacktrace and libunwind
--define:nimStackTraceOverride
# "--import:libbacktrace" is added to NIM_PARAMS inside the Makefile,
# because it doesn't work in here ("Error: undeclared identifier: 'copyMem'", like it kicks in in some other NimScript file)
--define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9
# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'"
switch("warning", "CaseTransition:off")
-d:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9
--warning[CaseTransition]:off

View File

@ -27,7 +27,7 @@ RUN cd /root/nim-beacon-chain \
&& git fetch \
&& git reset --hard ${GIT_REVISION} \
&& make -j$(nproc) update \
&& make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
&& make LOG_LEVEL=TRACE NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
# --------------------------------- #
# Starting new image to reduce size #

View File

@ -1,7 +1,7 @@
#!/bin/bash
# Mostly a duplication of "tests/simulation/{start.sh,run_node.sh}", but with a focus on
# replicating testnets as close as possible, which means following the Docker execution labyrinth.
# replicating testnets as closely as possible, which means following the Docker execution labyrinth.
set -e
@ -22,7 +22,7 @@ source "scripts/${NETWORK}.env"
set +a
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
make LOG_LEVEL=DEBUG NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
rm -rf "${DEPOSITS_DIR}"
./build/beacon_node makeDeposits \

2
vendor/nim-json-rpc vendored

@ -1 +1 @@
Subproject commit fc0665f88a4f24e9f51fe059ad4e943f3eed927e
Subproject commit 09f6fd63556a22312af7c5a7e3fa105b26897316

1
vendor/nim-libbacktrace vendored Submodule

@ -0,0 +1 @@
Subproject commit 61f29dcfb4ec395bf1c0752e3ee61653c5b3aad3