reduce peak memory usage during build

.github/workflows/ci.yml

@@ -273,8 +273,7 @@ jobs:
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node
+          make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_beacon_node nimbus_validator_client
-          make -j$ncpu ARCH_OVERRIDE=$PLATFORM LOG_LEVEL=TRACE NIMFLAGS="-d:testnet_servers_image" nimbus_validator_client
 
       - name: Run nim-beacon-chain tests
         if: matrix.target.TEST_KIND == 'unit-tests'
@@ -288,11 +287,11 @@ jobs:
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port $(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port $(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port $(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
+          ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000  --base-metrics-port 8008 -- --verify-finalization --discv5:no
 
       - name: Run nim-beacon-chain testnet1 (mainnet)
         if: matrix.target.TEST_KIND == 'finalization-mainnet'
         shell: bash
         working-directory: nim-beacon-chain
         run: |
-          ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port $(( 9000 + EXECUTOR_NUMBER * 100 )) --base-rpc-port $(( 7000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port $(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --discv5:no
+          ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --stop-at-epoch 5 --log-level DEBUG --disable-htop --enable-logtrace --data-dir local_testnet0_data --base-port 9000 --base-rpc-port 7000 --base-metrics-port 8008 -- --verify-finalization --discv5:no

Makefile

@@ -113,23 +113,26 @@ endif
 
 DEPOSITS_DELAY := 0
 
-# "--define:release" cannot be added to config.nims
+#- "--define:release" cannot be added to "config.nims"
+#- disable Nim's default parallelisation because it starts too many processes for too little gain
+NIM_PARAMS := $(NIM_PARAMS) -d:release --parallelBuild:1
+
 ifeq ($(USE_LIBBACKTRACE), 0)
 # Blame Jacek for the lack of line numbers in your stack traces ;-)
-NIM_PARAMS := $(NIM_PARAMS) -d:release --stacktrace:on --excessiveStackTrace:on --linetrace:off -d:disable_libbacktrace
+NIM_PARAMS := $(NIM_PARAMS) --stacktrace:on --excessiveStackTrace:on --linetrace:off -d:disable_libbacktrace
-else
-NIM_PARAMS := $(NIM_PARAMS) -d:release
 endif
 
-deps: | deps-common nat-libs beacon_chain.nims
+deps: | deps-common nat-libs beacon_chain.nims build/generate_makefile
 ifneq ($(USE_LIBBACKTRACE), 0)
 deps: | libbacktrace
 endif
 
 #- deletes and recreates "beacon_chain.nims" which on Windows is a copy instead of a proper symlink
+#- deletes binaries that might need to be rebuilt after a Git pull
 update: | update-common
 	rm -f beacon_chain.nims && \
 		"$(MAKE)" beacon_chain.nims $(HANDLE_OUTPUT)
+	rm -f build/generate_makefile
 
 # symlink
 beacon_chain.nims:
@@ -148,14 +151,26 @@ ifeq ($(DISABLE_TEST_FIXTURES_SCRIPT), 0)
 endif
 	+ $(ENV_SCRIPT) nim test $(NIM_PARAMS) beacon_chain.nims && rm -f 0000-*.json
 
-#- GCC's LTO parallelisation is able to detect a GNU Make jobserver and get its
+# It's OK to only build this once. `make update` deletes the binary, forcing a rebuild.
-#  maximum number of processes from there, but only if we use the "+" prefix.
+ifneq ($(USE_LIBBACKTRACE), 0)
-#  Without it, it will default to the number of CPU cores, which can be a
+build/generate_makefile: | libbacktrace
-#  problem on low-memory systems.
+endif
+build/generate_makefile: tools/generate_makefile.nim | deps-common
+	$(ENV_SCRIPT) nim c -o:$@ $(NIM_PARAMS) tools/generate_makefile.nim
+
+# GCC's LTO parallelisation is able to detect a GNU Make jobserver and get its
+# maximum number of processes from there, but only if we use the "+" prefix.
+# Without it, it will default to the number of CPU cores, which can be a
+# problem on low-memory systems.
+# It also requires Make to pass open file descriptors to the GCC process,
+# which is not possible if we let Nim handle this, so we generate and use a
+# makefile instead.
 $(TOOLS): | build deps
 	+ for D in $(TOOLS_DIRS); do [ -e "$${D}/$@.nim" ] && TOOL_DIR="$${D}" && break; done && \
 		echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim c -o:build/$@ $(NIM_PARAMS) "$${TOOL_DIR}/$@.nim" && \
+		$(ENV_SCRIPT) nim c --compileOnly -o:build/$@ $(NIM_PARAMS) "$${TOOL_DIR}/$@.nim" && \
+		build/generate_makefile "nimcache/release/$@/$@.json" "nimcache/release/$@/$@.makefile" && \
+		"$(MAKE)" -f "nimcache/release/$@/$@.makefile" --no-print-directory build $(HANDLE_OUTPUT) && \
 		echo -e $(BUILD_END_MSG) "build/$@"
 
 clean_eth2_network_simulation_data:
@@ -331,7 +346,7 @@ ntu: | build deps
 	+ $(ENV_SCRIPT) nim -d:danger -o:vendor/.nimble/bin/ntu c vendor/nim-testutils/ntu.nim
 
 clean: | clean-common
-	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,nimbus_beacon_node*,beacon_node_*,block_sim,state_sim,transition*}
+	rm -rf build/{$(TOOLS_CSV),all_tests,test_*,proto_array,fork_choice,*.a,*.so,*_node,*ssz*,nimbus_*,beacon_node*,block_sim,state_sim,transition*,generate_makefile}
 ifneq ($(USE_LIBBACKTRACE), 0)
 	+ "$(MAKE)" -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif

README.md

@@ -345,7 +345,7 @@ git config --global core.longpaths true
 You can now follow those instructions in the previous section by replacing `make` with `mingw32-make` (regardless of your 32-bit or 64-bit architecture):
 
 ```bash
-mingw32-make test # run the test suite
+mingw32-make -j $(nproc) test # run the test suite
 ```
 
 ### Linux, MacOS
@@ -356,15 +356,15 @@ After cloning the repo:
 # The first `make` invocation will update all Git submodules.
 # You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date.
 
-# Build nimbus_beacon_node and all the tools, using 4 parallel Make jobs
+# Build nimbus_beacon_node and all the tools, using as many parallel jobs as you have CPU cores
-make -j4
+make -j $(nproc)
 
 # Run tests
-make test
+make -j $(nproc) test
 
 # Update to latest version
 git pull
-make update
+make -j $(nproc) update
 ```
 
 To run a command that might use binaries from the Status Nim fork:
@@ -409,7 +409,7 @@ sudo apt-get install git
 
 ```bash
 # $(nproc) corresponds to the number of cores you have
-make -j$(nproc)
+make -j $(nproc)
 ```
 
 - build a specific tool:

config.nims

@@ -11,8 +11,8 @@ if defined(release) and not defined(disableLTO):
     switch("passC", "-flto=thin")
     switch("passL", "-flto=thin -Wl,-object_path_lto," & nimCachePath & "/lto")
   elif defined(linux):
-    switch("passC", "-flto=auto")
+    switch("passC", "-flto=jobserver")
-    switch("passL", "-flto=auto")
+    switch("passL", "-flto=jobserver")
     switch("passC", "-finline-limit=100000")
     switch("passL", "-finline-limit=100000")
   else:

scripts/launch_local_testnet.sh

@@ -185,11 +185,13 @@ else
   MAKE="make"
 fi
 
-NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}")
+# Build the binaries
-$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" nimbus_beacon_node nimbus_signing_process nimbus_validator_client deposit_contract
+BINARIES="nimbus_beacon_node nimbus_signing_process nimbus_validator_client deposit_contract"
 if [[ "$ENABLE_LOGTRACE" == "1" ]]; then
-  $MAKE LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" logtrace
+  BINARIES="${BINARIES} logtrace"
 fi
+NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh "${NETWORK}")
+$MAKE -j $(nproc) LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" ${BINARIES}
 
 PIDS=""
 WEB3_ARG=""

tools/generate_makefile.nim

@@ -0,0 +1,94 @@
+# Copyright (c) 2020 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# Generate a Makefile from the JSON file produce by the Nim compiler with
+# "--compileOnly". Suitable for Make-controlled parallelisation, down to the GCC
+# LTO level.
+
+import json, os, strutils
+
+# Ripped off from Nim's `linkViaResponseFile()` in "compiler/extccomp.nim".
+# It lets us get around a command line length limit on Windows.
+proc processLinkCmd(cmd, linkerArgs: string): string =
+  # Extracting the linker.exe here is a bit hacky but the best solution
+  # given ``buildLib``'s design.
+  var
+    i = 0
+    last = 0
+
+  if cmd.len > 0 and cmd[0] == '"':
+    inc i
+    while i < cmd.len and cmd[i] != '"': inc i
+    last = i
+    inc i
+  else:
+    while i < cmd.len and cmd[i] != ' ': inc i
+    last = i
+
+  while i < cmd.len and cmd[i] == ' ': inc i
+
+  let args = cmd.substr(i)
+  writeFile(linkerArgs, args.replace('\\', '/'))
+
+  return cmd.substr(0, last) & " @" & linkerArgs
+
+proc main() =
+  let nrParams = paramCount()
+
+  if nrParams != 2:
+    echo "Usage: ", paramStr(0), " input.json output.makefile"
+    quit(QuitFailure)
+
+  let
+    jsonPath = paramStr(1)
+    makefilePath = paramStr(2)
+
+  if not fileExists(jsonPath):
+    echo "No such file: ", jsonPath
+    quit(QuitFailure)
+
+  let
+    data = json.parseFile(jsonPath)
+    makefile = open(makefilePath, fmWrite)
+
+  defer:
+    makefile.close()
+
+  var
+    objectPath: string
+    found: bool
+    cmd: string
+
+  for compile in data["compile"]:
+    cmd = compile[1].getStr()
+    objectPath = ""
+    found = false
+    for token in split(cmd):
+      if found and token.len > 0 and token.endsWith(".o"):
+        objectPath = token
+        break
+      if token == "-o":
+        found = true
+    if found == false or objectPath == "":
+      echo "Could not find the object file in this command: ", cmd
+      quit(QuitFailure)
+    makefile.writeLine("$#: $#" % [objectPath, compile[0].getStr()])
+    makefile.writeLine("\t+ $#\n" % cmd)
+
+  var objects: seq[string]
+  for obj in data["link"]:
+    objects.add(obj.getStr())
+  makefile.writeLine("OBJECTS := $#\n" % objects.join(" \\\n"))
+
+  makefile.writeLine(".PHONY: build")
+  makefile.writeLine("build: $(OBJECTS)")
+  makefile.writeLine("\t+ $#" % processLinkCmd(data["linkcmd"].getStr(), makefilePath & ".linkerArgs"))
+  if data.hasKey("extraCmds"):
+    for cmd in data["extraCmds"]:
+      makefile.writeLine("\t+ $#" % cmd.getStr())
+
+main()
+