new Make target: "benchmarks" (#2174)

It runs some benchmarks, collects the output and generates HTML charts.

.gitignore

@@ -49,4 +49,5 @@ build/
 beacon_chain/sync_protocol.nim.generated.nim
 
 /dist
+/benchmark_results
 

.gitmodules

@@ -198,3 +198,8 @@
 	url = https://github.com/status-im/nim-zxcvbn.git
 	ignore = dirty
 	branch = master
+[submodule "vendor/nimbus-benchmarking"]
+	path = vendor/nimbus-benchmarking
+	url = https://github.com/status-im/nimbus-benchmarking.git
+	ignore = dirty
+	branch = master

Makefile

@@ -78,7 +78,8 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 	libbacktrace \
 	book \
 	publish-book \
-	dist
+	dist \
+	benchmarks
 
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included, so we update the submodules.
@@ -407,4 +408,11 @@ dist-test:
 			docker run --rm --name nimbus-eth2-dist-test -v $(CURDIR):/home/user/nimbus-eth2 nimbus-eth2-dist-test; \
 		done
 
+#- Build and run benchmarks using an external repo (which can be used easily on
+#  older commits, before this Make target was added).
+#- It's up to the user to create a benchmarking environment that minimises the
+#  results spread. We're showing a 95% CI bar to help visualise that.
+benchmarks:
+	+ vendor/nimbus-benchmarking/run_nbc_benchmarks.sh --output-type d3
+
 endif # "variables.mk" was not included

docs/the_nimbus_book/src/developers.md

@@ -113,6 +113,12 @@ make -j$(nproc) NIMFLAGS="-d:release" USE_MULTITAIL=yes eth2_network_simulation
 make USE_LIBBACKTRACE=0 # expect the resulting binaries to be 2-3 times slower
 ```
 
+- run some benchmarks and generate HTML charts
+
+```bash
+make benchmarks
+```
+
 ### Multi-client interop scripts
 
 [This repository](https://github.com/eth2-clients/multinet) contains a set of scripts used by the client implementation teams to test interop between the clients (in certain simplified scenarios). It mostly helps us find and debug issues.

vendor/nimbus-benchmarking

@@ -0,0 +1 @@
+Subproject commit ce544b93170d18cb50f30697a748edd0c59130a6