diff --git a/Makefile b/Makefile index 3abfcde86..9658724d7 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system # we don't want an error here, so we can handle things later, in the build-system-checks target -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk -TOOLS := beacon_node bench_bls_sig_agggregation state_sim ncli_hash_tree_root ncli_pretty ncli_signing_root ncli_transition -TOOLS_DIRS := beacon_chain benchmarks research ncli +TOOLS := beacon_node bench_bls_sig_agggregation state_sim ncli_hash_tree_root ncli_pretty ncli_signing_root ncli_transition process_dashboard +TOOLS_DIRS := beacon_chain benchmarks research ncli tests/simulation TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) .PHONY: all build-system-checks deps update p2pd test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0 clean-testnet1 testnet1 clean @@ -67,7 +67,7 @@ $(TOOLS): | build deps clean_eth2_network_simulation_files: rm -rf tests/simulation/{data,validators} -eth2_network_simulation: | build deps p2pd clean_eth2_network_simulation_files +eth2_network_simulation: | build deps p2pd clean_eth2_network_simulation_files process_dashboard GIT_ROOT="$$PWD" tests/simulation/start.sh testnet0 testnet1: | build deps diff --git a/README.md b/README.md index 28e57198d..237c21809 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,26 @@ You can find out more about it in the [development update](https://our.status.im _Alternatively, fire up our [experimental Vagrant instance with Nim pre-installed](https://our.status.im/setting-up-a-local-vagrant-environment-for-nim-development/) and give us yout feedback about the process!_ +### Visualising simulation metrics + +Those [generic instructions from the Nimbus repo](https://github.com/status-im/nimbus/#metric-visualisation) apply here as well. + +Specific steps: + +```bash +# This will generate the Prometheus config and the Grafana dashboard on the fly, +# based on the number of nodes (which you can control by passing something like NODES=6 to `make`). +make eth2_network_simulation + +# In another terminal tab, after the sim started: +cd tests/simulation/prometheus +prometheus +``` + +The dashboard you need to import in Grafana is "tests/simulation/beacon-chain-sim-all-nodes-Grafana-dashboard.json". + +[Obligatory screenshot.](https://i.imgur.com/pLvLhID.png) + ### Makefile tips and tricks for developers - build all those tools known to the Makefile: diff --git a/beacon_chain/validator_keygen.nim b/beacon_chain/validator_keygen.nim index a275c7f04..3d12af670 100644 --- a/beacon_chain/validator_keygen.nim +++ b/beacon_chain/validator_keygen.nim @@ -9,11 +9,11 @@ contract(DepositContract): proc writeTextFile(filename: string, contents: string) = writeFile(filename, contents) - echo "Wrote ", filename + # echo "Wrote ", filename proc writeFile(filename: string, value: auto) = Json.saveFile(filename, value, pretty = true) - echo "Wrote ", filename + # echo "Wrote ", filename proc ethToWei(eth: UInt256): UInt256 = eth * 1000000000000000000.u256 diff --git a/tests/simulation/.gitignore b/tests/simulation/.gitignore index 274d05777..743be97b5 100644 --- a/tests/simulation/.gitignore +++ b/tests/simulation/.gitignore @@ -1,3 +1,5 @@ data/ validators/ +prometheus/ +beacon-chain-sim-all-nodes-Grafana-dashboard.json diff --git a/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json b/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json new file mode 100644 index 000000000..589ae2811 --- /dev/null +++ b/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json @@ -0,0 +1,442 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 6, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "process_resident_memory_bytes{instance=\"localhost:9093\",job=\"nimbus\"}", + "yaxis": 2 + }, + { + "alias": "process_virtual_memory_bytes{instance=\"localhost:9093\",job=\"nimbus\"}", + "yaxis": 2 + }, + { + "alias": "nim_gc_mem_bytes{instance=\"localhost:9093\",job=\"nimbus\"}", + "yaxis": 2 + }, + { + "alias": "nim_gc_mem_occupied_bytes{instance=\"localhost:9093\",job=\"nimbus\"}", + "yaxis": 2 + }, + { + "alias": "RSS", + "yaxis": 2 + }, + { + "alias": "virtual mem", + "yaxis": 2 + }, + { + "alias": "Nim GC mem total", + "yaxis": 2 + }, + { + "alias": "Nim GC mem used", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{node=\"0\"}[2s]) * 100", + "legendFormat": "CPU usage %", + "refId": "A" + }, + { + "expr": "process_open_fds{node=\"0\"}", + "legendFormat": "open file descriptors", + "refId": "C" + }, + { + "expr": "process_resident_memory_bytes{node=\"0\"}", + "legendFormat": "RSS", + "refId": "D" + }, + { + "expr": "nim_gc_mem_bytes{node=\"0\"}", + "legendFormat": "Nim GC mem total", + "refId": "F" + }, + { + "expr": "nim_gc_mem_occupied_bytes{node=\"0\"}", + "legendFormat": "Nim GC mem used", + "refId": "G" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "beacon_node #0", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 8 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pluginVersion": "6.3.5", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "process_resident_memory_bytes{node=\"0\"}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "RSS mem #0", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 5, + "y": 8 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pluginVersion": "6.3.5", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{node=\"0\"}[2s]) * 100", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "CPU usage #0", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 8 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "libp2p_peers{node=\"0\"}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "peers #0", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "beacon chain sim", + "uid": "pgeNfj2Wz", + "version": 15 +} \ No newline at end of file diff --git a/tests/simulation/process_dashboard.nim b/tests/simulation/process_dashboard.nim new file mode 100644 index 000000000..20ffac1c2 --- /dev/null +++ b/tests/simulation/process_dashboard.nim @@ -0,0 +1,55 @@ +import json, parseopt, strutils + +# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json +var + p = initOptParser() + nodes: int + inputFileName, outputFilename: string + +while true: + p.next() + case p.kind: + of cmdEnd: + break + of cmdShortOption, cmdLongOption: + if p.key == "nodes": + nodes = p.val.parseInt() + elif p.key == "in": + inputFileName = p.val + elif p.key == "out": + outputFileName = p.val + else: + echo "unsupported argument: ", p.key + of cmdArgument: + echo "unsupported argument: ", p.key + +var + inputData = parseFile(inputFileName) + panels = inputData["panels"].copy() + numPanels = len(panels) + gridHeight = 0 + outputData = inputData + +for panel in panels: + if panel["gridPos"]["x"].getInt() == 0: + gridHeight += panel["gridPos"]["h"].getInt() + +outputData["panels"] = %* [] +for nodeNum in 0 .. (nodes - 1): + var + nodePanels = panels.copy() + panelIndex = 0 + for panel in nodePanels.mitems: + panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum) + panel["id"] = %* (panelIndex + (nodeNum * numPanels)) + panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight)) + var targets = panel["targets"] + for target in targets.mitems: + target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}") + outputData["panels"].add(panel) + panelIndex.inc() + +outputData["uid"] = %* (outputData["uid"].getStr() & "a") +outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)") +writeFile(outputFilename, pretty(outputData)) + diff --git a/tests/simulation/run_node.sh b/tests/simulation/run_node.sh index 852ab64e7..b966aa5fd 100755 --- a/tests/simulation/run_node.sh +++ b/tests/simulation/run_node.sh @@ -38,5 +38,8 @@ $BEACON_NODE_BIN \ --stateSnapshot:$SNAPSHOT_FILE \ $DEPOSIT_WEB3_URL_ARG \ --depositContractAddress=$DEPOSIT_CONTRACT_ADDRESS \ - $* + --metricsServer=true \ + --metricsServerAddress="127.0.0.1" \ + --metricsServerPort="$(( $BASE_METRICS_PORT + $NODE_ID ))" + "$@" diff --git a/tests/simulation/start.sh b/tests/simulation/start.sh index ee788dd64..4805c1864 100755 --- a/tests/simulation/start.sh +++ b/tests/simulation/start.sh @@ -19,7 +19,7 @@ mkdir -p "$VALIDATORS_DIR" cd "$GIT_ROOT" -NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --opt:speed --debuginfo" +NIMFLAGS="-d:chronicles_log_level=DEBUG --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo" # Run with "SHARD_COUNT=4 ./start.sh" to change these DEFS="" @@ -75,6 +75,23 @@ MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no" +# Prometheus config (continued inside the loop) +mkdir -p "${METRICS_DIR}" +cat > "${METRICS_DIR}/prometheus.yml" <> "${METRICS_DIR}/prometheus.yml" <