From 48391744ff33e8cf2bc08eeac4067ffe4a45de01 Mon Sep 17 00:00:00 2001 From: kdeme Date: Thu, 16 Jan 2020 14:30:53 +0100 Subject: [PATCH] Add metrics config files generation for wakusim --- Makefile | 4 +- waku/README.md | 54 ++-- waku/examples/waku-grafana-dashboard.json | 338 ++++++++++++++++------ waku/process_dashboard.nim | 55 ++++ waku/start_network.sh | 39 ++- 5 files changed, 383 insertions(+), 107 deletions(-) create mode 100644 waku/process_dashboard.nim diff --git a/Makefile b/Makefile index 51cdd9900..47ec90b75 100644 --- a/Makefile +++ b/Makefile @@ -14,8 +14,8 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk # debugging tools + testing tools -TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTestGen -TOOLS_DIRS := premix tests +TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTestGen process_dashboard +TOOLS_DIRS := premix tests waku # comma-separated values for the "clean" target TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) diff --git a/waku/README.md b/waku/README.md index 50841e155..523f8f55e 100644 --- a/waku/README.md +++ b/waku/README.md @@ -14,24 +14,6 @@ make wakunode ./build/wakunode --help ``` -# Testing Waku Protocol -One can set up several nodes, get them connected and then instruct them via the -JSON-RPC interface. This can be done via e.g. web3.js, nim-web3 (needs to be -updated) or simply curl your way out. - -The JSON-RPC interface is currently the same as the one of Whisper. The only -difference is the addition of broadcasting the topics interest when a filter -with a certain set of topics is subcribed. - -Example of a quick simulation test using this approach: -```bash -./waku/start_network.sh -# Or when multitail is installed -USE_MULTITAIL="yes" ./waku/start_network.sh - -./build/quicksim -``` - # Using Metrics Metrics are available for valid envelopes and dropped envelopes. @@ -57,3 +39,39 @@ For visualisation, similar steps can be used as is written down for Nimbus There is a similar example dashboard that includes visualisation of the envelopes available at `waku/examples/waku-grafana-dashboard.json`. + +# Testing Waku Protocol +One can set up several nodes, get them connected and then instruct them via the +JSON-RPC interface. This can be done via e.g. web3.js, nim-web3 (needs to be +updated) or simply curl your way out. + +The JSON-RPC interface is currently the same as the one of Whisper. The only +difference is the addition of broadcasting the topics interest when a filter +with a certain set of topics is subcribed. + +Example of a quick simulation using this approach: +```bash +# Build wakunode + quicksim +make NIMFLAGS="-d:insecure" wakusim + +# Start the simulation nodes +./waku/start_network.sh +# Or when multitail is installed +USE_MULTITAIL="yes" ./waku/start_network.sh + +# In another shell +./build/quicksim +``` + +The `start_network.sh` script will also provide a `prometheus.yml` with targets +set to all simulation nodes that are started. This way you can easily start +prometheus with this config, e.g.: + +```bash +cd waku/metrics/prometheus +prometheus +``` + +A Grafana dashboard containing the example dashboard for each simulation node +is also generated and can be imported in case you have Grafana running. +This dashboard can be found at `./waku/metrics/waku-sim-all-nodes-grafana-dashboard.json` \ No newline at end of file diff --git a/waku/examples/waku-grafana-dashboard.json b/waku/examples/waku-grafana-dashboard.json index 1310e88bf..258813bdf 100644 --- a/waku/examples/waku-grafana-dashboard.json +++ b/waku/examples/waku-grafana-dashboard.json @@ -15,17 +15,233 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 4, + "id": 1, "links": [], "panels": [ { "datasource": null, "gridPos": { - "h": 5, - "w": 8, + "h": 4, + "w": 6, "x": 0, "y": 0 }, + "id": 16, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "override": {}, + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "6.4.5", + "targets": [ + { + "expr": "connected_peers{node=\"0\"}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connected Peers #0", + "type": "gauge" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "valid_envelopes_total{node=\"0\"}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Valid Envelopes #0", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 10, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pluginVersion": "6.4.5", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "expr": "dropped_expired_envelopes_total{node=\"0\"} + dropped_from_future_envelopes_total{node=\"0\"} + dropped_low_pow_envelopes_total{node=\"0\"} + dropped_too_large_envelopes_total{node=\"0\"} + dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"} + dropped_topic_mismatch_envelopes_total{node=\"0\"} +dropped_benign_duplicate_envelopes_total{node=\"0\"} + dropped_malicious_duplicate_envelopes_total{node=\"0\"}", + "legendFormat": "dropped envelopes", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Dropped Envelopes #0", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "datasource": null, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 0 + }, "id": 14, "options": { "fieldOptions": { @@ -58,70 +274,22 @@ "pluginVersion": "6.4.5", "targets": [ { - "expr": "rate(process_cpu_seconds_total[5s]) * 100", + "expr": "rate(process_cpu_seconds_total{node=\"0\"}[5s]) * 100", "legendFormat": "CPU Usage", "refId": "A" } ], "timeFrom": null, "timeShift": null, - "title": "CPU Usage", + "title": "CPU Usage #0", "type": "gauge" }, { "datasource": null, "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 16, - "options": { - "fieldOptions": { - "calcs": [ - "last" - ], - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "override": {}, - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.4.5", - "targets": [ - { - "expr": "connected_peers", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Connected Peers", - "type": "gauge" - }, - { - "datasource": null, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, + "h": 4, + "w": 5, + "x": 19, "y": 0 }, "id": 18, @@ -156,13 +324,13 @@ "pluginVersion": "6.4.5", "targets": [ { - "expr": "process_resident_memory_bytes", + "expr": "process_resident_memory_bytes{node=\"0\"}", "refId": "A" } ], "timeFrom": null, "timeShift": null, - "title": "RSS Memory", + "title": "RSS Memory #0", "type": "gauge" }, { @@ -177,7 +345,7 @@ "h": 9, "w": 12, "x": 0, - "y": 5 + "y": 4 }, "id": 6, "legend": { @@ -206,57 +374,57 @@ "steppedLine": false, "targets": [ { - "expr": "valid_envelopes_total", + "expr": "valid_envelopes_total{node=\"0\"}", "hide": false, "instant": false, "legendFormat": "Valid", "refId": "A" }, { - "expr": "dropped_benign_duplicate_envelopes_total", + "expr": "dropped_benign_duplicate_envelopes_total{node=\"0\"}", "hide": false, "instant": false, "legendFormat": "Benign duplicate", "refId": "B" }, { - "expr": "dropped_malicious_duplicate_envelopes_total", + "expr": "dropped_malicious_duplicate_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Malicious duplicate", "refId": "C" }, { - "expr": "dropped_expired_envelopes_total", + "expr": "dropped_expired_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Expired", "refId": "D" }, { - "expr": "dropped_from_future_envelopes_total", + "expr": "dropped_from_future_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Future timestamped", "refId": "E" }, { - "expr": "dropped_low_pow_envelopes_total", + "expr": "dropped_low_pow_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Too low PoW", "refId": "F" }, { - "expr": "dropped_bloom_filter_mismatch_envelopes_total", + "expr": "dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Bloom filter mismatch", "refId": "G" }, { - "expr": "dropped_topic_mismatch_envelopes_total", + "expr": "dropped_topic_mismatch_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Topic mismatch", "refId": "H" }, { - "expr": "dropped_too_large_envelopes_total", + "expr": "dropped_too_large_envelopes_total{node=\"0\"}", "hide": false, "legendFormat": "Too Large", "refId": "I" @@ -266,7 +434,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Waku Envelopes", + "title": "Waku Envelopes #0", "tooltip": { "shared": true, "sort": 1, @@ -315,7 +483,7 @@ "h": 9, "w": 12, "x": 12, - "y": 5 + "y": 4 }, "id": 2, "legend": { @@ -348,20 +516,20 @@ "steppedLine": false, "targets": [ { - "expr": "connected_peers", + "expr": "connected_peers{node=\"0\"}", "intervalFactor": 1, "legendFormat": "Connected Peers", "refId": "A" }, { - "expr": "process_resident_memory_bytes", + "expr": "process_resident_memory_bytes{node=\"0\"}", "interval": "", "intervalFactor": 1, "legendFormat": "RSS Memory", "refId": "B" }, { - "expr": "rate(process_cpu_seconds_total[15s]) * 100", + "expr": "rate(process_cpu_seconds_total{node=\"0\"}[15s]) * 100", "legendFormat": "CPU usage %", "refId": "C" } @@ -370,7 +538,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Waku Node", + "title": "Waku Node #0", "tooltip": { "shared": true, "sort": 0, @@ -419,7 +587,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 13 }, "id": 8, "legend": { @@ -447,12 +615,12 @@ "steppedLine": false, "targets": [ { - "expr": "process_max_fds", + "expr": "process_max_fds{node=\"0\"}", "legendFormat": "Maximum file descriptors", "refId": "A" }, { - "expr": "process_open_fds", + "expr": "process_open_fds{node=\"0\"}", "legendFormat": "Open file descriptors", "refId": "B" } @@ -461,7 +629,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "File Descriptors", + "title": "File Descriptors #0", "tooltip": { "shared": true, "sort": 0, @@ -510,7 +678,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 13 }, "id": 4, "legend": { @@ -538,22 +706,22 @@ "steppedLine": false, "targets": [ { - "expr": "nim_gc_mem_bytes", + "expr": "nim_gc_mem_bytes{node=\"0\"}", "legendFormat": "Nim GC total memory", "refId": "A" }, { - "expr": "nim_gc_mem_occupied_bytes", + "expr": "nim_gc_mem_occupied_bytes{node=\"0\"}", "legendFormat": "Nim GC used memory", "refId": "B" }, { - "expr": "process_resident_memory_bytes", + "expr": "process_resident_memory_bytes{node=\"0\"}", "legendFormat": "RSS memory", "refId": "C" }, { - "expr": "process_virtual_memory_bytes", + "expr": "process_virtual_memory_bytes{node=\"0\"}", "legendFormat": "Virtual memory", "refId": "D" } @@ -562,7 +730,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Usage", + "title": "Memory Usage #0", "tooltip": { "shared": true, "sort": 0, @@ -628,5 +796,5 @@ "timezone": "", "title": "Waku Node", "uid": "K7Z6IoBZk", - "version": 10 + "version": 5 } \ No newline at end of file diff --git a/waku/process_dashboard.nim b/waku/process_dashboard.nim new file mode 100644 index 000000000..cf741c434 --- /dev/null +++ b/waku/process_dashboard.nim @@ -0,0 +1,55 @@ +# copy from https://github.com/status-im/nim-beacon-chain/blob/master/tests/simulation/process_dashboard.nim +import json, parseopt, strutils + +# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json +var + p = initOptParser() + nodes: int + inputFileName, outputFilename: string + +while true: + p.next() + case p.kind: + of cmdEnd: + break + of cmdShortOption, cmdLongOption: + if p.key == "nodes": + nodes = p.val.parseInt() + elif p.key == "in": + inputFileName = p.val + elif p.key == "out": + outputFileName = p.val + else: + echo "unsupported argument: ", p.key + of cmdArgument: + echo "unsupported argument: ", p.key + +var + inputData = parseFile(inputFileName) + panels = inputData["panels"].copy() + numPanels = len(panels) + gridHeight = 0 + outputData = inputData + +for panel in panels: + if panel["gridPos"]["x"].getInt() == 0: + gridHeight += panel["gridPos"]["h"].getInt() + +outputData["panels"] = %* [] +for nodeNum in 0 .. (nodes - 1): + var + nodePanels = panels.copy() + panelIndex = 0 + for panel in nodePanels.mitems: + panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum) + panel["id"] = %* (panelIndex + (nodeNum * numPanels)) + panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight)) + var targets = panel["targets"] + for target in targets.mitems: + target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}") + outputData["panels"].add(panel) + panelIndex.inc() + +outputData["uid"] = %* (outputData["uid"].getStr() & "a") +outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)") +writeFile(outputFilename, pretty(outputData)) diff --git a/waku/start_network.sh b/waku/start_network.sh index b251296f7..74484e643 100755 --- a/waku/start_network.sh +++ b/waku/start_network.sh @@ -6,16 +6,51 @@ set -e WAKU_NODE_BIN="./build/wakunode" NODE_PK="5dc5381cae54ba3174dc0d46040fe11614d0cc94d41185922585198b4fcef9d3" NODE_ENODE="enode://e5fd642a0f630bbb1e4cd7df629d7b8b019457a9a74f983c0484a045cebb176def86a54185b50bbba6bbf97779173695e92835d63109c23471e6da382f922fdb@0.0.0.0:30303" -DEFAULTS="--log-level:DEBUG --discovery:0 --log-metrics" +DEFAULTS="--log-level:DEBUG --discovery:off --log-metrics --metrics-server" LIGHT_NODE="--light-node:1" WAKU_LIGHT_NODE="--waku-mode:WakuChan ${LIGHT_NODE}" +METRICS_DIR="./waku/metrics" # multitail support MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no" -# TODO: metrics configs +# TODO: This is based on the nim-beacon-chain eth2_network_simulation but quite +# the much worse version due to the custom nodes we need. Need to rework this +# to be less hardcoded, but might use a Nim application for this, hence the +# quick and dirty way for now. + +mkdir -p "${METRICS_DIR}"/prometheus/ +cat > "${METRICS_DIR}/prometheus/prometheus.yml" <