Add metrics config files generation for wakusim

Makefile

@@ -14,8 +14,8 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system
 -include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
 
 # debugging tools + testing tools
-TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTestGen
+TOOLS := premix persist debug dumper hunter regress tracerTestGen persistBlockTestGen process_dashboard
-TOOLS_DIRS := premix tests
+TOOLS_DIRS := premix tests waku
 # comma-separated values for the "clean" target
 TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 

waku/README.md

@@ -14,24 +14,6 @@ make wakunode
 ./build/wakunode --help
 ```
 
-# Testing Waku Protocol
-One can set up several nodes, get them connected and then instruct them via the
-JSON-RPC interface. This can be done via e.g. web3.js, nim-web3 (needs to be
-updated) or simply curl your way out.
-
-The JSON-RPC interface is currently the same as the one of Whisper. The only
-difference is the addition of broadcasting the topics interest when a filter
-with a certain set of topics is subcribed.
-
-Example of a quick simulation test using this approach:
-```bash
-./waku/start_network.sh
-# Or when multitail is installed
-USE_MULTITAIL="yes" ./waku/start_network.sh
-
-./build/quicksim
-```
-
 # Using Metrics
 
 Metrics are available for valid envelopes and dropped envelopes.
@@ -57,3 +39,39 @@ For visualisation, similar steps can be used as is written down for Nimbus
 
 There is a similar example dashboard that includes visualisation of the
 envelopes available at `waku/examples/waku-grafana-dashboard.json`.
+
+# Testing Waku Protocol
+One can set up several nodes, get them connected and then instruct them via the
+JSON-RPC interface. This can be done via e.g. web3.js, nim-web3 (needs to be
+updated) or simply curl your way out.
+
+The JSON-RPC interface is currently the same as the one of Whisper. The only
+difference is the addition of broadcasting the topics interest when a filter
+with a certain set of topics is subcribed.
+
+Example of a quick simulation using this approach:
+```bash
+# Build wakunode + quicksim
+make NIMFLAGS="-d:insecure" wakusim
+
+# Start the simulation nodes
+./waku/start_network.sh
+# Or when multitail is installed
+USE_MULTITAIL="yes" ./waku/start_network.sh
+
+# In another shell
+./build/quicksim
+```
+
+The `start_network.sh` script will also provide a `prometheus.yml` with targets
+set to all simulation nodes that are started. This way you can easily start
+prometheus with this config, e.g.:
+
+```bash
+cd waku/metrics/prometheus
+prometheus
+```
+
+A Grafana dashboard containing the example dashboard for each simulation node
+is also generated and can be imported in case you have Grafana running.
+This dashboard can be found at `./waku/metrics/waku-sim-all-nodes-grafana-dashboard.json`

waku/examples/waku-grafana-dashboard.json

@@ -15,17 +15,233 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 4,
+  "id": 1,
   "links": [],
   "panels": [
     {
       "datasource": null,
       "gridPos": {
-        "h": 5,
+        "h": 4,
-        "w": 8,
+        "w": 6,
         "x": 0,
         "y": 0
       },
+      "id": 16,
+      "options": {
+        "fieldOptions": {
+          "calcs": [
+            "last"
+          ],
+          "defaults": {
+            "mappings": [],
+            "max": 100,
+            "min": 0,
+            "thresholds": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "override": {},
+          "values": false
+        },
+        "orientation": "auto",
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "6.4.5",
+      "targets": [
+        {
+          "expr": "connected_peers{node=\"0\"}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Connected Peers #0",
+      "type": "gauge"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": null,
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 6,
+        "y": 0
+      },
+      "id": 22,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false,
+        "ymax": null,
+        "ymin": null
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "valid_envelopes_total{node=\"0\"}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Valid Envelopes #0",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": null,
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 10,
+        "y": 0
+      },
+      "id": 20,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "pluginVersion": "6.4.5",
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false,
+        "ymax": null,
+        "ymin": null
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "dropped_expired_envelopes_total{node=\"0\"} + dropped_from_future_envelopes_total{node=\"0\"} + dropped_low_pow_envelopes_total{node=\"0\"} + dropped_too_large_envelopes_total{node=\"0\"} + dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"} + dropped_topic_mismatch_envelopes_total{node=\"0\"} +dropped_benign_duplicate_envelopes_total{node=\"0\"} + dropped_malicious_duplicate_envelopes_total{node=\"0\"}",
+          "legendFormat": "dropped envelopes",
+          "refId": "A"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Dropped Envelopes #0",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "datasource": null,
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
       "id": 14,
       "options": {
         "fieldOptions": {
@@ -58,70 +274,22 @@
       "pluginVersion": "6.4.5",
       "targets": [
         {
-          "expr": "rate(process_cpu_seconds_total[5s]) * 100",
+          "expr": "rate(process_cpu_seconds_total{node=\"0\"}[5s]) * 100",
           "legendFormat": "CPU Usage",
           "refId": "A"
         }
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "CPU Usage",
+      "title": "CPU Usage #0",
       "type": "gauge"
     },
     {
       "datasource": null,
       "gridPos": {
-        "h": 5,
+        "h": 4,
-        "w": 8,
+        "w": 5,
-        "x": 8,
+        "x": 19,
-        "y": 0
-      },
-      "id": 16,
-      "options": {
-        "fieldOptions": {
-          "calcs": [
-            "last"
-          ],
-          "defaults": {
-            "mappings": [],
-            "max": 100,
-            "min": 0,
-            "thresholds": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "override": {},
-          "values": false
-        },
-        "orientation": "auto",
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true
-      },
-      "pluginVersion": "6.4.5",
-      "targets": [
-        {
-          "expr": "connected_peers",
-          "refId": "A"
-        }
-      ],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Connected Peers",
-      "type": "gauge"
-    },
-    {
-      "datasource": null,
-      "gridPos": {
-        "h": 5,
-        "w": 8,
-        "x": 16,
         "y": 0
       },
       "id": 18,
@@ -156,13 +324,13 @@
       "pluginVersion": "6.4.5",
       "targets": [
         {
-          "expr": "process_resident_memory_bytes",
+          "expr": "process_resident_memory_bytes{node=\"0\"}",
           "refId": "A"
         }
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "RSS Memory",
+      "title": "RSS Memory #0",
       "type": "gauge"
     },
     {
@@ -177,7 +345,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 5
+        "y": 4
       },
       "id": 6,
       "legend": {
@@ -206,57 +374,57 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "valid_envelopes_total",
+          "expr": "valid_envelopes_total{node=\"0\"}",
           "hide": false,
           "instant": false,
           "legendFormat": "Valid",
           "refId": "A"
         },
         {
-          "expr": "dropped_benign_duplicate_envelopes_total",
+          "expr": "dropped_benign_duplicate_envelopes_total{node=\"0\"}",
           "hide": false,
           "instant": false,
           "legendFormat": "Benign duplicate",
           "refId": "B"
         },
         {
-          "expr": "dropped_malicious_duplicate_envelopes_total",
+          "expr": "dropped_malicious_duplicate_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Malicious duplicate",
           "refId": "C"
         },
         {
-          "expr": "dropped_expired_envelopes_total",
+          "expr": "dropped_expired_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Expired",
           "refId": "D"
         },
         {
-          "expr": "dropped_from_future_envelopes_total",
+          "expr": "dropped_from_future_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Future timestamped",
           "refId": "E"
         },
         {
-          "expr": "dropped_low_pow_envelopes_total",
+          "expr": "dropped_low_pow_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Too low PoW",
           "refId": "F"
         },
         {
-          "expr": "dropped_bloom_filter_mismatch_envelopes_total",
+          "expr": "dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Bloom filter mismatch",
           "refId": "G"
         },
         {
-          "expr": "dropped_topic_mismatch_envelopes_total",
+          "expr": "dropped_topic_mismatch_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Topic mismatch",
           "refId": "H"
         },
         {
-          "expr": "dropped_too_large_envelopes_total",
+          "expr": "dropped_too_large_envelopes_total{node=\"0\"}",
           "hide": false,
           "legendFormat": "Too Large",
           "refId": "I"
@@ -266,7 +434,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Waku Envelopes",
+      "title": "Waku Envelopes #0",
       "tooltip": {
         "shared": true,
         "sort": 1,
@@ -315,7 +483,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 5
+        "y": 4
       },
       "id": 2,
       "legend": {
@@ -348,20 +516,20 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "connected_peers",
+          "expr": "connected_peers{node=\"0\"}",
           "intervalFactor": 1,
           "legendFormat": "Connected Peers",
           "refId": "A"
         },
         {
-          "expr": "process_resident_memory_bytes",
+          "expr": "process_resident_memory_bytes{node=\"0\"}",
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "RSS Memory",
           "refId": "B"
         },
         {
-          "expr": "rate(process_cpu_seconds_total[15s]) * 100",
+          "expr": "rate(process_cpu_seconds_total{node=\"0\"}[15s]) * 100",
           "legendFormat": "CPU usage %",
           "refId": "C"
         }
@@ -370,7 +538,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Waku Node",
+      "title": "Waku Node #0",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -419,7 +587,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 14
+        "y": 13
       },
       "id": 8,
       "legend": {
@@ -447,12 +615,12 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "process_max_fds",
+          "expr": "process_max_fds{node=\"0\"}",
           "legendFormat": "Maximum file descriptors",
           "refId": "A"
         },
         {
-          "expr": "process_open_fds",
+          "expr": "process_open_fds{node=\"0\"}",
           "legendFormat": "Open file descriptors",
           "refId": "B"
         }
@@ -461,7 +629,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "File Descriptors",
+      "title": "File Descriptors #0",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -510,7 +678,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 14
+        "y": 13
       },
       "id": 4,
       "legend": {
@@ -538,22 +706,22 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "nim_gc_mem_bytes",
+          "expr": "nim_gc_mem_bytes{node=\"0\"}",
           "legendFormat": "Nim GC total memory",
           "refId": "A"
         },
         {
-          "expr": "nim_gc_mem_occupied_bytes",
+          "expr": "nim_gc_mem_occupied_bytes{node=\"0\"}",
           "legendFormat": "Nim GC used memory",
           "refId": "B"
         },
         {
-          "expr": "process_resident_memory_bytes",
+          "expr": "process_resident_memory_bytes{node=\"0\"}",
           "legendFormat": "RSS memory",
           "refId": "C"
         },
         {
-          "expr": "process_virtual_memory_bytes",
+          "expr": "process_virtual_memory_bytes{node=\"0\"}",
           "legendFormat": "Virtual memory",
           "refId": "D"
         }
@@ -562,7 +730,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Memory Usage",
+      "title": "Memory Usage #0",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -628,5 +796,5 @@
   "timezone": "",
   "title": "Waku Node",
   "uid": "K7Z6IoBZk",
-  "version": 10
+  "version": 5
 }

waku/process_dashboard.nim

@@ -0,0 +1,55 @@
+# copy from https://github.com/status-im/nim-beacon-chain/blob/master/tests/simulation/process_dashboard.nim
+import json, parseopt, strutils
+
+# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json
+var
+  p = initOptParser()
+  nodes: int
+  inputFileName, outputFilename: string
+
+while true:
+  p.next()
+  case p.kind:
+    of cmdEnd:
+      break
+    of cmdShortOption, cmdLongOption:
+      if p.key == "nodes":
+        nodes = p.val.parseInt()
+      elif p.key == "in":
+        inputFileName = p.val
+      elif p.key == "out":
+        outputFileName = p.val
+      else:
+        echo "unsupported argument: ", p.key
+    of cmdArgument:
+      echo "unsupported argument: ", p.key
+
+var
+  inputData = parseFile(inputFileName)
+  panels = inputData["panels"].copy()
+  numPanels = len(panels)
+  gridHeight = 0
+  outputData = inputData
+
+for panel in panels:
+  if panel["gridPos"]["x"].getInt() == 0:
+    gridHeight += panel["gridPos"]["h"].getInt()
+
+outputData["panels"] = %* []
+for nodeNum in 0 .. (nodes - 1):
+  var
+    nodePanels = panels.copy()
+    panelIndex = 0
+  for panel in nodePanels.mitems:
+    panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum)
+    panel["id"] = %* (panelIndex + (nodeNum * numPanels))
+    panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight))
+    var targets = panel["targets"]
+    for target in targets.mitems:
+      target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}")
+    outputData["panels"].add(panel)
+    panelIndex.inc()
+
+outputData["uid"] = %* (outputData["uid"].getStr() & "a")
+outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)")
+writeFile(outputFilename, pretty(outputData))

waku/start_network.sh

@@ -6,16 +6,51 @@ set -e
 WAKU_NODE_BIN="./build/wakunode"
 NODE_PK="5dc5381cae54ba3174dc0d46040fe11614d0cc94d41185922585198b4fcef9d3"
 NODE_ENODE="enode://e5fd642a0f630bbb1e4cd7df629d7b8b019457a9a74f983c0484a045cebb176def86a54185b50bbba6bbf97779173695e92835d63109c23471e6da382f922fdb@0.0.0.0:30303"
-DEFAULTS="--log-level:DEBUG --discovery:0 --log-metrics"
+DEFAULTS="--log-level:DEBUG --discovery:off --log-metrics --metrics-server"
 LIGHT_NODE="--light-node:1"
 WAKU_LIGHT_NODE="--waku-mode:WakuChan ${LIGHT_NODE}"
+METRICS_DIR="./waku/metrics"
 
 # multitail support
 MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
 USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
 type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
 
-# TODO: metrics configs
+# TODO: This is based on the nim-beacon-chain eth2_network_simulation but quite
+# the much worse version due to the custom nodes we need. Need to rework this
+# to be less hardcoded, but might use a Nim application for this, hence the
+# quick and dirty way for now.
+
+mkdir -p "${METRICS_DIR}"/prometheus/
+cat > "${METRICS_DIR}/prometheus/prometheus.yml" <<EOF
+global:
+  scrape_interval: 1s
+
+scrape_configs:
+  - job_name: "wakusim"
+    static_configs:
+    - targets: ['127.0.0.1:8008']
+      labels:
+        node: '0'
+    - targets: ['127.0.0.1:8009']
+      labels:
+        node: '1'
+    - targets: ['127.0.0.1:8010']
+      labels:
+        node: '2'
+    - targets: ['127.0.0.1:8011']
+      labels:
+        node: '3'
+    - targets: ['127.0.0.1:8012']
+      labels:
+        node: '4'
+EOF
+
+# use the exported Grafana dashboard for a single node to create one for all nodes
+"./build/process_dashboard" \
+  --nodes=5 \
+  --in="waku/examples/waku-grafana-dashboard.json" \
+  --out="${METRICS_DIR}/waku-sim-all-nodes-grafana-dashboard.json"
 
 if [[ "$USE_MULTITAIL" != "no" ]]; then
   SLEEP=0