Merge pull request #1138 from status-im/grafana

Grafana: only show one node per page [skip ci]

scripts/launch_local_testnet.sh

@@ -177,7 +177,6 @@ EOF
 
 	# use the exported Grafana dashboard for a single node to create one for all nodes
 	./build/process_dashboard \
-	  --nodes=${NUM_NODES} \
 	  --in="tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json" \
 	  --out="${DATA_DIR}/local-testnet-all-nodes-Grafana-dashboard.json"
 fi

scripts/reset_testnet.sh

@@ -69,7 +69,6 @@ make -j2 NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" be
 echo "Generating Grafana dashboards for remote testnet servers"
 for testnet in 0 1; do
   ./build/process_dashboard \
-    --nodes=20 \
     --in="tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json" \
     --out="docker/beacon-chain-sim-remote-testnet${testnet}-Grafana-dashboard.json" \
     --type="remote" \

tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json

@@ -47,22 +47,6 @@
   "id": 13,
   "links": [],
   "panels": [
-    {
-      "content": "",
-      "datasource": null,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 44,
-      "mode": "markdown",
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "node #0",
-      "type": "text"
-    },
     {
       "aliasColors": {},
       "bars": false,
@@ -75,7 +59,7 @@
         "h": 6,
         "w": 14,
         "x": 0,
-        "y": 1
+        "y": 0
       },
       "hiddenSeries": false,
       "id": 2,
@@ -195,7 +179,7 @@
         "h": 5,
         "w": 10,
         "x": 14,
-        "y": 1
+        "y": 0
       },
       "hiddenSeries": false,
       "id": 16,
@@ -289,7 +273,7 @@
         "h": 5,
         "w": 10,
         "x": 14,
-        "y": 6
+        "y": 5
       },
       "hiddenSeries": false,
       "id": 20,
@@ -385,7 +369,7 @@
         "h": 6,
         "w": 14,
         "x": 0,
-        "y": 7
+        "y": 6
       },
       "hiddenSeries": false,
       "id": 18,
@@ -480,7 +464,7 @@
         "h": 5,
         "w": 10,
         "x": 14,
-        "y": 11
+        "y": 10
       },
       "hiddenSeries": false,
       "id": 24,
@@ -569,7 +553,7 @@
         "h": 6,
         "w": 14,
         "x": 0,
-        "y": 13
+        "y": 12
       },
       "hiddenSeries": false,
       "id": 22,
@@ -672,7 +656,7 @@
         "h": 2,
         "w": 5,
         "x": 14,
-        "y": 16
+        "y": 15
       },
       "id": 6,
       "interval": null,
@@ -755,7 +739,7 @@
         "h": 2,
         "w": 5,
         "x": 19,
-        "y": 16
+        "y": 15
       },
       "id": 8,
       "interval": null,
@@ -838,7 +822,7 @@
         "h": 2,
         "w": 3,
         "x": 14,
-        "y": 18
+        "y": 17
       },
       "id": 28,
       "interval": null,
@@ -924,7 +908,7 @@
         "h": 2,
         "w": 4,
         "x": 17,
-        "y": 18
+        "y": 17
       },
       "id": 13,
       "interval": null,
@@ -1009,7 +993,7 @@
         "h": 2,
         "w": 3,
         "x": 21,
-        "y": 18
+        "y": 17
       },
       "id": 14,
       "interval": null,
@@ -1084,7 +1068,7 @@
         "h": 6,
         "w": 14,
         "x": 0,
-        "y": 19
+        "y": 18
       },
       "hiddenSeries": false,
       "id": 38,
@@ -1188,7 +1172,7 @@
         "h": 2,
         "w": 3,
         "x": 14,
-        "y": 20
+        "y": 19
       },
       "id": 32,
       "interval": null,
@@ -1272,7 +1256,7 @@
         "h": 2,
         "w": 4,
         "x": 17,
-        "y": 20
+        "y": 19
       },
       "id": 34,
       "interval": null,
@@ -1357,7 +1341,7 @@
         "h": 2,
         "w": 3,
         "x": 21,
-        "y": 20
+        "y": 19
       },
       "id": 40,
       "interval": null,
@@ -1442,7 +1426,7 @@
         "h": 2,
         "w": 3,
         "x": 14,
-        "y": 22
+        "y": 21
       },
       "id": 12,
       "interval": null,
@@ -1524,7 +1508,7 @@
         "h": 2,
         "w": 4,
         "x": 17,
-        "y": 22
+        "y": 21
       },
       "id": 36,
       "interval": null,
@@ -1598,7 +1582,7 @@
         "h": 5,
         "w": 14,
         "x": 0,
-        "y": 25
+        "y": 24
       },
       "hiddenSeries": false,
       "id": 30,
@@ -1700,7 +1684,7 @@
         "h": 6,
         "w": 14,
         "x": 0,
-        "y": 30
+        "y": 29
       },
       "heatmap": {},
       "hideZeroBuckets": false,
@@ -1780,5 +1764,5 @@
   "variables": {
     "list": []
   },
-  "version": 35
+  "version": 38
 }

tests/simulation/process_dashboard.nim

@@ -1,41 +1,23 @@
 import json, parseopt, strutils
 
-# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json --type=local --testnet=0
+# usage: process_dashboard --in=node0_dashboard.json --out=all_nodes_dashboard.json --type=local --testnet=0
 type
   OutputType = enum
     local
     remote
 var
   p = initOptParser()
-  nodes: int
   inputFileName, outputFilename: string
   outputType = OutputType.local
   testnet = 0
 
-let
-  hosts = [
-    "master-01",
-    "node-01",
-    "node-02",
-    "node-03",
-    "node-04",
-    "node-05",
-    "node-06",
-    "node-07",
-    "node-08",
-    "node-09",
-  ]
-  nodesPerHost = 2
-
 while true:
   p.next()
   case p.kind:
     of cmdEnd:
       break
     of cmdShortOption, cmdLongOption:
-      if p.key == "nodes":
+      if p.key == "in":
-        nodes = p.val.parseInt()
-      elif p.key == "in":
         inputFileName = p.val
       elif p.key == "out":
         outputFileName = p.val
@@ -51,52 +33,148 @@ while true:
 var
   inputData = parseFile(inputFileName)
   panels = inputData["panels"].copy()
-  numPanels = len(panels)
-  gridHeight = 0
   outputData = inputData
 
-for panel in panels:
+#############
-  if panel["gridPos"]["x"].getInt() == 0:
+# variables #
-    gridHeight += panel["gridPos"]["h"].getInt()
+#############
-
-outputData["panels"] = %* []
-if outputType == OutputType.remote:
-  var annotations = outputData["annotations"]["list"]
-  for annotation in annotations.mitems:
-    annotation["datasource"] = %* "-- Grafana --"
-
-for nodeNum in 0 .. (nodes - 1):
-  var
-    nodePanels = panels.copy()
-    panelIndex = 0
-  for panel in nodePanels.mitems:
-    panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum)
-    panel["id"] = %* (panelIndex + (nodeNum * numPanels))
-    panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight))
-    if outputType == OutputType.remote:
-      panel["datasource"] = newJNull()
-    if panel.hasKey("targets"):
-      var targets = panel["targets"]
-      for target in targets.mitems:
-        case outputType:
-          of OutputType.local:
-            target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}")
-          of OutputType.remote:
-            # The remote Prometheus instance polls once per minute, so the
-            # minimum rate() interval is 2 minutes.
-            target["expr"] = %* multiReplace(target["expr"].getStr(),
-                                  ("{node=\"0\"}", "{container=\"beacon-node-testnet" & $testnet & "-" & $((nodeNum mod 2) + 1) & "\",instance=\"" & (hosts[nodeNum div nodesPerHost]) & ".aws-eu-central-1a.nimbus.test\"}"),
-                                  ("[2s]", "[2m]"),
-                                  ("[4s]) * 3", "[2m]) * 120"))
-    outputData["panels"].add(panel)
-    panelIndex.inc()
 
 case outputType:
   of OutputType.local:
-    outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)")
+    outputData["templating"]["list"] = parseJson("""
+      [
+        {
+          "allValue": null,
+          "current": {
+            "tags": [],
+            "text": "0",
+            "value": "0"
+          },
+          "datasource": "Prometheus",
+          "definition": "label_values(process_virtual_memory_bytes,node)",
+          "hide": 0,
+          "includeAll": false,
+          "index": -1,
+          "label": null,
+          "multi": false,
+          "name": "node",
+          "options": [],
+          "query": "label_values(process_virtual_memory_bytes,node)",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "tagValuesQuery": "",
+          "tags": [],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        }
+      ]
+    """)
+  of OutputType.remote:
+    outputData["templating"]["list"] = parseJson("""
+      [
+        {
+          "allValue": null,
+          "current": {
+            "tags": [],
+            "text": "beacon-node-testnet""" & $testnet & """-1",
+            "value": "beacon-node-testnet""" & $testnet & """-1"
+          },
+          "datasource": "master-01.do-ams3.metrics.hq",
+          "definition": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},container)",
+          "hide": 0,
+          "includeAll": false,
+          "index": -1,
+          "label": null,
+          "multi": false,
+          "name": "container",
+          "options": [],
+          "query": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},container)",
+          "refresh": 1,
+          "regex": "/.*testnet""" & $testnet & """.*/",
+          "skipUrlSync": false,
+          "sort": 1,
+          "tagValuesQuery": "",
+          "tags": [],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        },
+        {
+          "allValue": null,
+          "current": {
+            "tags": [],
+            "text": "master-01.aws-eu-central-1a.nimbus.test",
+            "value": "master-01.aws-eu-central-1a.nimbus.test"
+          },
+          "datasource": "master-01.do-ams3.metrics.hq",
+          "definition": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},instance)",
+          "hide": 0,
+          "includeAll": false,
+          "index": -1,
+          "label": null,
+          "multi": false,
+          "name": "instance",
+          "options": [],
+          "query": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},instance)",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 1,
+          "tagValuesQuery": "",
+          "tags": [],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        }
+      ]
+    """)
+
+##########
+# panels #
+##########
+
+outputData["panels"] = %* []
+for panel in panels.mitems:
+  case outputType:
+    of OutputType.local:
+      panel["title"] = %* replace(panel["title"].getStr(), "#0", "#${node}")
+    of OutputType.remote:
+      panel["title"] = %* replace(panel["title"].getStr(), "#0", "#${container}@${instance}")
+      panel["datasource"] = newJNull()
+  if panel.hasKey("targets"):
+    var targets = panel["targets"]
+    for target in targets.mitems:
+      case outputType:
+        of OutputType.local:
+          target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"${node}\"}")
+        of OutputType.remote:
+          # The remote Prometheus instance polls once per minute, so the
+          # minimum rate() interval is 2 minutes.
+          target["expr"] = %* multiReplace(target["expr"].getStr(),
+                                ("{node=\"0\"}", "{job=\"beacon-node-metrics\",container=\"${container}\",instance=\"${instance}\"}"),
+                                ("sum(beacon_attestations_sent_total)", "sum(beacon_attestations_sent_total{job=\"beacon-node-metrics\",container=~\"beacon-node-testnet" & $testnet & "-.\"})"),
+                                ("[2s]", "[2m]"),
+                                ("[4s]) * 3", "[2m]) * 120"))
+  outputData["panels"].add(panel)
+
+########
+# misc #
+########
+
+case outputType:
+  of OutputType.local:
+    outputData["title"] = %* "NBC local testnet/sim (all nodes)"
     outputData["uid"] = %* (outputData["uid"].getStr() & "a")
   of OutputType.remote:
     outputData["title"] = %* ("Nimbus testnet" & $testnet)
     outputData["uid"] = %* (outputData["uid"].getStr() & $testnet)
+    # our annotations only work with a 1s resolution
+    var annotation = outputData["annotations"]["list"][0].copy()
+    annotation["datasource"] = %* "-- Grafana --"
+    outputData["annotations"]["list"] = %* [annotation]
+
 writeFile(outputFilename, pretty(outputData))
 

tests/simulation/start.sh

@@ -167,7 +167,6 @@ fi
 # use the exported Grafana dashboard for a single node to create one for all nodes
 echo Creating grafana dashboards...
 ./build/process_dashboard \
-  --nodes=${TOTAL_NODES} \
   --in="${SIM_ROOT}/beacon-chain-sim-node0-Grafana-dashboard.json" \
   --out="${SIM_ROOT}/beacon-chain-sim-all-nodes-Grafana-dashboard.json"