diff --git a/scripts/launch_local_testnet.sh b/scripts/launch_local_testnet.sh index 27123126a..d274532df 100755 --- a/scripts/launch_local_testnet.sh +++ b/scripts/launch_local_testnet.sh @@ -177,7 +177,6 @@ EOF # use the exported Grafana dashboard for a single node to create one for all nodes ./build/process_dashboard \ - --nodes=${NUM_NODES} \ --in="tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json" \ --out="${DATA_DIR}/local-testnet-all-nodes-Grafana-dashboard.json" fi diff --git a/scripts/reset_testnet.sh b/scripts/reset_testnet.sh index 3a36ef464..a27cfc05f 100755 --- a/scripts/reset_testnet.sh +++ b/scripts/reset_testnet.sh @@ -69,7 +69,6 @@ make -j2 NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" be echo "Generating Grafana dashboards for remote testnet servers" for testnet in 0 1; do ./build/process_dashboard \ - --nodes=20 \ --in="tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json" \ --out="docker/beacon-chain-sim-remote-testnet${testnet}-Grafana-dashboard.json" \ --type="remote" \ diff --git a/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json b/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json index 619163077..da256b59a 100644 --- a/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json +++ b/tests/simulation/beacon-chain-sim-node0-Grafana-dashboard.json @@ -47,22 +47,6 @@ "id": 13, "links": [], "panels": [ - { - "content": "", - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 44, - "mode": "markdown", - "timeFrom": null, - "timeShift": null, - "title": "node #0", - "type": "text" - }, { "aliasColors": {}, "bars": false, @@ -75,7 +59,7 @@ "h": 6, "w": 14, "x": 0, - "y": 1 + "y": 0 }, "hiddenSeries": false, "id": 2, @@ -195,7 +179,7 @@ "h": 5, "w": 10, "x": 14, - "y": 1 + "y": 0 }, "hiddenSeries": false, "id": 16, @@ -289,7 +273,7 @@ "h": 5, "w": 10, "x": 14, - "y": 6 + "y": 5 }, "hiddenSeries": false, "id": 20, @@ -385,7 +369,7 @@ "h": 6, "w": 14, "x": 0, - "y": 7 + "y": 6 }, "hiddenSeries": false, "id": 18, @@ -480,7 +464,7 @@ "h": 5, "w": 10, "x": 14, - "y": 11 + "y": 10 }, "hiddenSeries": false, "id": 24, @@ -569,7 +553,7 @@ "h": 6, "w": 14, "x": 0, - "y": 13 + "y": 12 }, "hiddenSeries": false, "id": 22, @@ -672,7 +656,7 @@ "h": 2, "w": 5, "x": 14, - "y": 16 + "y": 15 }, "id": 6, "interval": null, @@ -755,7 +739,7 @@ "h": 2, "w": 5, "x": 19, - "y": 16 + "y": 15 }, "id": 8, "interval": null, @@ -838,7 +822,7 @@ "h": 2, "w": 3, "x": 14, - "y": 18 + "y": 17 }, "id": 28, "interval": null, @@ -924,7 +908,7 @@ "h": 2, "w": 4, "x": 17, - "y": 18 + "y": 17 }, "id": 13, "interval": null, @@ -1009,7 +993,7 @@ "h": 2, "w": 3, "x": 21, - "y": 18 + "y": 17 }, "id": 14, "interval": null, @@ -1084,7 +1068,7 @@ "h": 6, "w": 14, "x": 0, - "y": 19 + "y": 18 }, "hiddenSeries": false, "id": 38, @@ -1188,7 +1172,7 @@ "h": 2, "w": 3, "x": 14, - "y": 20 + "y": 19 }, "id": 32, "interval": null, @@ -1272,7 +1256,7 @@ "h": 2, "w": 4, "x": 17, - "y": 20 + "y": 19 }, "id": 34, "interval": null, @@ -1357,7 +1341,7 @@ "h": 2, "w": 3, "x": 21, - "y": 20 + "y": 19 }, "id": 40, "interval": null, @@ -1442,7 +1426,7 @@ "h": 2, "w": 3, "x": 14, - "y": 22 + "y": 21 }, "id": 12, "interval": null, @@ -1524,7 +1508,7 @@ "h": 2, "w": 4, "x": 17, - "y": 22 + "y": 21 }, "id": 36, "interval": null, @@ -1598,7 +1582,7 @@ "h": 5, "w": 14, "x": 0, - "y": 25 + "y": 24 }, "hiddenSeries": false, "id": 30, @@ -1700,7 +1684,7 @@ "h": 6, "w": 14, "x": 0, - "y": 30 + "y": 29 }, "heatmap": {}, "hideZeroBuckets": false, @@ -1780,5 +1764,5 @@ "variables": { "list": [] }, - "version": 35 + "version": 38 } diff --git a/tests/simulation/process_dashboard.nim b/tests/simulation/process_dashboard.nim index 76e94d8c0..fb33c4664 100644 --- a/tests/simulation/process_dashboard.nim +++ b/tests/simulation/process_dashboard.nim @@ -1,41 +1,23 @@ import json, parseopt, strutils -# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json --type=local --testnet=0 +# usage: process_dashboard --in=node0_dashboard.json --out=all_nodes_dashboard.json --type=local --testnet=0 type OutputType = enum local remote var p = initOptParser() - nodes: int inputFileName, outputFilename: string outputType = OutputType.local testnet = 0 -let - hosts = [ - "master-01", - "node-01", - "node-02", - "node-03", - "node-04", - "node-05", - "node-06", - "node-07", - "node-08", - "node-09", - ] - nodesPerHost = 2 - while true: p.next() case p.kind: of cmdEnd: break of cmdShortOption, cmdLongOption: - if p.key == "nodes": - nodes = p.val.parseInt() - elif p.key == "in": + if p.key == "in": inputFileName = p.val elif p.key == "out": outputFileName = p.val @@ -51,52 +33,148 @@ while true: var inputData = parseFile(inputFileName) panels = inputData["panels"].copy() - numPanels = len(panels) - gridHeight = 0 outputData = inputData -for panel in panels: - if panel["gridPos"]["x"].getInt() == 0: - gridHeight += panel["gridPos"]["h"].getInt() - -outputData["panels"] = %* [] -if outputType == OutputType.remote: - var annotations = outputData["annotations"]["list"] - for annotation in annotations.mitems: - annotation["datasource"] = %* "-- Grafana --" - -for nodeNum in 0 .. (nodes - 1): - var - nodePanels = panels.copy() - panelIndex = 0 - for panel in nodePanels.mitems: - panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum) - panel["id"] = %* (panelIndex + (nodeNum * numPanels)) - panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight)) - if outputType == OutputType.remote: - panel["datasource"] = newJNull() - if panel.hasKey("targets"): - var targets = panel["targets"] - for target in targets.mitems: - case outputType: - of OutputType.local: - target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}") - of OutputType.remote: - # The remote Prometheus instance polls once per minute, so the - # minimum rate() interval is 2 minutes. - target["expr"] = %* multiReplace(target["expr"].getStr(), - ("{node=\"0\"}", "{container=\"beacon-node-testnet" & $testnet & "-" & $((nodeNum mod 2) + 1) & "\",instance=\"" & (hosts[nodeNum div nodesPerHost]) & ".aws-eu-central-1a.nimbus.test\"}"), - ("[2s]", "[2m]"), - ("[4s]) * 3", "[2m]) * 120")) - outputData["panels"].add(panel) - panelIndex.inc() +############# +# variables # +############# case outputType: of OutputType.local: - outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)") + outputData["templating"]["list"] = parseJson(""" + [ + { + "allValue": null, + "current": { + "tags": [], + "text": "0", + "value": "0" + }, + "datasource": "Prometheus", + "definition": "label_values(process_virtual_memory_bytes,node)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "node", + "options": [], + "query": "label_values(process_virtual_memory_bytes,node)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + """) + of OutputType.remote: + outputData["templating"]["list"] = parseJson(""" + [ + { + "allValue": null, + "current": { + "tags": [], + "text": "beacon-node-testnet""" & $testnet & """-1", + "value": "beacon-node-testnet""" & $testnet & """-1" + }, + "datasource": "master-01.do-ams3.metrics.hq", + "definition": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},container)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "container", + "options": [], + "query": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},container)", + "refresh": 1, + "regex": "/.*testnet""" & $testnet & """.*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "master-01.aws-eu-central-1a.nimbus.test", + "value": "master-01.aws-eu-central-1a.nimbus.test" + }, + "datasource": "master-01.do-ams3.metrics.hq", + "definition": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},instance)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(process_virtual_memory_bytes{job=\"beacon-node-metrics\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + """) + +########## +# panels # +########## + +outputData["panels"] = %* [] +for panel in panels.mitems: + case outputType: + of OutputType.local: + panel["title"] = %* replace(panel["title"].getStr(), "#0", "#${node}") + of OutputType.remote: + panel["title"] = %* replace(panel["title"].getStr(), "#0", "#${container}@${instance}") + panel["datasource"] = newJNull() + if panel.hasKey("targets"): + var targets = panel["targets"] + for target in targets.mitems: + case outputType: + of OutputType.local: + target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"${node}\"}") + of OutputType.remote: + # The remote Prometheus instance polls once per minute, so the + # minimum rate() interval is 2 minutes. + target["expr"] = %* multiReplace(target["expr"].getStr(), + ("{node=\"0\"}", "{job=\"beacon-node-metrics\",container=\"${container}\",instance=\"${instance}\"}"), + ("sum(beacon_attestations_sent_total)", "sum(beacon_attestations_sent_total{job=\"beacon-node-metrics\",container=~\"beacon-node-testnet" & $testnet & "-.\"})"), + ("[2s]", "[2m]"), + ("[4s]) * 3", "[2m]) * 120")) + outputData["panels"].add(panel) + +######## +# misc # +######## + +case outputType: + of OutputType.local: + outputData["title"] = %* "NBC local testnet/sim (all nodes)" outputData["uid"] = %* (outputData["uid"].getStr() & "a") of OutputType.remote: outputData["title"] = %* ("Nimbus testnet" & $testnet) outputData["uid"] = %* (outputData["uid"].getStr() & $testnet) + # our annotations only work with a 1s resolution + var annotation = outputData["annotations"]["list"][0].copy() + annotation["datasource"] = %* "-- Grafana --" + outputData["annotations"]["list"] = %* [annotation] + writeFile(outputFilename, pretty(outputData)) diff --git a/tests/simulation/start.sh b/tests/simulation/start.sh index 0aa09e0f3..cafbb314c 100755 --- a/tests/simulation/start.sh +++ b/tests/simulation/start.sh @@ -167,7 +167,6 @@ fi # use the exported Grafana dashboard for a single node to create one for all nodes echo Creating grafana dashboards... ./build/process_dashboard \ - --nodes=${TOTAL_NODES} \ --in="${SIM_ROOT}/beacon-chain-sim-node0-Grafana-dashboard.json" \ --out="${SIM_ROOT}/beacon-chain-sim-all-nodes-Grafana-dashboard.json"