Add metrics configuration to start_network and remove shell script
This commit is contained in:
kdeme
zah
parent
a03d0be31f
commit
7e92897b50
|
|
@ -54,16 +54,13 @@ Example of a quick simulation using this approach:
|
|||
# Build wakunode + quicksim
|
||||
make NIMFLAGS="-d:insecure" wakusim
|
||||
|
||||
# Start the simulation nodes
|
||||
./waku/start_network.sh
|
||||
# Or when multitail is installed
|
||||
USE_MULTITAIL="yes" ./waku/start_network.sh
|
||||
|
||||
# In another shell
|
||||
# Start the simulation nodes, this currently requires multitail to be installed
|
||||
./build/start_network --topology:FullMesh --amount:6 --test-node-peers:2
|
||||
# In another shell run
|
||||
./build/quicksim
|
||||
```
|
||||
|
||||
The `start_network.sh` script will also provide a `prometheus.yml` with targets
|
||||
The `start_network` tool will also provide a `prometheus.yml` with targets
|
||||
set to all simulation nodes that are started. This way you can easily start
|
||||
prometheus with this config, e.g.:
|
||||
|
||||
|
|
@ -74,4 +71,4 @@ prometheus
|
|||
|
||||
A Grafana dashboard containing the example dashboard for each simulation node
|
||||
is also generated and can be imported in case you have Grafana running.
|
||||
This dashboard can be found at `./waku/metrics/waku-sim-all-nodes-grafana-dashboard.json`
|
||||
This dashboard can be found at `./waku/metrics/waku-sim-all-nodes-grafana-dashboard.json`
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
# copy from https://github.com/status-im/nim-beacon-chain/blob/master/tests/simulation/process_dashboard.nim
|
||||
import json, parseopt, strutils
|
||||
|
||||
# usage: process_dashboard --nodes=2 --in=node0_dashboard.json --out=all_nodes_dashboard.json
|
||||
var
|
||||
p = initOptParser()
|
||||
nodes: int
|
||||
inputFileName, outputFilename: string
|
||||
|
||||
while true:
|
||||
p.next()
|
||||
case p.kind:
|
||||
of cmdEnd:
|
||||
break
|
||||
of cmdShortOption, cmdLongOption:
|
||||
if p.key == "nodes":
|
||||
nodes = p.val.parseInt()
|
||||
elif p.key == "in":
|
||||
inputFileName = p.val
|
||||
elif p.key == "out":
|
||||
outputFileName = p.val
|
||||
else:
|
||||
echo "unsupported argument: ", p.key
|
||||
of cmdArgument:
|
||||
echo "unsupported argument: ", p.key
|
||||
|
||||
var
|
||||
inputData = parseFile(inputFileName)
|
||||
panels = inputData["panels"].copy()
|
||||
numPanels = len(panels)
|
||||
gridHeight = 0
|
||||
outputData = inputData
|
||||
|
||||
for panel in panels:
|
||||
if panel["gridPos"]["x"].getInt() == 0:
|
||||
gridHeight += panel["gridPos"]["h"].getInt()
|
||||
|
||||
outputData["panels"] = %* []
|
||||
for nodeNum in 0 .. (nodes - 1):
|
||||
var
|
||||
nodePanels = panels.copy()
|
||||
panelIndex = 0
|
||||
for panel in nodePanels.mitems:
|
||||
panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum)
|
||||
panel["id"] = %* (panelIndex + (nodeNum * numPanels))
|
||||
panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight))
|
||||
var targets = panel["targets"]
|
||||
for target in targets.mitems:
|
||||
target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}")
|
||||
outputData["panels"].add(panel)
|
||||
panelIndex.inc()
|
||||
|
||||
outputData["uid"] = %* (outputData["uid"].getStr() & "a")
|
||||
outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)")
|
||||
writeFile(outputFilename, pretty(outputData))
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
import
|
||||
strformat, osproc, net, confutils, strformat, chronicles,
|
||||
strformat, os, osproc, net, confutils, strformat, chronicles, json, strutils,
|
||||
eth/keys, eth/p2p/enode
|
||||
|
||||
const
|
||||
defaults ="--log-level:DEBUG --log-metrics --metrics-server --rpc"
|
||||
wakuNodeBin = "./build/wakunode"
|
||||
wakuNodeBin = "build" / "wakunode"
|
||||
metricsDir = "waku" / "metrics"
|
||||
portOffset = 2
|
||||
|
||||
type
|
||||
|
|
@ -38,6 +39,7 @@ type
|
|||
cmd: string
|
||||
master: bool
|
||||
enode: string
|
||||
shift: int
|
||||
label: string
|
||||
|
||||
proc initNodeCmd(nodeType: NodeType, shift: int, staticNodes: seq[string] = @[],
|
||||
|
|
@ -66,6 +68,7 @@ proc initNodeCmd(nodeType: NodeType, shift: int, staticNodes: seq[string] = @[],
|
|||
|
||||
result.master = master
|
||||
result.enode = $enode
|
||||
result.shift = shift
|
||||
result.label = label
|
||||
|
||||
debug "Node command created.", cmd=result.cmd
|
||||
|
|
@ -95,37 +98,97 @@ proc discoveryNetwork(amount: int): seq[NodeInfo] =
|
|||
result.add(initNodeCmd(FullNode, portOffset + i, label = "full node",
|
||||
discovery = true, bootNodes = @[bootNode.enode]))
|
||||
|
||||
let conf = WakuNetworkConf.load()
|
||||
proc generatePrometheusConfig(nodes: seq[NodeInfo], outputFile: string) =
|
||||
var config = """
|
||||
global:
|
||||
scrape_interval: 1s
|
||||
|
||||
var nodes: seq[NodeInfo]
|
||||
case conf.topology:
|
||||
of Star:
|
||||
nodes = starNetwork(conf.amount)
|
||||
of FullMesh:
|
||||
nodes = fullMeshNetwork(conf.amount)
|
||||
of DiscoveryBased:
|
||||
nodes = discoveryNetwork(conf.amount)
|
||||
scrape_configs:
|
||||
- job_name: "wakusim"
|
||||
static_configs:"""
|
||||
var count = 0
|
||||
for node in nodes:
|
||||
let port = 8008 + node.shift
|
||||
config &= &"""
|
||||
|
||||
var staticnodes: seq[string]
|
||||
for i in 0..<conf.testNodePeers:
|
||||
staticnodes.add(nodes[i].enode)
|
||||
# Waku light node
|
||||
nodes.add(initNodeCmd(WakuNode, 0, staticnodes, label = "light Waku node"))
|
||||
# Regular light node
|
||||
nodes.add(initNodeCmd(LightNode, 1, staticnodes, label = "light node"))
|
||||
- targets: ['127.0.0.1:{port}']
|
||||
labels:
|
||||
node: '{count}'"""
|
||||
count += 1
|
||||
|
||||
var commandStr = "multitail -s 2 -M 0 -x \"Waku Simulation\""
|
||||
var count = 0
|
||||
var sleepDuration = 0
|
||||
for node in nodes:
|
||||
if conf.topology in {Star, DiscoveryBased}:
|
||||
sleepDuration = if node.master: 0
|
||||
else: 1
|
||||
commandStr &= &" -cT ansi -t 'node #{count} {node.label}' -l 'sleep {sleepDuration}; {node.cmd}; echo [node execution completed]; while true; do sleep 100; done'"
|
||||
if conf.topology == FullMesh:
|
||||
sleepDuration += 1
|
||||
count += 1
|
||||
writeFile(outputFile, config)
|
||||
|
||||
let errorCode = execCmd(commandStr)
|
||||
if errorCode != 0:
|
||||
error "launch command failed", command=commandStr
|
||||
proc proccessGrafanaDashboard(nodes: int, inputFile: string,
|
||||
outputFile: string) =
|
||||
# from https://github.com/status-im/nim-beacon-chain/blob/master/tests/simulation/process_dashboard.nim
|
||||
var
|
||||
inputData = parseFile(inputFile)
|
||||
panels = inputData["panels"].copy()
|
||||
numPanels = len(panels)
|
||||
gridHeight = 0
|
||||
outputData = inputData
|
||||
|
||||
for panel in panels:
|
||||
if panel["gridPos"]["x"].getInt() == 0:
|
||||
gridHeight += panel["gridPos"]["h"].getInt()
|
||||
|
||||
outputData["panels"] = %* []
|
||||
for nodeNum in 0 .. (nodes - 1):
|
||||
var
|
||||
nodePanels = panels.copy()
|
||||
panelIndex = 0
|
||||
for panel in nodePanels.mitems:
|
||||
panel["title"] = %* replace(panel["title"].getStr(), "#0", "#" & $nodeNum)
|
||||
panel["id"] = %* (panelIndex + (nodeNum * numPanels))
|
||||
panel["gridPos"]["y"] = %* (panel["gridPos"]["y"].getInt() + (nodeNum * gridHeight))
|
||||
var targets = panel["targets"]
|
||||
for target in targets.mitems:
|
||||
target["expr"] = %* replace(target["expr"].getStr(), "{node=\"0\"}", "{node=\"" & $nodeNum & "\"}")
|
||||
outputData["panels"].add(panel)
|
||||
panelIndex.inc()
|
||||
|
||||
outputData["uid"] = %* (outputData["uid"].getStr() & "a")
|
||||
outputData["title"] = %* (outputData["title"].getStr() & " (all nodes)")
|
||||
writeFile(outputFile, pretty(outputData))
|
||||
|
||||
when isMainModule:
|
||||
let conf = WakuNetworkConf.load()
|
||||
|
||||
var nodes: seq[NodeInfo]
|
||||
case conf.topology:
|
||||
of Star:
|
||||
nodes = starNetwork(conf.amount)
|
||||
of FullMesh:
|
||||
nodes = fullMeshNetwork(conf.amount)
|
||||
of DiscoveryBased:
|
||||
nodes = discoveryNetwork(conf.amount)
|
||||
|
||||
var staticnodes: seq[string]
|
||||
for i in 0..<conf.testNodePeers:
|
||||
# TODO: could also select nodes randomly
|
||||
staticnodes.add(nodes[i].enode)
|
||||
# Waku light node
|
||||
nodes.add(initNodeCmd(WakuNode, 0, staticnodes, label = "light Waku node"))
|
||||
# Regular light node
|
||||
nodes.add(initNodeCmd(LightNode, 1, staticnodes, label = "light node"))
|
||||
|
||||
var commandStr = "multitail -s 2 -M 0 -x \"Waku Simulation\""
|
||||
var count = 0
|
||||
var sleepDuration = 0
|
||||
for node in nodes:
|
||||
if conf.topology in {Star, DiscoveryBased}:
|
||||
sleepDuration = if node.master: 0
|
||||
else: 1
|
||||
commandStr &= &" -cT ansi -t 'node #{count} {node.label}' -l 'sleep {sleepDuration}; {node.cmd}; echo [node execution completed]; while true; do sleep 100; done'"
|
||||
if conf.topology == FullMesh:
|
||||
sleepDuration += 1
|
||||
count += 1
|
||||
|
||||
generatePrometheusConfig(nodes, metricsDir / "prometheus" / "prometheus.yml")
|
||||
proccessGrafanaDashboard(nodes.len,
|
||||
"waku" / "examples" / "waku-grafana-dashboard.json",
|
||||
metricsDir / "waku-sim-all-nodes-grafana-dashboard.json")
|
||||
|
||||
let errorCode = execCmd(commandStr)
|
||||
if errorCode != 0:
|
||||
error "launch command failed", command=commandStr
|
||||
|
|
|
|||
|
|
@ -1,99 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# TODO: improve this bin location
|
||||
WAKU_NODE_BIN="./build/wakunode"
|
||||
NODE_PK="5dc5381cae54ba3174dc0d46040fe11614d0cc94d41185922585198b4fcef9d3"
|
||||
NODE_ENODE="enode://e5fd642a0f630bbb1e4cd7df629d7b8b019457a9a74f983c0484a045cebb176def86a54185b50bbba6bbf97779173695e92835d63109c23471e6da382f922fdb@0.0.0.0:30305"
|
||||
DEFAULTS="--log-level:DEBUG --discovery:off --log-metrics --metrics-server"
|
||||
LIGHT_NODE="--light-node:1"
|
||||
WAKU_LIGHT_NODE="--waku-mode:WakuChan ${LIGHT_NODE}"
|
||||
METRICS_DIR="./waku/metrics"
|
||||
|
||||
# multitail support
|
||||
MULTITAIL="${MULTITAIL:-multitail}" # to allow overriding the program name
|
||||
USE_MULTITAIL="${USE_MULTITAIL:-no}" # make it an opt-in
|
||||
type "$MULTITAIL" &>/dev/null || USE_MULTITAIL="no"
|
||||
|
||||
# TODO: This is based on the nim-beacon-chain eth2_network_simulation but quite
|
||||
# the much worse version due to the custom nodes we need. Need to rework this
|
||||
# to be less hardcoded, but might use a Nim application for this, hence the
|
||||
# quick and dirty way for now.
|
||||
|
||||
mkdir -p "${METRICS_DIR}"/prometheus/
|
||||
cat > "${METRICS_DIR}/prometheus/prometheus.yml" <<EOF
|
||||
global:
|
||||
scrape_interval: 1s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "wakusim"
|
||||
static_configs:
|
||||
- targets: ['127.0.0.1:8008']
|
||||
labels:
|
||||
node: '0'
|
||||
- targets: ['127.0.0.1:8009']
|
||||
labels:
|
||||
node: '1'
|
||||
- targets: ['127.0.0.1:8010']
|
||||
labels:
|
||||
node: '2'
|
||||
- targets: ['127.0.0.1:8011']
|
||||
labels:
|
||||
node: '3'
|
||||
- targets: ['127.0.0.1:8012']
|
||||
labels:
|
||||
node: '4'
|
||||
EOF
|
||||
|
||||
# use the exported Grafana dashboard for a single node to create one for all nodes
|
||||
"./build/process_dashboard" \
|
||||
--nodes=5 \
|
||||
--in="waku/examples/waku-grafana-dashboard.json" \
|
||||
--out="${METRICS_DIR}/waku-sim-all-nodes-grafana-dashboard.json"
|
||||
|
||||
if [[ "$USE_MULTITAIL" != "no" ]]; then
|
||||
SLEEP=0
|
||||
# Direct connect with staticnodes, simple star topology for now
|
||||
# Master node to connect to
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --nodekey:${NODE_PK} --ports-shift:2 --waku-mode:WakuSan"
|
||||
COMMANDS+=( " -cT ansi -t 'master node' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
|
||||
|
||||
SLEEP=1
|
||||
# Node under test 1: light waku node (topics)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:0 ${WAKU_LIGHT_NODE}"
|
||||
COMMANDS+=( " -cT ansi -t 'light waku node' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
|
||||
# Node under test 2: light node (bloomfilter)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:1 ${LIGHT_NODE}"
|
||||
COMMANDS+=( " -cT ansi -t 'light node' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
|
||||
# Node under test 3: full node
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:3"
|
||||
COMMANDS+=( " -cT ansi -t 'full node' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
|
||||
# Traffic generation node(s)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:4"
|
||||
COMMANDS+=( " -cT ansi -t 'traffic full node' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
|
||||
else
|
||||
# Direct connect with staticnodes, simple star topology for now
|
||||
# Master node to connect to
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --nodekey:${NODE_PK} --waku-mode:WakuSan"
|
||||
eval ${CMD} &
|
||||
sleep 1
|
||||
# Node under test 1: light waku node (topics)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:1 ${WAKU_LIGHT_NODE}"
|
||||
eval ${CMD} &
|
||||
# Node under test 2: light node (bloomfilter)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:2 ${LIGHT_NODE}"
|
||||
eval ${CMD} &
|
||||
# Node under test 3: full node
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:3"
|
||||
eval ${CMD} &
|
||||
# Traffic generation node(s)
|
||||
CMD="$WAKU_NODE_BIN $DEFAULTS --rpc --staticnodes:${NODE_ENODE} --ports-shift:4"
|
||||
eval ${CMD} &
|
||||
fi
|
||||
|
||||
if [[ "$USE_MULTITAIL" != "no" ]]; then
|
||||
eval $MULTITAIL -s 2 -M 0 -x \"Waku Simulation\" "${COMMANDS[@]}"
|
||||
else
|
||||
wait # Stop when all nodes have gone down
|
||||
fi
|
||||
Loading…
Reference in New Issue