feat: misc. updates for discovery network analysis (#2930)

added metrics, a way to start without RLN and a new avg latency algorithm
This commit is contained in:
Simon-Pierre Vivier 2024-08-07 14:58:28 -04:00 committed by GitHub
parent b839b1c81f
commit 6b22823b64
3 changed files with 827 additions and 16 deletions

View File

@ -1,9 +1,8 @@
{.push raises: [].}
import
std/[tables, strutils, times, sequtils, random],
std/[net, tables, strutils, times, sequtils, random],
results,
stew/shims/net,
chronicles,
chronicles/topics_registry,
chronos,
@ -40,7 +39,7 @@ logScope:
const ReconnectTime = 60
const MaxConnectionRetries = 5
const ResetRetriesAfter = 1200
const AvgPingWindow = 10.0
const PingSmoothing = 0.3
const MaxConnectedPeers = 150
const git_version* {.strdefine.} = "n/a"
@ -55,6 +54,23 @@ proc setDiscoveredPeersCapabilities(routingTableNodes: seq[waku_enr.Record]) =
int64(nOfNodesWithCapability), labelValues = [$capability]
)
proc setDiscoveredPeersCluster(routingTableNodes: seq[Node]) =
var clusters: CountTable[uint16]
for node in routingTableNodes:
let typedRec = node.record.toTyped().valueOr:
clusters.inc(0)
continue
let relayShard = typedRec.relaySharding().valueOr:
clusters.inc(0)
continue
clusters.inc(relayShard.clusterId)
for (key, value) in clusters.pairs:
networkmonitor_peer_cluster_as_per_enr.set(int64(value), labelValues = [$key])
proc analyzePeer(
customPeerInfo: CustomPeerInfoRef,
peerInfo: RemotePeerInfo,
@ -87,16 +103,17 @@ proc analyzePeer(
info "successfully pinged peer", peer = peerInfo, duration = pingDelay.millis
networkmonitor_peer_ping.observe(pingDelay.millis)
if customPeerInfo.avgPingDuration == 0.millis:
customPeerInfo.avgPingDuration = pingDelay
# We are using a smoothed moving average
customPeerInfo.avgPingDuration =
if customPeerInfo.avgPingDuration.millis == 0:
pingDelay
else:
let newAvg =
(float64(pingDelay.millis) * PingSmoothing) +
float64(customPeerInfo.avgPingDuration.millis) * (1.0 - PingSmoothing)
int64(newAvg).millis
# TODO: check why the calculation ends up losing precision
customPeerInfo.avgPingDuration = int64(
(
float64(customPeerInfo.avgPingDuration.millis) * (AvgPingWindow - 1.0) +
float64(pingDelay.millis)
) / AvgPingWindow
).millis
customPeerInfo.lastPingDuration = pingDelay
return ok(customPeerInfo.peerId)
@ -306,6 +323,9 @@ proc crawlNetwork(
# populate metrics related to capabilities as advertised by the ENR (see waku field)
setDiscoveredPeersCapabilities(discoveredNodes)
# populate cluster metrics as advertised by the ENR
setDiscoveredPeersCluster(flatNodes)
# tries to connect to all newly discovered nodes
# and populates metrics related to peers we could connect
# note random discovered nodes can be already known
@ -589,7 +609,7 @@ when isMainModule:
waitFor node.mountRelay()
waitFor node.mountLibp2pPing()
if conf.rlnRelayEthContractAddress != "":
if conf.rlnRelay and conf.rlnRelayEthContractAddress != "":
let rlnConf = WakuRlnConfig(
rlnRelayDynamic: conf.rlnRelayDynamic,
rlnRelayCredIndex: some(uint(0)),

View File

@ -1,7 +1,7 @@
{.push raises: [].}
import
std/[json, tables, sequtils],
std/[net, json, tables, sequtils],
chronicles,
chronicles/topics_registry,
chronos,
@ -10,8 +10,7 @@ import
metrics/chronos_httpserver,
presto/route,
presto/server,
results,
stew/shims/net
results
logScope:
topics = "networkmonitor_metrics"
@ -26,6 +25,9 @@ declarePublicGauge networkmonitor_peer_type_as_per_enr,
"Number of peers supporting each capability according to the ENR",
labels = ["capability"]
declarePublicGauge networkmonitor_peer_cluster_as_per_enr,
"Number of peers on each cluster according to the ENR", labels = ["cluster"]
declarePublicGauge networkmonitor_peer_type_as_per_protocol,
"Number of peers supporting each protocol, after a successful connection) ",
labels = ["protocols"]

View File

@ -0,0 +1,789 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 3,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "networkmonitor_peer_count{connected=\"false\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Unconnected",
"range": true,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "networkmonitor_peer_count{connected=\"true\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Connected",
"range": true,
"refId": "B",
"useBackend": false
}
],
"title": "Discovered Peers",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"description": "Ratio of peers supporting each protocol",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"displayLabels": [
"value"
],
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/filter-subscribe/2.0.0-beta1\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Filter",
"range": false,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/lightpush/2.0.0-beta1\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Light Push",
"range": false,
"refId": "B",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/peer-exchange/2.0.0-alpha1\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Peer Exchange",
"range": false,
"refId": "C",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/relay/2.0.0\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Relay",
"range": false,
"refId": "D",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/store-query/3.0.0\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Store v3",
"range": false,
"refId": "E",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/store/2.0.0-beta4\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Store v2",
"range": false,
"refId": "F",
"useBackend": false
}
],
"title": "Network Protocol Support",
"transparent": true,
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"displayLabels": [
"value"
],
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "discovery_session_lru_cache_hits_total",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Hits",
"range": false,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "discovery_session_lru_cache_misses_total",
"format": "time_series",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Miss",
"range": false,
"refId": "B",
"useBackend": false
}
],
"title": "Discovery Session LRU Cache",
"transparent": true,
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "routing_table_nodes{state!=\"seen\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Total",
"range": true,
"refId": "B",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"expr": "routing_table_nodes{state=\"seen\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Has connected",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Routing Table",
"transparent": true,
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"displayLabels": [
"value"
],
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"1\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Cluster 1",
"range": false,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"16\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Cluster 16 (Status)",
"range": false,
"refId": "B",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"0\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Cluster 0",
"range": false,
"refId": "C",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_cluster_as_per_enr{cluster!~\"0|16|1\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Others",
"range": false,
"refId": "D",
"useBackend": false
}
],
"title": "Clusters as per ENR",
"transparent": true,
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"description": "Check all the ENRs in the network.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"displayLabels": [
"value"
],
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_enr{capability=\"Filter\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Filter",
"range": false,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_enr{capability=\"Lightpush\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Light Push",
"range": false,
"refId": "B",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_enr{capability=\"Relay\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Relay",
"range": false,
"refId": "C",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "bdspz2dtm1clce"
},
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "networkmonitor_peer_type_as_per_enr{capability=\"Store\"}",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "Store",
"range": false,
"refId": "D",
"useBackend": false
}
],
"title": "Capabilities as per ENR",
"transparent": true,
"type": "piechart"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Network Monitor Discovery",
"uid": "adsq0j7yxz2f4e",
"version": 9,
"weekStart": ""
}