diff --git a/apps/networkmonitor/networkmonitor.nim b/apps/networkmonitor/networkmonitor.nim index 38591efae..c8c898625 100644 --- a/apps/networkmonitor/networkmonitor.nim +++ b/apps/networkmonitor/networkmonitor.nim @@ -1,9 +1,8 @@ {.push raises: [].} import - std/[tables, strutils, times, sequtils, random], + std/[net, tables, strutils, times, sequtils, random], results, - stew/shims/net, chronicles, chronicles/topics_registry, chronos, @@ -40,7 +39,7 @@ logScope: const ReconnectTime = 60 const MaxConnectionRetries = 5 const ResetRetriesAfter = 1200 -const AvgPingWindow = 10.0 +const PingSmoothing = 0.3 const MaxConnectedPeers = 150 const git_version* {.strdefine.} = "n/a" @@ -55,6 +54,23 @@ proc setDiscoveredPeersCapabilities(routingTableNodes: seq[waku_enr.Record]) = int64(nOfNodesWithCapability), labelValues = [$capability] ) +proc setDiscoveredPeersCluster(routingTableNodes: seq[Node]) = + var clusters: CountTable[uint16] + + for node in routingTableNodes: + let typedRec = node.record.toTyped().valueOr: + clusters.inc(0) + continue + + let relayShard = typedRec.relaySharding().valueOr: + clusters.inc(0) + continue + + clusters.inc(relayShard.clusterId) + + for (key, value) in clusters.pairs: + networkmonitor_peer_cluster_as_per_enr.set(int64(value), labelValues = [$key]) + proc analyzePeer( customPeerInfo: CustomPeerInfoRef, peerInfo: RemotePeerInfo, @@ -87,16 +103,17 @@ proc analyzePeer( info "successfully pinged peer", peer = peerInfo, duration = pingDelay.millis networkmonitor_peer_ping.observe(pingDelay.millis) - if customPeerInfo.avgPingDuration == 0.millis: - customPeerInfo.avgPingDuration = pingDelay + # We are using a smoothed moving average + customPeerInfo.avgPingDuration = + if customPeerInfo.avgPingDuration.millis == 0: + pingDelay + else: + let newAvg = + (float64(pingDelay.millis) * PingSmoothing) + + float64(customPeerInfo.avgPingDuration.millis) * (1.0 - PingSmoothing) + + int64(newAvg).millis - # TODO: check why the calculation ends up losing precision - customPeerInfo.avgPingDuration = int64( - ( - float64(customPeerInfo.avgPingDuration.millis) * (AvgPingWindow - 1.0) + - float64(pingDelay.millis) - ) / AvgPingWindow - ).millis customPeerInfo.lastPingDuration = pingDelay return ok(customPeerInfo.peerId) @@ -306,6 +323,9 @@ proc crawlNetwork( # populate metrics related to capabilities as advertised by the ENR (see waku field) setDiscoveredPeersCapabilities(discoveredNodes) + # populate cluster metrics as advertised by the ENR + setDiscoveredPeersCluster(flatNodes) + # tries to connect to all newly discovered nodes # and populates metrics related to peers we could connect # note random discovered nodes can be already known @@ -589,7 +609,7 @@ when isMainModule: waitFor node.mountRelay() waitFor node.mountLibp2pPing() - if conf.rlnRelayEthContractAddress != "": + if conf.rlnRelay and conf.rlnRelayEthContractAddress != "": let rlnConf = WakuRlnConfig( rlnRelayDynamic: conf.rlnRelayDynamic, rlnRelayCredIndex: some(uint(0)), diff --git a/apps/networkmonitor/networkmonitor_metrics.nim b/apps/networkmonitor/networkmonitor_metrics.nim index 90d72e0ed..cbea7a16a 100644 --- a/apps/networkmonitor/networkmonitor_metrics.nim +++ b/apps/networkmonitor/networkmonitor_metrics.nim @@ -1,7 +1,7 @@ {.push raises: [].} import - std/[json, tables, sequtils], + std/[net, json, tables, sequtils], chronicles, chronicles/topics_registry, chronos, @@ -10,8 +10,7 @@ import metrics/chronos_httpserver, presto/route, presto/server, - results, - stew/shims/net + results logScope: topics = "networkmonitor_metrics" @@ -26,6 +25,9 @@ declarePublicGauge networkmonitor_peer_type_as_per_enr, "Number of peers supporting each capability according to the ENR", labels = ["capability"] +declarePublicGauge networkmonitor_peer_cluster_as_per_enr, + "Number of peers on each cluster according to the ENR", labels = ["cluster"] + declarePublicGauge networkmonitor_peer_type_as_per_protocol, "Number of peers supporting each protocol, after a successful connection) ", labels = ["protocols"] diff --git a/metrics/waku-network-monitor-discovery.json b/metrics/waku-network-monitor-discovery.json new file mode 100644 index 000000000..7716c58f3 --- /dev/null +++ b/metrics/waku-network-monitor-discovery.json @@ -0,0 +1,789 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "networkmonitor_peer_count{connected=\"false\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Unconnected", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "networkmonitor_peer_count{connected=\"true\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Connected", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Discovered Peers", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "description": "Ratio of peers supporting each protocol", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/filter-subscribe/2.0.0-beta1\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Filter", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/lightpush/2.0.0-beta1\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Light Push", + "range": false, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/peer-exchange/2.0.0-alpha1\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Peer Exchange", + "range": false, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/relay/2.0.0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Relay", + "range": false, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/store-query/3.0.0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Store v3", + "range": false, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_protocol{protocols=\"/vac/waku/store/2.0.0-beta4\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Store v2", + "range": false, + "refId": "F", + "useBackend": false + } + ], + "title": "Network Protocol Support", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "discovery_session_lru_cache_hits_total", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Hits", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "discovery_session_lru_cache_misses_total", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Miss", + "range": false, + "refId": "B", + "useBackend": false + } + ], + "title": "Discovery Session LRU Cache", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "routing_table_nodes{state!=\"seen\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "routing_table_nodes{state=\"seen\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Has connected", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Routing Table", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"1\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Cluster 1", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"16\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Cluster 16 (Status)", + "range": false, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_cluster_as_per_enr{cluster=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Cluster 0", + "range": false, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_cluster_as_per_enr{cluster!~\"0|16|1\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Others", + "range": false, + "refId": "D", + "useBackend": false + } + ], + "title": "Clusters as per ENR", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "description": "Check all the ENRs in the network.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_enr{capability=\"Filter\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Filter", + "range": false, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_enr{capability=\"Lightpush\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Light Push", + "range": false, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_enr{capability=\"Relay\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Relay", + "range": false, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "bdspz2dtm1clce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "networkmonitor_peer_type_as_per_enr{capability=\"Store\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Store", + "range": false, + "refId": "D", + "useBackend": false + } + ], + "title": "Capabilities as per ENR", + "transparent": true, + "type": "piechart" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Network Monitor Discovery", + "uid": "adsq0j7yxz2f4e", + "version": 9, + "weekStart": "" +} \ No newline at end of file