waku-org
/
nwaku
mirror of https://github.com/waku-org/nwaku.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use metrics with labels, fix #24 (#27)

This commit is contained in:
Kim De Mey 2020-06-09 13:44:39 +02:00 committed by GitHub
parent 89ced4253c
commit 37d8720d52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 168 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

290
metrics/waku-grafana-dashboard.json
View File

@ -15,28 +15,23 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 8,
"id": 2,
"links": [],
"panels": [
{
"datasource": null,
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 16,
"options": {
"fieldOptions": {
"calcs": [
"last"
],
"defaults": {
"mappings": [],
"max": 100,
"min": 0,
"thresholds": [
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
@ -46,15 +41,30 @@
"value": 80
}
]
},
"override": {},
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 16,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"orientation": "auto",
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "6.4.5",
"pluginVersion": "7.0.3",
"targets": [
{
"expr": "connected_peers{node=\"0\"}",
@ -76,6 +86,12 @@
"#d44a3a"
],
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"format": "none",
"gauge": {
"maxValue": 100,
@ -107,7 +123,6 @@
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
@ -127,10 +142,12 @@
"ymax": null,
"ymin": null
},
"tableColumn": "",
"tableColumn": "envelopes_valid_total{instance=\"127.0.0.1:8010\", job=\"wakusim\", node=\"0\"}",
"targets": [
{
"expr": "valid_envelopes_total{node=\"0\"}",
"expr": "envelopes_valid_total{node=\"0\"}",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
@ -159,6 +176,12 @@
"#d44a3a"
],
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"format": "none",
"gauge": {
"maxValue": 100,
@ -190,7 +213,6 @@
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"pluginVersion": "6.4.5",
"postfix": "",
"postfixFontSize": "50%",
@ -211,18 +233,19 @@
"ymax": null,
"ymin": null
},
"tableColumn": "",
"tableColumn": "Dropped envelopes",
"targets": [
{
"expr": "dropped_expired_envelopes_total{node=\"0\"} + dropped_from_future_envelopes_total{node=\"0\"} + dropped_low_pow_envelopes_total{node=\"0\"} + dropped_too_large_envelopes_total{node=\"0\"} + dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"} + dropped_topic_mismatch_envelopes_total{node=\"0\"} +dropped_benign_duplicate_envelopes_total{node=\"0\"} + dropped_duplicate_envelopes_total{node=\"0\"}",
"legendFormat": "Invalid envelopes",
"expr": "sum(envelopes_dropped_total{node=\"0\"})",
"interval": "",
"legendFormat": "Dropped envelopes",
"refId": "A"
}
],
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Invalid Envelopes #0",
"title": "Dropped Envelopes #0",
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
@ -236,23 +259,18 @@
},
{
"datasource": null,
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 0
},
"id": 14,
"options": {
"fieldOptions": {
"calcs": [
"lastNotNull"
],
"defaults": {
"mappings": [],
"max": 200,
"min": 0,
"thresholds": [
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"max": 200,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
@ -261,17 +279,32 @@
"color": "red",
"value": 200
}
],
"unit": "percent"
]
},
"override": {},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 0
},
"id": 14,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "6.4.5",
"pluginVersion": "7.0.3",
"targets": [
{
"expr": "rate(process_cpu_seconds_total{node=\"0\"}[5s]) * 100",
@ -286,23 +319,18 @@
},
{
"datasource": null,
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"y": 0
},
"id": 18,
"options": {
"fieldOptions": {
"calcs": [
"lastNotNull"
],
"defaults": {
"mappings": [],
"max": 2147483648,
"min": 0,
"thresholds": [
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"max": 2147483648,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
@ -311,17 +339,32 @@
"color": "red",
"value": 2147483648
}
],
"unit": "bytes"
]
},
"override": {},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"y": 0
},
"id": 18,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "6.4.5",
"pluginVersion": "7.0.3",
"targets": [
{
"expr": "process_resident_memory_bytes{node=\"0\"}",
@ -339,6 +382,12 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
@ -347,6 +396,7 @@
"x": 0,
"y": 4
},
"hiddenSeries": false,
"id": 6,
"legend": {
"avg": false,
@ -374,72 +424,20 @@
"steppedLine": false,
"targets": [
{
"expr": "valid_envelopes_total{node=\"0\"}",
"expr": "envelopes_valid_total{node=\"0\"}",
"hide": false,
"instant": false,
"interval": "",
"legendFormat": "Valid",
"refId": "A"
},
{
"expr": "dropped_benign_duplicate_envelopes_total{node=\"0\"}",
"expr": "envelopes_dropped_total{node=\"0\"}",
"hide": false,
"instant": false,
"legendFormat": "Benign duplicate",
"interval": "",
"legendFormat": "Dropped {{reason}}",
"refId": "B"
},
{
"expr": "dropped_duplicate_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Duplicate",
"refId": "C"
},
{
"expr": "dropped_expired_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Expired",
"refId": "D"
},
{
"expr": "dropped_from_future_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Future timestamped",
"refId": "E"
},
{
"expr": "dropped_low_pow_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Too low PoW",
"refId": "F"
},
{
"expr": "dropped_bloom_filter_mismatch_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Bloom filter mismatch",
"refId": "G"
},
{
"expr": "dropped_topic_mismatch_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Topic mismatch",
"refId": "H"
},
{
"expr": "dropped_too_large_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Too Large",
"refId": "I"
},
{
"expr": "dropped_full_queue_new_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Full queue new",
"refId": "J"
},
{
"expr": "dropped_full_queue_old_envelopes_total{node=\"0\"}",
"hide": false,
"legendFormat": "Full queue old",
"refId": "K"
}
],
"thresholds": [],
@ -489,6 +487,12 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
@ -497,6 +501,7 @@
"x": 12,
"y": 4
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
@ -593,6 +598,12 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
@ -601,6 +612,7 @@
"x": 0,
"y": 13
},
"hiddenSeries": false,
"id": 8,
"legend": {
"avg": false,
@ -684,6 +696,12 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
@ -692,6 +710,7 @@
"x": 12,
"y": 13
},
"hiddenSeries": false,
"id": 4,
"legend": {
"avg": false,
@ -781,7 +800,7 @@
}
],
"refresh": "5s",
"schemaVersion": 20,
"schemaVersion": 25,
"style": "dark",
"tags": [],
"templating": {
@ -793,7 +812,6 @@
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
@ -806,7 +824,7 @@
]
},
"timezone": "",
"title": "Waku Node",
"title": "Waku Node2",
"uid": "K7Z6IoBZk",
"version": 2
}
"version": 5
}

7
tests/v1/test_rpc_waku.nim
View File

@ -1,15 +1,14 @@
import
unittest, strformat, options, stew/byteutils, json_rpc/[rpcserver, rpcclient],
unittest, strformat, options, os, stew/byteutils,
json_rpc/[rpcserver, rpcclient],
eth/common as eth_common, eth/[rlp, keys, p2p],
../../waku/protocol/v1/waku_protocol,
../../waku/node/v1/rpc/[hexstrings, rpc_types, waku, key_storage],
./test_helpers
from os import DirSep, ParDir
## Generate client convenience marshalling wrappers from forward declarations
## For testing, ethcallsigs needs to be kept in sync with ../waku/node/v1/rpc/waku
const sigPath = &"{sourceDir}{DirSep}{ParDir}{DirSep}{ParDir}{DirSep}waku{DirSep}node{DirSep}v1{DirSep}rpc{DirSep}wakucallsigs.nim"
const sigPath = sourceDir / ParDir / ParDir / "waku" / "node" / "v1" / "rpc" / "wakucallsigs.nim"
createRpcSigs(RpcSocketClient, sigPath)
proc setupNode(capabilities: varargs[ProtocolInfo, `protocolInfo`]): EthereumNode =

2
vendor/nim-eth vendored

@ -1 +1 @@
Subproject commit a110f091af38e070781de28fea400303d5b3cf43
Subproject commit be9a87848e068d68aa8fa1a7bfa07d7c7271eba7

2
vendor/nim-stew vendored

@ -1 +1 @@
Subproject commit a99dafab420bcbbffee35e9bd847a9014eafaffe
Subproject commit cf82e2d51d12c3ca461cb170c2a4e2eada3bfe67

15
waku/node/v1/wakunode.nim
View File

@ -128,18 +128,11 @@ proc run(config: WakuNodeConf) =
proc logMetrics(udata: pointer) {.closure, gcsafe.} =
{.gcsafe.}:
let
connectedPeers = connected_peers.value
validEnvelopes = waku_protocol.valid_envelopes.value
invalidEnvelopes = waku_protocol.dropped_expired_envelopes.value +
waku_protocol.dropped_from_future_envelopes.value +
waku_protocol.dropped_low_pow_envelopes.value +
waku_protocol.dropped_too_large_envelopes.value +
waku_protocol.dropped_bloom_filter_mismatch_envelopes.value +
waku_protocol.dropped_topic_mismatch_envelopes.value +
waku_protocol.dropped_benign_duplicate_envelopes.value +
waku_protocol.dropped_duplicate_envelopes.value
connectedPeers = connected_peers
validEnvelopes = waku_protocol.envelopes_valid
droppedEnvelopes = waku_protocol.envelopes_dropped
info "Node metrics", connectedPeers, validEnvelopes, invalidEnvelopes
info "Node metrics", connectedPeers, validEnvelopes, droppedEnvelopes
addTimer(Moment.fromNow(2.seconds), logMetrics)
addTimer(Moment.fromNow(2.seconds), logMetrics)

21
waku/protocol/v1/waku_protocol.nim
View File

@ -48,17 +48,6 @@ export
logScope:
topics = "waku"
declarePublicCounter dropped_low_pow_envelopes,
"Dropped envelopes because of too low PoW"
declarePublicCounter dropped_too_large_envelopes,
"Dropped envelopes because larger than maximum allowed size"
declarePublicCounter dropped_bloom_filter_mismatch_envelopes,
"Dropped envelopes because not matching with bloom filter"
declarePublicCounter dropped_topic_mismatch_envelopes,
"Dropped envelopes because of not matching topics"
declarePublicCounter dropped_duplicate_envelopes,
"Dropped duplicate envelopes"
const
defaultQueueCapacity = 2048
wakuVersion* = 1 ## Waku version.
@ -191,23 +180,23 @@ proc allowed*(msg: Message, config: WakuConfig): bool =
# Check max msg size, already happens in RLPx but there is a specific waku
# max msg size which should always be < RLPx max msg size
if msg.size > config.maxMsgSize:
dropped_too_large_envelopes.inc()
envelopes_dropped.inc(labelValues = ["too_large"])
warn "Message size too large", size = msg.size
return false
if msg.pow < config.powRequirement:
dropped_low_pow_envelopes.inc()
envelopes_dropped.inc(labelValues = ["low_pow"])
warn "Message PoW too low", pow = msg.pow, minPow = config.powRequirement
return false
if config.topics.isSome():
if msg.env.topic notin config.topics.get():
dropped_topic_mismatch_envelopes.inc()
envelopes_dropped.inc(labelValues = ["topic_mismatch"])
warn "Message topic does not match Waku topic list"
return false
else:
if config.bloom.isSome() and not bloomFilterMatch(config.bloom.get(), msg.bloom):
dropped_bloom_filter_mismatch_envelopes.inc()
envelopes_dropped.inc(labelValues = ["bloom_filter_mismatch"])
warn "Message does not match node bloom filter"
return false
@ -314,7 +303,7 @@ p2pProtocol Waku(version = wakuVersion,
# this node to a peer and that same message arriving from that peer (after
# it was received from another peer) here.
if peer.state.received.containsOrIncl(msg.hash):
dropped_duplicate_envelopes.inc()
envelopes_dropped.inc(labelValues = ["duplicate"])
trace "Peer sending duplicate messages", peer, hash = $msg.hash
# await peer.disconnect(SubprotocolReason)
continue