chore: improve sonda dashboard (#2918)

This commit is contained in:
gabrielmer 2024-07-19 14:31:23 +03:00 committed by GitHub
parent 718e54f80d
commit d9a48321d2

View File

@ -18,7 +18,6 @@
"editable": true, "editable": true,
"fiscalYearStartMonth": 0, "fiscalYearStartMonth": 0,
"graphTooltip": 0, "graphTooltip": 0,
"id": 2,
"links": [], "links": [],
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
@ -88,7 +87,7 @@
"x": 0, "x": 0,
"y": 0 "y": 0
}, },
"id": 1, "id": 9,
"options": { "options": {
"legend": { "legend": {
"calcs": [], "calcs": [],
@ -252,7 +251,108 @@
"type": "prometheus", "type": "prometheus",
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"description": "Store responses including the latest Sonda message ", "description": "Sonda messages successfully sent to the network",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "increase(successful_sonda_msgs_total[5m])/5",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Sent Sonda Messages Rate per Minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Sonda messages that failed to be sent to the network",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {
@ -313,7 +413,7 @@
"options": { "options": {
"mode": "exclude", "mode": "exclude",
"names": [ "names": [
"{__name__=\"successful_store_queries_total\" , node=\"/dns4/store-02.gc-us-central1-a.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmJnVR7ZzFaYvciPVafUXuYGLHPzSUigqAmeNw9nJUVGeM\"}" "{instance=\"sonda:8004\", job=\"nwaku\"}"
], ],
"prefix": "All except:", "prefix": "All except:",
"readOnly": true "readOnly": true
@ -335,9 +435,109 @@
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 12,
"y": 8 "y": 8
}, },
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "increase(failed_sonda_msgs_total[5m]) / 5",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Failed Sonda Messages per Minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Store responses including the latest Sonda message ",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 3, "id": 3,
"options": { "options": {
"legend": { "legend": {
@ -358,8 +558,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "successful_store_queries_total", "expr": "successful_store_queries_total{node=~\"^$node$\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -438,7 +638,7 @@
"options": { "options": {
"mode": "exclude", "mode": "exclude",
"names": [ "names": [
"{__name__=\"failed_store_queries_total\" , error=\"300 BAD_RESPONSE: Future operation cancelled!\", node=\"/dns4/store-01.ac-cn-hongkong-c.shards.test.status.im/tcp/30303/p2p/16Uiu2HAm2M7xs7cLPc3jamawkEqbr7cUJX11uvY7LxQ6WFUdUKUT\"}" "{__name__=\"failed_store_queries_total\" , error=\"504 PEER_DIAL_FAILURE: 16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\", node=\"/dns4/store-01.do-ams3.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\"}"
], ],
"prefix": "All except:", "prefix": "All except:",
"readOnly": true "readOnly": true
@ -461,7 +661,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 8 "y": 16
}, },
"id": 4, "id": 4,
"options": { "options": {
@ -483,8 +683,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "failed_store_queries_total", "expr": "failed_store_queries_total{node=~\"^$node$\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -497,6 +697,231 @@
"title": "Failed Store Queries", "title": "Failed Store Queries",
"type": "timeseries" "type": "timeseries"
}, },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Store responses including the latest Sonda message ",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"{__name__=\"\" , node=\"/dns4/store-01.do-ams3.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\"}"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"id": 11,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "increase(successful_store_queries_total{node=~\"^$node$\"}[5m]) / 5",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{__name__=\"{{__name__}}\" , node=\"{{node}}\"}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Successful Store Responses per Minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Store queries with a non-200 response",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"id": 12,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "increase(failed_store_queries_total{node=~\"^$node$\"}[5m]) / 5",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{__name__=\"{{__name__}}\" , error=\"{{error}}\", node=\"{{node}}\"}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Failed Store Queries per Minute",
"type": "timeseries"
},
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
@ -586,7 +1011,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 16 "y": 32
}, },
"id": 5, "id": 5,
"options": { "options": {
@ -608,8 +1033,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "empty_store_responses_total", "expr": "empty_store_responses_total{node=~\"^$node$\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -627,7 +1052,7 @@
"type": "prometheus", "type": "prometheus",
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"description": "Latency of each store query", "description": "Store responses that didn't include our latest Sonda message",
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {
@ -688,7 +1113,7 @@
"options": { "options": {
"mode": "exclude", "mode": "exclude",
"names": [ "names": [
"{__name__=\"store_query_latency\" , node=\"/dns4/store-01.ac-cn-hongkong-c.shards.test.status.im/tcp/30303/p2p/16Uiu2HAm2M7xs7cLPc3jamawkEqbr7cUJX11uvY7LxQ6WFUdUKUT\"}" "{__name__=\"\" , node=\"/dns4/store-01.do-ams3.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\"}"
], ],
"prefix": "All except:", "prefix": "All except:",
"readOnly": true "readOnly": true
@ -711,9 +1136,9 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 16 "y": 32
}, },
"id": 6, "id": 13,
"options": { "options": {
"legend": { "legend": {
"calcs": [], "calcs": [],
@ -733,8 +1158,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "store_query_latency", "expr": "increase(empty_store_responses_total{node=~\"^$node$\"}[5m]) / 5",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -744,7 +1169,7 @@
"useBackend": false "useBackend": false
} }
], ],
"title": "Store Query Latency (seconds)", "title": "Empty Store Responses per Minute",
"type": "timeseries" "type": "timeseries"
}, },
{ {
@ -814,7 +1239,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 24 "y": 40
}, },
"id": 7, "id": 7,
"options": { "options": {
@ -833,8 +1258,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "store_query_latency", "expr": "store_query_latency{node=~\"^$node$\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -847,6 +1272,131 @@
"title": "Store Query Latency (seconds)", "title": "Store Query Latency (seconds)",
"type": "histogram" "type": "histogram"
}, },
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"description": "Latency of each store query",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"{__name__=\"store_query_latency\" , node=\"/dns4/store-01.do-ams3.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\"}"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 40
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "store_query_latency{node=~\"^$node$\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{__name__=\"{{__name__}}\" , node=\"{{node}}\"}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Store Query Latency (seconds)",
"type": "timeseries"
},
{ {
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
@ -913,7 +1463,7 @@
"options": { "options": {
"mode": "exclude", "mode": "exclude",
"names": [ "names": [
"{__name__=\"node_health\" , node=\"/dns4/store-01.gc-us-central1-a.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmMELCo218hncCtTvC2Dwbej3rbyHQcR8erXNnKGei7WPZ\"}" "{__name__=\"node_health\" , node=\"/dns4/store-01.do-ams3.shards.test.status.im/tcp/30303/p2p/16Uiu2HAmAUdrQ3uwzuE4Gy4D56hX6uLKEeerJAnhKEHZ3DxF1EfT\"}"
], ],
"prefix": "All except:", "prefix": "All except:",
"readOnly": true "readOnly": true
@ -935,8 +1485,8 @@
"gridPos": { "gridPos": {
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 12, "x": 0,
"y": 24 "y": 48
}, },
"id": 8, "id": 8,
"options": { "options": {
@ -958,8 +1508,8 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "builder", "editorMode": "code",
"expr": "node_health", "expr": "node_health{node=~\"^$node$\"}",
"fullMetaSearch": false, "fullMetaSearch": false,
"includeNullMetadata": true, "includeNullMetadata": true,
"instant": false, "instant": false,
@ -977,10 +1527,39 @@
"schemaVersion": 39, "schemaVersion": 39,
"tags": [], "tags": [],
"templating": { "templating": {
"list": [] "list": [
{
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"definition": "label_values(node)",
"hide": 0,
"includeAll": true,
"label": "node",
"multi": false,
"name": "node",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(node)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
}, },
"time": { "time": {
"from": "now-5m", "from": "now-1h",
"to": "now" "to": "now"
}, },
"timepicker": {}, "timepicker": {},