diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index ecfaf3a4..5672d263 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -12,6 +12,7 @@ import std/sequtils import pkg/chronos import pkg/chronicles import pkg/libp2p +import pkg/metrics import ../protobuf/presence @@ -27,6 +28,8 @@ import ./pendingblocks logScope: topics = "codex discovery engine" +declareGauge(codex_inflight_discovery, "inflight discovery requests") + const DefaultConcurrentDiscRequests = 10 DefaultConcurrentAdvertRequests = 10 @@ -104,12 +107,15 @@ proc advertiseTaskLoop(b: DiscoveryEngine) {.async.} = continue try: - trace "Advertising block", cid = $cid let request = b.discovery.provide(cid) b.inFlightAdvReqs[cid] = request + codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) + trace "Advertising block", cid = $cid, inflight = b.inFlightAdvReqs.len await request finally: b.inFlightAdvReqs.del(cid) + codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) + trace "Advertised block", cid = $cid, inflight = b.inFlightAdvReqs.len except CatchableError as exc: trace "Exception in advertise task runner", exc = exc.msg @@ -141,6 +147,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} = .wait(DefaultDiscoveryTimeout) b.inFlightDiscReqs[cid] = request + codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) let peers = await request @@ -149,6 +156,7 @@ proc discoveryTaskLoop(b: DiscoveryEngine) {.async.} = await allFinished(peers.mapIt( b.network.dialPeer(it.data)))) finally: b.inFlightDiscReqs.del(cid) + codex_inflight_discovery.set(b.inFlightAdvReqs.len.int64) except CatchableError as exc: trace "Exception in discovery task runner", exc = exc.msg diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 00000000..411d6357 --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,43 @@ +# Codex Metrics and Dashboard + +> This readme should help you to get started with collecting and visualizing metrics exposed by the Codex process. + +## Metrics + +Metrics are collected using the [nim-metrics](https://github.com/status-im/nim-metrics) backend and should be enabled with the `--metrics` flag. By default metrics are exposed on the `localhost:8008/metrics` end point. + +Use the `--metrics-address` and `--metrics-port` flags to to adjust the address and port as necessary. + +## General guidelines for adding new metrics + +Metrics are useful to monitor the health of the process and should aid in identifying and debugging potential issues that would be hard to notice otherwise. + +All Codex metrics should be prefixed with the `codex_` prefix to be able to differentiate from metrics exposed by other subsystems. For example libp2p generally prefixed with the `libp2p_` prefix. + +Metrics can be added on an as needed basis, however, keep in mind the potential overhead they might introduce. In particular, be careful with labels as they will generate as many metrics as there are labels for a specific collector. If a metrics or a set of metrics are expensive, it is usually advisable to put them behind a compile time flag. + +## Prometheus and Grafana + +The exposed metrics can be aggregate by the [Prometheus](https://prometheus.io/) monitoring systems and additionally graphed through [Grafana](https://grafana.com/). + +This directory contains both the default `prometheus.yml` config file as well as a basic `codex-grafana-dashboard.json` file that can be augmented with additional panels and metrics on an as needed basis. + +Additionally, please consider installing the [node_exporter](https://github.com/prometheus/node_exporter) agent to collect machine level metrics such as overall memory, process, networking, disc IO, etc... + +### Using the Grafana dashboard + +To use the dashboard open grafana and head to `Dashboards`, hit import in the top rightmost button right next to the `New Dashboard` and `New Folder`. + +![](assets/main.png) + +This will take you to the import page. + +![](assets/import.png) + +Use either one of the presented methods (upload json, load from a url or copy paste the json into the text-box), to upload the `codex-grafana-dashboard.json` file. + +Finally, you'll be presented with the following screen where you can change the name and the `UID` of the imported dashboard. This is only necessary if there is already a dashboard with the same name or `UID`. + +![](./assets/imported.png) + +Once imported, the dashboard should show up on the main dashboard page. diff --git a/metrics/assets/import.png b/metrics/assets/import.png new file mode 100644 index 00000000..43264490 Binary files /dev/null and b/metrics/assets/import.png differ diff --git a/metrics/assets/imported.png b/metrics/assets/imported.png new file mode 100644 index 00000000..62e67011 Binary files /dev/null and b/metrics/assets/imported.png differ diff --git a/metrics/assets/main.png b/metrics/assets/main.png new file mode 100644 index 00000000..07a3c9dc Binary files /dev/null and b/metrics/assets/main.png differ diff --git a/metrics/codex-grafana-dashboard.json b/metrics/codex-grafana-dashboard.json new file mode 100644 index 00000000..3f2913e4 --- /dev/null +++ b/metrics/codex-grafana-dashboard.json @@ -0,0 +1,1226 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 13, + "iteration": 1659993037535, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 48, + "panels": [], + "title": "Main", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 58, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "codex_inflight_discovery{}", + "refId": "A" + } + ], + "title": "Codex Inflight Discovery", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "nim_gc_heap_instance_occupied_bytes{node=\"${node}\"}", + "interval": "", + "legendFormat": "{{type_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "GC heap objects #${node}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": false + }, + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^process_resident_memory_bytes{instance=\"127.0.0.1:8008\", job=\"nimbus\", node=\"0\"}$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "process_resident_memory_bytes{node=\"${node}\"}", + "refId": "A" + } + ], + "title": "RSS mem #${node}", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 8, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^{instance=\"127.0.0.1:8008\", job=\"nimbus\", node=\"0\"}$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "rate(node_cpu_seconds_total{job=\"codex\"}[1m])", + "refId": "A" + } + ], + "title": "CPU usage #${node}", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "RSS", + "yaxis": 2 + }, + { + "alias": "Nim GC mem total", + "yaxis": 2 + }, + { + "alias": "Nim GC mem used", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "rate(process_cpu_seconds_total{node=\"${node}\"}[2s]) * 100", + "legendFormat": "CPU usage %", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "process_open_fds{node=\"${node}\"}", + "legendFormat": "open file descriptors", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "process_resident_memory_bytes{node=\"${node}\"}", + "legendFormat": "RSS", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "nim_gc_mem_bytes{node=\"${node}\"}", + "legendFormat": "Nim GC mem total", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "nim_gc_mem_occupied_bytes{node=\"${node}\"}", + "legendFormat": "Nim GC mem used", + "refId": "G" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "resources #${node}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 50, + "panels": [], + "title": "libp2p", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum by(type) (libp2p_peers{node=\"${node}\"})", + "interval": "", + "legendFormat": "connected peers", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum (libp2p_pubsub_peers{node=\"${node}\"})", + "interval": "", + "legendFormat": "pubsub peers", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum (nbc_peers{node=\"${node}\"})", + "interval": "", + "legendFormat": "nbc peers", + "refId": "C" + } + ], + "title": "libp2p peers #${node}", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 16, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum by(type) (libp2p_open_streams{node=\"${node}\"})", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum (libp2p_open_streams{node=\"${node}\"})", + "interval": "", + "legendFormat": "combined", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum by(type, dir) (libp2p_open_streams{node=\"${node}\"})", + "interval": "", + "legendFormat": "{{type}, {dir}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "libp2p open streams #${node}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum by(initiator,node)(libp2p_mplex_channels{node=\"${node}\"})", + "interval": "", + "legendFormat": "initiator {{initiator}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sum(libp2p_mplex_channels{node=\"${node}\"})", + "interval": "", + "legendFormat": "total", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "libp2p mplex channels #${node}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "sort_desc(sum by(peer, initiator)(libp2p_mplex_channels{node=\"${node}\"}))", + "interval": "", + "legendFormat": "peer {{peer}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "libp2p mplex channels per peer #${node}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 52, + "panels": [], + "title": "Discovery", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "discovery_message_requests_incoming_total{}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "discovery_message_requests_outgoing_total{}", + "hide": false, + "refId": "B" + } + ], + "title": "Discovery Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "discovery_session_lru_cache_hits_total{instance=\"127.0.0.1:8008\", job=\"codex\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "discovery_session_lru_cache_hits_total{}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "expr": "discovery_session_lru_cache_misses_total{}", + "hide": false, + "refId": "B" + } + ], + "title": "Discovery LRU", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "_v_qlxkVz" + }, + "definition": "label_values(process_virtual_memory_bytes,node)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(process_virtual_memory_bytes,node)", + "refId": "Prometheus-node-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Codex Dashboard", + "uid": "pgeNfj2Wz2b", + "version": 24, + "weekStart": "" +} diff --git a/metrics/prometheus.yml b/metrics/prometheus.yml new file mode 100644 index 00000000..79b99e1c --- /dev/null +++ b/metrics/prometheus.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 12s + +scrape_configs: + - job_name: "codex" + static_configs: + - targets: ['127.0.0.1:8008'] + - job_name: "node_exporter" + static_configs: + - targets: ['127.0.0.1:9100']