feat(chat2): metrics server + metrics logging (#1279)

* feat(rln-relay): periodically log metrics

* fix(chat2): enable metrics by default in chat2 config

* test(chat2): metrics, gc compilation error

* chore(metrics): make metrics a util, and import into wakunode2 apps dir

* fix(metrics): raise error

* fix(metrics): gc error

* fix(wakunode2): remove setup_metrics

* chore(metrics): waku utils metrics

* fix(metrics): create waku_metrics in node dir

* fix(metrics): log scope

* fix(chat2): disable metrics server by default

* fix(utils): collectorAsF64 proc def

* fix(metrics): store metrics path

apps/chat2/chat2.nim

@@ -26,7 +26,7 @@ import
   ../../waku/v2/protocol/waku_lightpush,
   ../../waku/v2/protocol/waku_filter, 
   ../../waku/v2/protocol/waku_store,
-  ../../waku/v2/node/[waku_node, waku_payload],
+  ../../waku/v2/node/[waku_node, waku_payload, waku_metrics],
   ../../waku/v2/node/dnsdisc/waku_dnsdisc,
   ../../waku/v2/node/peer_manager/peer_manager,
   ../../waku/v2/utils/[peers, time],
@@ -551,6 +551,14 @@ proc processInput(rfd: AsyncFD) {.async.} =
           echo "your rln identity key is: ", node.wakuRlnRelay.membershipKeyPair.idKey.inHex()
           echo "your rln identity commitment key is: ", node.wakuRlnRelay.membershipKeyPair.idCommitment.inHex()
 
+  if conf.metricsLogging:
+    startMetricsLog()
+
+  if conf.metricsServer:
+    startMetricsServer(conf.metricsServerAddress,
+                       Port(conf.metricsServerPort + conf.portsShift))
+    
+
   await chat.readWriteLoop()
 
   if conf.keepAlive:

apps/chat2/config_chat2.nim

@@ -180,7 +180,7 @@ type
 
     metricsLogging* {.
       desc: "Enable metrics logging: true|false"
-      defaultValue: false
+      defaultValue: true
       name: "metrics-logging" }: bool
     
     ## DNS discovery config

apps/wakunode2/wakunode2.nim

@@ -31,13 +31,10 @@ import
   ../../waku/v2/node/storage/message/sqlite_store,
   ../../waku/v2/node/storage/message/message_retention_policy_capacity,
   ../../waku/v2/node/storage/message/message_retention_policy_time,
-  ../../waku/v2/node/wakuswitch,
-  ../../waku/v2/node/waku_node,
-  ../../waku/v2/utils/peers,
-  ../../waku/v2/utils/wakuenr,
+  ../../waku/v2/node/[wakuswitch, waku_node, waku_metrics],
+  ../../waku/v2/utils/[peers, wakuenr],
   ../../waku/common/utils/nat,
   ./wakunode2_setup_rest,
-  ./wakunode2_setup_metrics,
   ./wakunode2_setup_rpc,
   ./wakunode2_setup_sql_migrations,
   ./config

waku/v2/node/waku_metrics.nim

@@ -8,22 +8,22 @@ import
   metrics,
   metrics/chronos_httpserver
 import
-  ../../waku/v2/protocol/waku_filter,
-  ../../waku/v2/protocol/waku_store/protocol_metrics,
-  ../../waku/v2/protocol/waku_lightpush,
-  ../../waku/v2/protocol/waku_swap/waku_swap,
-  ../../waku/v2/protocol/waku_peer_exchange,
-  ../../waku/v2/utils/collector,
-  ../../waku/v2/node/peer_manager/peer_manager,
-  ../../waku/v2/node/waku_node,
-  ./config
+  ../protocol/waku_filter,
+  ../protocol/waku_store/protocol_metrics,
+  ../protocol/waku_lightpush,
+  ../protocol/waku_swap/waku_swap,
+  ../protocol/waku_peer_exchange,
+  ../utils/collector,
+  ./peer_manager/peer_manager,
+  ./waku_node
 
 when defined(rln) or defined(rlnzerokit):
-  import ../../waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics
+  import ../protocol/waku_rln_relay/waku_rln_relay_metrics
 
+const LogInterval = 30.seconds
 
 logScope:
-  topics = "wakunode.setup.metrics"
+  topics = "waku.metrics"
 
 
 proc startMetricsServer*(serverIp: ValidIpAddress, serverPort: Port) =
@@ -36,11 +36,12 @@ proc startMetricsServer*(serverIp: ValidIpAddress, serverPort: Port) =
 
     info "Metrics HTTP server started", serverIp, serverPort
 
-
+type
+  # https://github.com/nim-lang/Nim/issues/17369
+  MetricsLogger = proc(udata: pointer) {.gcsafe, raises: [Defect].}
 
 proc startMetricsLog*() =
-  # https://github.com/nim-lang/Nim/issues/17369
-  var logMetrics: proc(udata: pointer) {.gcsafe, raises: [Defect].}
+  var logMetrics: MetricsLogger
 
   var cumulativeErrors = 0.float64
   var cumulativeConns = 0.float64
@@ -56,23 +57,21 @@ proc startMetricsLog*() =
       # track cumulative values
       let freshErrorCount = parseAndAccumulate(waku_node_errors, cumulativeErrors)
       let freshConnCount = parseAndAccumulate(waku_node_conns_initiated, cumulativeConns)
-      
+
       info "Total connections initiated", count = freshConnCount
-      info "Total messages", count = parseCollectorIntoF64(waku_node_messages)
-      info "Total swap peers", count = parseCollectorIntoF64(waku_swap_peers_count)
-      info "Total filter peers", count = parseCollectorIntoF64(waku_filter_peers)
-      info "Total store peers", count = parseCollectorIntoF64(waku_store_peers)
-      info "Total lightpush peers", count = parseCollectorIntoF64(waku_lightpush_peers)
-      info "Total peer exchange peers", count = parseCollectorIntoF64(waku_px_peers)
+      info "Total messages", count = collectorAsF64(waku_node_messages)
+      info "Total swap peers", count = collectorAsF64(waku_swap_peers_count)
+      info "Total filter peers", count = collectorAsF64(waku_filter_peers)
+      info "Total store peers", count = collectorAsF64(waku_store_peers)
+      info "Total lightpush peers", count = collectorAsF64(waku_lightpush_peers)
+      info "Total peer exchange peers", count = collectorAsF64(waku_px_peers)
       info "Total errors", count = freshErrorCount
-      info "Total active filter subscriptions", count = parseCollectorIntoF64(waku_filter_subscribers)
+      info "Total active filter subscriptions", count = collectorAsF64(waku_filter_subscribers)
 
       # Start protocol specific metrics logging
       when defined(rln) or defined(rlnzerokit):
         logRlnMetrics()
 
-    discard setTimer(Moment.fromNow(30.seconds), logMetrics)
-  
-  discard setTimer(Moment.fromNow(30.seconds), logMetrics)
-
+    discard setTimer(Moment.fromNow(LogInterval), logMetrics)
   
+  discard setTimer(Moment.fromNow(LogInterval), logMetrics)

waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics.nim

@@ -41,8 +41,11 @@ declarePublicGauge(waku_rln_instance_creation_duration_seconds, "time taken to c
 declarePublicGauge(waku_rln_membership_insertion_duration_seconds, "time taken to insert a new member into the local merkle tree")
 declarePublicGauge(waku_rln_membership_credentials_import_duration_seconds, "time taken to import membership credentials")
 
-proc getRlnMetricsLogger*(): proc()  =
-  var logMetrics: proc() {.gcsafe, raises: [Defect].}
+type
+  RLNMetricsLogger = proc() {.gcsafe, raises: [Defect].}
+
+proc getRlnMetricsLogger*(): RLNMetricsLogger =
+  var logMetrics: RLNMetricsLogger
 
   var cumulativeErrors = 0.float64
   var cumulativeMessages = 0.float64

waku/v2/utils/collector.nim

@@ -1,19 +1,30 @@
-import
-    metrics
+{.push raises: [Defect].}
 
-proc parseCollectorIntoF64*(collector: Collector): float64 = 
-  var total = 0.float64
-  for key in collector.metrics.keys():
-    try:
-      total = total + collector.value(key)
-    except KeyError:
-      discard
-  return total
+import
+  metrics
+
+proc parseCollectorIntoF64(collector: Collector): float64 {.gcsafe, raises: [Defect] } = 
+  {.gcsafe.}:
+    var total = 0.float64
+    for key in collector.metrics.keys():
+      try:
+        total = total + collector.value(key)
+      except KeyError:
+        discard
+    return total
 
 template parseAndAccumulate*(collector: Collector, cumulativeValue: float64): float64 =
   ## This template is used to get metrics in a window 
   ## according to a cumulative value passed in
-  let total = parseCollectorIntoF64(collector)
-  let freshCount = total - cumulativeValue
-  cumulativeValue = total
-  freshCount
+  {.gcsafe.}:
+    let total = parseCollectorIntoF64(collector)
+    let freshCount = total - cumulativeValue
+    cumulativeValue = total
+    freshCount
+
+template collectorAsF64*(collector: Collector): float64 =
+  ## This template is used to get metrics from 0
+  ## Serves as a wrapper for parseCollectorIntoF64 which is gcsafe
+  {.gcsafe.}:
+    let total = parseCollectorIntoF64(collector)
+    total