From 0a208932ff42db191e7226042925f8d8a085e6b5 Mon Sep 17 00:00:00 2001 From: rymnc Date: Tue, 18 Oct 2022 12:13:33 +0000 Subject: [PATCH] deploy: 5d8f3a5b730f6c6a088f1bc6bb2306e0a3d38e12 --- waku/v2/node/wakunode2_setup_metrics.nim | 32 ++++++------- .../waku_rln_relay/waku_rln_relay_metrics.nim | 47 ++++++++++++++++++- waku/v2/utils/collector.nim | 19 ++++++++ 3 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 waku/v2/utils/collector.nim diff --git a/waku/v2/node/wakunode2_setup_metrics.nim b/waku/v2/node/wakunode2_setup_metrics.nim index b649bd826..091cb29df 100644 --- a/waku/v2/node/wakunode2_setup_metrics.nim +++ b/waku/v2/node/wakunode2_setup_metrics.nim @@ -14,7 +14,9 @@ import ../protocol/waku_store, ../protocol/waku_lightpush, ../protocol/waku_swap/waku_swap, - ../protocol/waku_peer_exchange + ../protocol/waku_peer_exchange, + ../protocol/waku_rln_relay/waku_rln_relay_metrics, + ../utils/collector logScope: topics = "wakunode.setup.metrics" @@ -30,14 +32,7 @@ proc startMetricsServer*(serverIp: ValidIpAddress, serverPort: Port) = info "Metrics HTTP server started", serverIp, serverPort -proc parseCollectorIntoF64(collector: Collector): float64 = - var total = 0.float64 - for key in collector.metrics.keys(): - try: - total = total + collector.value(key) - except KeyError: - discard - return total + proc startMetricsLog*() = # https://github.com/nim-lang/Nim/issues/17369 @@ -46,20 +41,18 @@ proc startMetricsLog*() = var cumulativeErrors = 0.float64 var cumulativeConns = 0.float64 + when defined(rln) or defined(rlnzerokit): + let logRlnMetrics = getRlnMetricsLogger() + logMetrics = proc(udata: pointer) = {.gcsafe.}: # TODO: libp2p_pubsub_peers is not public, so we need to make this either # public in libp2p or do our own peer counting after all. - let totalErrors = parseCollectorIntoF64(waku_node_errors) - let totalConnections = parseCollectorIntoF64(waku_node_conns_initiated) - - # track cumulative, and then max. - let freshErrorCount = max(totalErrors - cumulativeErrors, 0) - let freshConnCount = max(totalConnections - cumulativeConns, 0) + # track cumulative values + let freshErrorCount = parseAndAccumulate(waku_node_errors, cumulativeErrors) + let freshConnCount = parseAndAccumulate(waku_node_conns_initiated, cumulativeConns) - cumulativeErrors = totalErrors - cumulativeConns = totalConnections info "Total connections initiated", count = freshConnCount info "Total messages", count = parseCollectorIntoF64(waku_node_messages) info "Total swap peers", count = parseCollectorIntoF64(waku_swap_peers_count) @@ -70,7 +63,12 @@ proc startMetricsLog*() = info "Total errors", count = freshErrorCount info "Total active filter subscriptions", count = parseCollectorIntoF64(waku_filter_subscribers) + # Start protocol specific metrics logging + when defined(rln) or defined(rlnzerokit): + logRlnMetrics() + discard setTimer(Moment.fromNow(30.seconds), logMetrics) discard setTimer(Moment.fromNow(30.seconds), logMetrics) + diff --git a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics.nim b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics.nim index 546e661b8..1534d4748 100644 --- a/waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics.nim +++ b/waku/v2/protocol/waku_rln_relay/waku_rln_relay_metrics.nim @@ -1,9 +1,18 @@ +{.push raises: [Defect].} + import + chronicles, + chronos, metrics, - waku_rln_relay_constants + metrics/chronos_httpserver, + waku_rln_relay_constants, + ../../utils/collector export metrics +logScope: + topics = "waku-rln-relay.metrics" + func generateBucketsForHistogram*(length: int): seq[float64] = ## Generate a custom set of 5 buckets for a given length let numberOfBuckets = 5 @@ -31,3 +40,39 @@ declarePublicGauge(waku_rln_registration_duration_seconds, "time taken to regist declarePublicGauge(waku_rln_instance_creation_duration_seconds, "time taken to create an rln instance") declarePublicGauge(waku_rln_membership_insertion_duration_seconds, "time taken to insert a new member into the local merkle tree") declarePublicGauge(waku_rln_membership_credentials_import_duration_seconds, "time taken to import membership credentials") + +proc getRlnMetricsLogger*(): proc() = + var logMetrics: proc() {.gcsafe, raises: [Defect].} + + var cumulativeErrors = 0.float64 + var cumulativeMessages = 0.float64 + var cumulativeSpamMessages = 0.float64 + var cumulativeInvalidMessages = 0.float64 + var cumulativeValidMessages = 0.float64 + var cumulativeProofs = 0.float64 + + logMetrics = proc() = + {.gcsafe.}: + + let freshErrorCount = parseAndAccumulate(waku_rln_errors_total, + cumulativeErrors) + let freshMsgCount = parseAndAccumulate(waku_rln_messages_total, + cumulativeMessages) + let freshSpamCount = parseAndAccumulate(waku_rln_spam_messages_total, + cumulativeSpamMessages) + let freshInvalidMsgCount = parseAndAccumulate(waku_rln_invalid_messages_total, + cumulativeInvalidMessages) + let freshValidMsgCount = parseAndAccumulate(waku_rln_valid_messages_total, + cumulativeValidMessages) + let freshProofCount = parseAndAccumulate(waku_rln_proof_verification_total, + cumulativeProofs) + + + info "Total messages", count = freshMsgCount + info "Total spam messages", count = freshSpamCount + info "Total invalid messages", count = freshInvalidMsgCount + info "Total valid messages", count = freshValidMsgCount + info "Total errors", count = freshErrorCount + info "Total proofs verified", count = freshProofCount + return logMetrics + diff --git a/waku/v2/utils/collector.nim b/waku/v2/utils/collector.nim new file mode 100644 index 000000000..21428b8d2 --- /dev/null +++ b/waku/v2/utils/collector.nim @@ -0,0 +1,19 @@ +import + metrics + +proc parseCollectorIntoF64*(collector: Collector): float64 = + var total = 0.float64 + for key in collector.metrics.keys(): + try: + total = total + collector.value(key) + except KeyError: + discard + return total + +template parseAndAccumulate*(collector: Collector, cumulativeValue: float64): float64 = + ## This template is used to get metrics in a window + ## according to a cumulative value passed in + let total = parseCollectorIntoF64(collector) + let freshCount = total - cumulativeValue + cumulativeValue = total + freshCount \ No newline at end of file