From d16fdea83d24537114ee5cc391924610bcb00705 Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Fri, 19 Jul 2024 11:30:22 -0400 Subject: [PATCH] feat: storenode availability --- cmd/storemsgcounter/execute.go | 15 +++++++++++---- internal/metrics/metrics.go | 18 +++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cmd/storemsgcounter/execute.go b/cmd/storemsgcounter/execute.go index 67fc0de..11bd581 100644 --- a/cmd/storemsgcounter/execute.go +++ b/cmd/storemsgcounter/execute.go @@ -329,8 +329,9 @@ func retrieveHistory(ctx context.Context, runId string, storenodes []peer.AddrIn if err != nil { queryLogger.Error("could not store node unavailable", zap.Error(err)) } - metrics.RecordStorenodeUnavailable(storeAddr) + metrics.RecordStorenodeAvailability(storeAddr, false) } else { + metrics.RecordStorenodeAvailability(storeAddr, true) iteratorLbl: for !result.IsComplete() { @@ -371,8 +372,10 @@ func retrieveHistory(ctx context.Context, runId string, storenodes []peer.AddrIn if err != nil { queryLogger.Error("could not store recordnode unavailable", zap.String("cursor", hex.EncodeToString(result.Cursor())), zap.Error(err)) } - metrics.RecordStorenodeUnavailable(storeAddr) + metrics.RecordStorenodeAvailability(storeAddr, false) break iteratorLbl + } else { + metrics.RecordStorenodeAvailability(storeAddr, true) } } } @@ -421,9 +424,11 @@ queryLbl: if err != nil { queryLogger.Error("could not store recordnode unavailable", zap.Error(err)) } - metrics.RecordStorenodeUnavailable(storeAddr) + metrics.RecordStorenodeAvailability(storeAddr, false) } else { + metrics.RecordStorenodeAvailability(storeAddr, true) + for !result.IsComplete() { msgMapLock.Lock() for _, mkv := range result.Messages() { @@ -468,7 +473,9 @@ queryLbl: if err != nil { logger.Error("could not store recordnode unavailable", zap.Error(err), zap.String("cursor", hex.EncodeToString(result.Cursor())), zap.Stringer("storenode", peerInfo)) } - metrics.RecordStorenodeUnavailable(storeAddr) + metrics.RecordStorenodeAvailability(storeAddr, false) + } else { + metrics.RecordStorenodeAvailability(storeAddr, true) } } } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 09b6b66..782f43c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -14,23 +14,23 @@ var missingMessages = prometheus.NewGaugeVec( []string{"storenode", "status"}, ) -var storenodeUnavailable = prometheus.NewGaugeVec( +var storenodeAvailability = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "counter_storenode_unavailable", - Help: "Number of PubSub Topics node is subscribed to", + Name: "counter_storenode_availability", + Help: "Indicate whether a store node is available or not", }, []string{"storenode"}, ) var collectors = []prometheus.Collector{ missingMessages, - storenodeUnavailable, + storenodeAvailability, } // Metrics exposes the functions required to update prometheus metrics for relay protocol type Metrics interface { RecordMissingMessages(storenode string, status string, length int) - RecordStorenodeUnavailable(storenode string) + RecordStorenodeAvailability(storenode string, available bool) } type metricsImpl struct { @@ -52,8 +52,12 @@ func (m *metricsImpl) RecordMissingMessages(storenode string, status string, len }() } -func (m *metricsImpl) RecordStorenodeUnavailable(storenode string) { +func (m *metricsImpl) RecordStorenodeAvailability(storenode string, available bool) { go func() { - storenodeUnavailable.WithLabelValues(storenode).Set(1) + gaugeValue := float64(1) + if !available { + gaugeValue = 0 + } + storenodeAvailability.WithLabelValues(storenode).Set(gaugeValue) }() }