diff --git a/cmd/storemsgcounter/execute.go b/cmd/storemsgcounter/execute.go index 44437cd..973c814 100644 --- a/cmd/storemsgcounter/execute.go +++ b/cmd/storemsgcounter/execute.go @@ -237,10 +237,12 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno missingInSummary := make(map[peer.ID]int) unknownInSummary := make(map[peer.ID]int) + totalMissingMessages := 0 for msgHash, nodes := range msgMap { var missingIn []peer.ID var unknownIn []peer.ID + for _, node := range storenodes { if nodes[node.ID] == DoesNotExist { missingIn = append(missingIn, node.ID) @@ -257,6 +259,7 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno if err != nil { return err } + totalMissingMessages++ } if len(unknownIn) != 0 { @@ -278,6 +281,8 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno logger.Info("messages that could not be verified summary", zap.Stringer("storenode", s.ID), zap.Int("numMsgs", missingCnt)) } + app.metrics.RecordTotalMissingMessages(totalMissingMessages) + return nil } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 76fad20..7bb34a9 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -15,6 +15,13 @@ var missingMessages = prometheus.NewGaugeVec( []string{"storenode", "status"}, ) +var totalMissingMessages = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "msgcounter_total_missing_messages", + Help: "The global total number of missing messages", + }, +) + var storenodeAvailability = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "msgcounter_storenode_availability", @@ -32,6 +39,7 @@ var collectors = []prometheus.Collector{ type Metrics interface { RecordMissingMessages(peerID peer.ID, status string, length int) RecordStorenodeAvailability(peerID peer.ID, available bool) + RecordTotalMissingMessages(cnt int) } type metricsImpl struct { @@ -62,3 +70,9 @@ func (m *metricsImpl) RecordStorenodeAvailability(peerID peer.ID, available bool storenodeAvailability.WithLabelValues(peerID.String()).Set(gaugeValue) }() } + +func (m *metricsImpl) RecordTotalMissingMessages(cnt int) { + go func() { + totalMissingMessages.Set(float64(cnt)) + }() +}