feat: track total missing messages

This commit is contained in:
Richard Ramos 2024-07-19 16:22:13 -04:00
parent 7d4cc96cb6
commit ed15fd2dd8
No known key found for this signature in database
GPG Key ID: 1CE87DB518195760
2 changed files with 19 additions and 0 deletions

View File

@ -237,10 +237,12 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
missingInSummary := make(map[peer.ID]int)
unknownInSummary := make(map[peer.ID]int)
totalMissingMessages := 0
for msgHash, nodes := range msgMap {
var missingIn []peer.ID
var unknownIn []peer.ID
for _, node := range storenodes {
if nodes[node.ID] == DoesNotExist {
missingIn = append(missingIn, node.ID)
@ -257,6 +259,7 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
if err != nil {
return err
}
totalMissingMessages++
}
if len(unknownIn) != 0 {
@ -278,6 +281,8 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
logger.Info("messages that could not be verified summary", zap.Stringer("storenode", s.ID), zap.Int("numMsgs", missingCnt))
}
app.metrics.RecordTotalMissingMessages(totalMissingMessages)
return nil
}

View File

@ -15,6 +15,13 @@ var missingMessages = prometheus.NewGaugeVec(
[]string{"storenode", "status"},
)
var totalMissingMessages = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "msgcounter_total_missing_messages",
Help: "The global total number of missing messages",
},
)
var storenodeAvailability = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "msgcounter_storenode_availability",
@ -32,6 +39,7 @@ var collectors = []prometheus.Collector{
type Metrics interface {
RecordMissingMessages(peerID peer.ID, status string, length int)
RecordStorenodeAvailability(peerID peer.ID, available bool)
RecordTotalMissingMessages(cnt int)
}
type metricsImpl struct {
@ -62,3 +70,9 @@ func (m *metricsImpl) RecordStorenodeAvailability(peerID peer.ID, available bool
storenodeAvailability.WithLabelValues(peerID.String()).Set(gaugeValue)
}()
}
func (m *metricsImpl) RecordTotalMissingMessages(cnt int) {
go func() {
totalMissingMessages.Set(float64(cnt))
}()
}