feat: track total missing messages

This commit is contained in:
Richard Ramos 2024-07-19 16:22:13 -04:00
parent 7d4cc96cb6
commit ed15fd2dd8
No known key found for this signature in database
GPG Key ID: 1CE87DB518195760
2 changed files with 19 additions and 0 deletions

View File

@ -237,10 +237,12 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
missingInSummary := make(map[peer.ID]int) missingInSummary := make(map[peer.ID]int)
unknownInSummary := make(map[peer.ID]int) unknownInSummary := make(map[peer.ID]int)
totalMissingMessages := 0
for msgHash, nodes := range msgMap { for msgHash, nodes := range msgMap {
var missingIn []peer.ID var missingIn []peer.ID
var unknownIn []peer.ID var unknownIn []peer.ID
for _, node := range storenodes { for _, node := range storenodes {
if nodes[node.ID] == DoesNotExist { if nodes[node.ID] == DoesNotExist {
missingIn = append(missingIn, node.ID) missingIn = append(missingIn, node.ID)
@ -257,6 +259,7 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
if err != nil { if err != nil {
return err return err
} }
totalMissingMessages++
} }
if len(unknownIn) != 0 { if len(unknownIn) != 0 {
@ -278,6 +281,8 @@ func (app *Application) verifyHistory(ctx context.Context, runId string, storeno
logger.Info("messages that could not be verified summary", zap.Stringer("storenode", s.ID), zap.Int("numMsgs", missingCnt)) logger.Info("messages that could not be verified summary", zap.Stringer("storenode", s.ID), zap.Int("numMsgs", missingCnt))
} }
app.metrics.RecordTotalMissingMessages(totalMissingMessages)
return nil return nil
} }

View File

@ -15,6 +15,13 @@ var missingMessages = prometheus.NewGaugeVec(
[]string{"storenode", "status"}, []string{"storenode", "status"},
) )
var totalMissingMessages = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "msgcounter_total_missing_messages",
Help: "The global total number of missing messages",
},
)
var storenodeAvailability = prometheus.NewGaugeVec( var storenodeAvailability = prometheus.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: "msgcounter_storenode_availability", Name: "msgcounter_storenode_availability",
@ -32,6 +39,7 @@ var collectors = []prometheus.Collector{
type Metrics interface { type Metrics interface {
RecordMissingMessages(peerID peer.ID, status string, length int) RecordMissingMessages(peerID peer.ID, status string, length int)
RecordStorenodeAvailability(peerID peer.ID, available bool) RecordStorenodeAvailability(peerID peer.ID, available bool)
RecordTotalMissingMessages(cnt int)
} }
type metricsImpl struct { type metricsImpl struct {
@ -62,3 +70,9 @@ func (m *metricsImpl) RecordStorenodeAvailability(peerID peer.ID, available bool
storenodeAvailability.WithLabelValues(peerID.String()).Set(gaugeValue) storenodeAvailability.WithLabelValues(peerID.String()).Set(gaugeValue)
}() }()
} }
func (m *metricsImpl) RecordTotalMissingMessages(cnt int) {
go func() {
totalMissingMessages.Set(float64(cnt))
}()
}