feat: storenode availability

This commit is contained in:
Richard Ramos 2024-07-19 11:30:22 -04:00
parent fd282b3455
commit d16fdea83d
No known key found for this signature in database
GPG Key ID: 1CE87DB518195760
2 changed files with 22 additions and 11 deletions

View File

@ -329,8 +329,9 @@ func retrieveHistory(ctx context.Context, runId string, storenodes []peer.AddrIn
if err != nil {
queryLogger.Error("could not store node unavailable", zap.Error(err))
}
metrics.RecordStorenodeUnavailable(storeAddr)
metrics.RecordStorenodeAvailability(storeAddr, false)
} else {
metrics.RecordStorenodeAvailability(storeAddr, true)
iteratorLbl:
for !result.IsComplete() {
@ -371,8 +372,10 @@ func retrieveHistory(ctx context.Context, runId string, storenodes []peer.AddrIn
if err != nil {
queryLogger.Error("could not store recordnode unavailable", zap.String("cursor", hex.EncodeToString(result.Cursor())), zap.Error(err))
}
metrics.RecordStorenodeUnavailable(storeAddr)
metrics.RecordStorenodeAvailability(storeAddr, false)
break iteratorLbl
} else {
metrics.RecordStorenodeAvailability(storeAddr, true)
}
}
}
@ -421,9 +424,11 @@ queryLbl:
if err != nil {
queryLogger.Error("could not store recordnode unavailable", zap.Error(err))
}
metrics.RecordStorenodeUnavailable(storeAddr)
metrics.RecordStorenodeAvailability(storeAddr, false)
} else {
metrics.RecordStorenodeAvailability(storeAddr, true)
for !result.IsComplete() {
msgMapLock.Lock()
for _, mkv := range result.Messages() {
@ -468,7 +473,9 @@ queryLbl:
if err != nil {
logger.Error("could not store recordnode unavailable", zap.Error(err), zap.String("cursor", hex.EncodeToString(result.Cursor())), zap.Stringer("storenode", peerInfo))
}
metrics.RecordStorenodeUnavailable(storeAddr)
metrics.RecordStorenodeAvailability(storeAddr, false)
} else {
metrics.RecordStorenodeAvailability(storeAddr, true)
}
}
}

View File

@ -14,23 +14,23 @@ var missingMessages = prometheus.NewGaugeVec(
[]string{"storenode", "status"},
)
var storenodeUnavailable = prometheus.NewGaugeVec(
var storenodeAvailability = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "counter_storenode_unavailable",
Help: "Number of PubSub Topics node is subscribed to",
Name: "counter_storenode_availability",
Help: "Indicate whether a store node is available or not",
},
[]string{"storenode"},
)
var collectors = []prometheus.Collector{
missingMessages,
storenodeUnavailable,
storenodeAvailability,
}
// Metrics exposes the functions required to update prometheus metrics for relay protocol
type Metrics interface {
RecordMissingMessages(storenode string, status string, length int)
RecordStorenodeUnavailable(storenode string)
RecordStorenodeAvailability(storenode string, available bool)
}
type metricsImpl struct {
@ -52,8 +52,12 @@ func (m *metricsImpl) RecordMissingMessages(storenode string, status string, len
}()
}
func (m *metricsImpl) RecordStorenodeUnavailable(storenode string) {
func (m *metricsImpl) RecordStorenodeAvailability(storenode string, available bool) {
go func() {
storenodeUnavailable.WithLabelValues(storenode).Set(1)
gaugeValue := float64(1)
if !available {
gaugeValue = 0
}
storenodeAvailability.WithLabelValues(storenode).Set(gaugeValue)
}()
}