From c92513ec167e53390002866ed9671d73667d9222 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 21 Oct 2021 18:09:30 -0400 Subject: [PATCH] telemetry: set cert expiry metrics to NaN on start So that followers do not report 0, which would make alerting difficult. --- agent/agent.go | 2 +- agent/consul/leader_metrics.go | 11 ++++++++++- agent/consul/server.go | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index bb0b3e832d..0c3a1cc008 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -667,7 +667,7 @@ func (a *Agent) Start(ctx context.Context) error { } if a.tlsConfigurator.Cert() != nil { - m := consul.AgentTLSCertExpirationMonitor(a.tlsConfigurator, a.logger, a.config.Datacenter) + m := consul.AgentTLSCertExpirationMonitor(a.tlsConfigurator, a.logger) go m.Monitor(&lib.StopChannelContext{StopCh: a.shutdownCh}) } diff --git a/agent/consul/leader_metrics.go b/agent/consul/leader_metrics.go index 6da2d70a98..7828bf9f19 100644 --- a/agent/consul/leader_metrics.go +++ b/agent/consul/leader_metrics.go @@ -169,7 +169,7 @@ var metricsKeyAgentTLSCertExpiry = []string{"agent", "tls", "cert", "expiry"} // AgentTLSCertExpirationMonitor returns a CertExpirationMonitor which will // monitor the expiration of the certificate used for agent TLS. -func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger, dc string) CertExpirationMonitor { +func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger) CertExpirationMonitor { return CertExpirationMonitor{ Key: metricsKeyAgentTLSCertExpiry, Logger: logger, @@ -187,3 +187,12 @@ func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger, }, } } + +// initLeaderMetrics sets all metrics that are emitted only on leaders to a NaN +// value so that they don't incorrectly report 0 when a server starts as a +// follower. +func initLeaderMetrics() { + for _, g := range LeaderCertExpirationGauges { + metrics.SetGaugeWithLabels(g.Name, float32(math.NaN()), g.ConstLabels) + } +} diff --git a/agent/consul/server.go b/agent/consul/server.go index 969785a23a..524a0f1eae 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -389,6 +389,8 @@ func NewServer(config *Config, flat Deps) (*Server, error) { return nil, err } + initLeaderMetrics() + s.rpcLimiter.Store(rate.NewLimiter(config.RPCRateLimit, config.RPCMaxBurst)) configReplicatorConfig := ReplicatorConfig{