telemetry: set cert expiry metrics to NaN on start

So that followers do not report 0, which would make alerting difficult.

agent/agent.go

@@ -667,7 +667,7 @@ func (a *Agent) Start(ctx context.Context) error {
 	}
 
 	if a.tlsConfigurator.Cert() != nil {
-		m := consul.AgentTLSCertExpirationMonitor(a.tlsConfigurator, a.logger, a.config.Datacenter)
+		m := consul.AgentTLSCertExpirationMonitor(a.tlsConfigurator, a.logger)
 		go m.Monitor(&lib.StopChannelContext{StopCh: a.shutdownCh})
 	}
 

agent/consul/leader_metrics.go

@@ -169,7 +169,7 @@ var metricsKeyAgentTLSCertExpiry = []string{"agent", "tls", "cert", "expiry"}
 
 // AgentTLSCertExpirationMonitor returns a CertExpirationMonitor which will
 // monitor the expiration of the certificate used for agent TLS.
-func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger, dc string) CertExpirationMonitor {
+func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger) CertExpirationMonitor {
 	return CertExpirationMonitor{
 		Key:    metricsKeyAgentTLSCertExpiry,
 		Logger: logger,
@@ -187,3 +187,12 @@ func AgentTLSCertExpirationMonitor(c *tlsutil.Configurator, logger hclog.Logger,
 		},
 	}
 }
+
+// initLeaderMetrics sets all metrics that are emitted only on leaders to a NaN
+// value so that they don't incorrectly report 0 when a server starts as a
+// follower.
+func initLeaderMetrics() {
+	for _, g := range LeaderCertExpirationGauges {
+		metrics.SetGaugeWithLabels(g.Name, float32(math.NaN()), g.ConstLabels)
+	}
+}

agent/consul/server.go

@@ -389,6 +389,8 @@ func NewServer(config *Config, flat Deps) (*Server, error) {
 		return nil, err
 	}
 
+	initLeaderMetrics()
+
 	s.rpcLimiter.Store(rate.NewLimiter(config.RPCRateLimit, config.RPCMaxBurst))
 
 	configReplicatorConfig := ReplicatorConfig{