From 5e0e4098c9fe1a714e11a87e20cda44404c1ff5c Mon Sep 17 00:00:00 2001 From: Kit Patella Date: Mon, 16 Nov 2020 12:44:47 -0800 Subject: [PATCH] push prometheus sink definiitons into prometheus.PrometheusOpts --- agent/agent_endpoint.go | 2 +- agent/config/builder.go | 5 ++++- agent/setup.go | 14 ++++++------ connect/proxy/proxy.go | 2 +- lib/telemetry.go | 47 +++++++++++++---------------------------- 5 files changed, 28 insertions(+), 42 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 73e0f53640..49721f9125 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -136,7 +136,7 @@ func (s *HTTPHandlers) AgentMetrics(resp http.ResponseWriter, req *http.Request) return nil, acl.ErrPermissionDenied } if enablePrometheusOutput(req) { - if s.agent.config.Telemetry.PrometheusRetentionTime < 1 { + if s.agent.config.Telemetry.PrometheusOpts.Expiration < 1 { resp.WriteHeader(http.StatusUnsupportedMediaType) fmt.Fprint(resp, "Prometheus is not enabled since its retention time is not positive") return nil, nil diff --git a/agent/config/builder.go b/agent/config/builder.go index 1c1798dc76..062fb440f7 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "github.com/armon/go-metrics/prometheus" "github.com/hashicorp/go-bexpr" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" @@ -942,13 +943,15 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { DisableHostname: b.boolVal(c.Telemetry.DisableHostname), DogstatsdAddr: b.stringVal(c.Telemetry.DogstatsdAddr), DogstatsdTags: c.Telemetry.DogstatsdTags, - PrometheusRetentionTime: b.durationVal("prometheus_retention_time", c.Telemetry.PrometheusRetentionTime), FilterDefault: b.boolVal(c.Telemetry.FilterDefault), AllowedPrefixes: telemetryAllowedPrefixes, BlockedPrefixes: telemetryBlockedPrefixes, MetricsPrefix: b.stringVal(c.Telemetry.MetricsPrefix), StatsdAddr: b.stringVal(c.Telemetry.StatsdAddr), StatsiteAddr: b.stringVal(c.Telemetry.StatsiteAddr), + PrometheusOpts: prometheus.PrometheusOpts{ + Expiration: b.durationVal("prometheus_retention_time", c.Telemetry.PrometheusRetentionTime), + }, }, // Agent diff --git a/agent/setup.go b/agent/setup.go index 49a586ebbc..744840a155 100644 --- a/agent/setup.go +++ b/agent/setup.go @@ -78,7 +78,11 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer) (BaseDeps, error) return d, fmt.Errorf("failed to setup node ID: %w", err) } - d.MetricsHandler, err = lib.InitTelemetry(cfg.Telemetry, getPrometheusDefs()) + gauges, counters, summaries := getPrometheusDefs() + cfg.Telemetry.PrometheusOpts.GaugeDefinitions = gauges + cfg.Telemetry.PrometheusOpts.CounterDefinitions = counters + cfg.Telemetry.PrometheusOpts.SummaryDefinitions = summaries + d.MetricsHandler, err = lib.InitTelemetry(cfg.Telemetry) if err != nil { return d, fmt.Errorf("failed to initialize telemetry: %w", err) } @@ -186,7 +190,7 @@ func registerWithGRPC(b grpcresolver.Builder) { // getPrometheusDefs reaches into every slice of prometheus defs we've defined in each part of the agent, and appends // all of our slices into one nice slice of definitions per metric type for the Consul agent to pass to go-metrics. -func getPrometheusDefs() lib.PrometheusDefs { +func getPrometheusDefs() ([]prometheus.GaugeDefinition, []prometheus.CounterDefinition, []prometheus.SummaryDefinition) { serviceName := []string{"consul"} var gauges = [][]prometheus.GaugeDefinition{ cache.Gauges, @@ -290,9 +294,5 @@ func getPrometheusDefs() lib.PrometheusDefs { summaryDefs = append(summaryDefs, withService...) } - return lib.PrometheusDefs{ - Gauges: gaugeDefs, - Counters: counterDefs, - Summaries: summaryDefs, - } + return gaugeDefs, counterDefs, summaryDefs } diff --git a/connect/proxy/proxy.go b/connect/proxy/proxy.go index 54df4e309b..a29cf352e8 100644 --- a/connect/proxy/proxy.go +++ b/connect/proxy/proxy.go @@ -56,7 +56,7 @@ func (p *Proxy) Serve() error { // Setup telemetry if configured // NOTE(kit): As far as I can tell, all of the metrics in the proxy are generated at runtime, so we // don't have any static metrics we initialize at start. - _, err := lib.InitTelemetry(newCfg.Telemetry, lib.EmptyPrometheusDefs()) + _, err := lib.InitTelemetry(newCfg.Telemetry) if err != nil { p.logger.Error("proxy telemetry config error", "error", err) } diff --git a/lib/telemetry.go b/lib/telemetry.go index fe360172a6..27ef0043df 100644 --- a/lib/telemetry.go +++ b/lib/telemetry.go @@ -154,14 +154,6 @@ type TelemetryConfig struct { // hcl: telemetry { dogstatsd_tags = []string } DogstatsdTags []string `json:"dogstatsd_tags,omitempty" mapstructure:"dogstatsd_tags"` - // PrometheusRetentionTime is the retention time for prometheus metrics if greater than 0. - // A value of 0 disable Prometheus support. Regarding Prometheus, it is considered a good - // practice to put large values here (such as a few days), and at least the interval between - // prometheus requests. - // - // hcl: telemetry { prometheus_retention_time = "duration" } - PrometheusRetentionTime time.Duration `json:"prometheus_retention_time,omitempty" mapstructure:"prometheus_retention_time"` - // FilterDefault is the default for whether to allow a metric that's not // covered by the filter. // @@ -199,6 +191,13 @@ type TelemetryConfig struct { // // hcl: telemetry { statsite_address = string } StatsiteAddr string `json:"statsite_address,omitempty" mapstructure:"statsite_address"` + + // PrometheusOpts provides configuration for the PrometheusSink. Currently the only configuration + // we acquire from hcl is the retention time. We also use definition slices that are set in agent setup + // before being passed to InitTelemmetry. + // + // hcl: telemetry { prometheus_retention_time = "duration" } + PrometheusOpts prometheus.PrometheusOpts } // MergeDefaults copies any non-zero field from defaults into the current @@ -276,17 +275,17 @@ func dogstatdSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, err return sink, nil } -func prometheusSink(cfg TelemetryConfig, hostname string, defs PrometheusDefs) (metrics.MetricSink, error) { +func prometheusSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, error) { - if cfg.PrometheusRetentionTime.Nanoseconds() < 1 { + if cfg.PrometheusOpts.Expiration.Nanoseconds() < 1 { return nil, nil } prometheusOpts := prometheus.PrometheusOpts{ - Expiration: cfg.PrometheusRetentionTime, - GaugeDefinitions: defs.Gauges, - CounterDefinitions: defs.Counters, - SummaryDefinitions: defs.Summaries, + Expiration: cfg.PrometheusOpts.Expiration, + GaugeDefinitions: cfg.PrometheusOpts.GaugeDefinitions, + CounterDefinitions: cfg.PrometheusOpts.CounterDefinitions, + SummaryDefinitions: cfg.PrometheusOpts.SummaryDefinitions, } sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) if err != nil { @@ -337,25 +336,9 @@ func circonusSink(cfg TelemetryConfig, hostname string) (metrics.MetricSink, err return sink, nil } -// PrometheusDefs wraps collections of metric definitions to pass into the PrometheusSink -type PrometheusDefs struct { - Gauges []prometheus.GaugeDefinition - Counters []prometheus.CounterDefinition - Summaries []prometheus.SummaryDefinition -} - -// EmptyPrometheusDefs returns a PrometheusDefs struct where each of the slices have zero elements, but not nil. -func EmptyPrometheusDefs() PrometheusDefs { - return PrometheusDefs{ - Gauges: []prometheus.GaugeDefinition{}, - Counters: []prometheus.CounterDefinition{}, - Summaries: []prometheus.SummaryDefinition{}, - } -} - // InitTelemetry configures go-metrics based on map of telemetry config // values as returned by Runtimecfg.Config(). -func InitTelemetry(cfg TelemetryConfig, defs PrometheusDefs) (*metrics.InmemSink, error) { +func InitTelemetry(cfg TelemetryConfig) (*metrics.InmemSink, error) { if cfg.Disable { return nil, nil } @@ -395,7 +378,7 @@ func InitTelemetry(cfg TelemetryConfig, defs PrometheusDefs) (*metrics.InmemSink return nil, err } - promSink, err := prometheusSink(cfg, metricsConf.HostName, defs) + promSink, err := prometheusSink(cfg, metricsConf.HostName) if err != nil { return nil, err }