From e231f0ee9bc11b8fdfd2d74a3bcae688fc50b30b Mon Sep 17 00:00:00 2001 From: Eric Haberkorn Date: Thu, 11 Apr 2024 15:20:09 -0400 Subject: [PATCH] Add an agent config option to diable per tenancy usage metrics. (#20976) --- .changelog/_9142.txt | 3 + agent/config/builder.go | 1 + agent/config/config.go | 1 + agent/config/runtime_test.go | 5 +- .../TestRuntimeConfig_Sanitize.golden | 5 +- agent/config/testdata/full-config.hcl | 1 + agent/config/testdata/full-config.json | 5 +- agent/consul/config.go | 2 + agent/consul/server.go | 1 + agent/consul/usagemetrics/usagemetrics.go | 125 ++- agent/consul/usagemetrics/usagemetrics_ce.go | 97 +- .../usagemetrics/usagemetrics_ce_test.go | 938 +----------------- .../consul/usagemetrics/usagemetrics_test.go | 861 +++++++++++++++- lib/telemetry.go | 7 +- .../docs/agent/config/config-files.mdx | 3 + 15 files changed, 1035 insertions(+), 1020 deletions(-) create mode 100644 .changelog/_9142.txt diff --git a/.changelog/_9142.txt b/.changelog/_9142.txt new file mode 100644 index 0000000000..9a10a8a0ca --- /dev/null +++ b/.changelog/_9142.txt @@ -0,0 +1,3 @@ +```release-note: improvement +telemetry: Add `telemetry.disable_per_tenancy_usage_metrics` in agent configuration to disable setting tenancy labels on usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces. +``` diff --git a/agent/config/builder.go b/agent/config/builder.go index 44784fcc61..64e9120fde 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -947,6 +947,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { CirconusSubmissionInterval: stringVal(c.Telemetry.CirconusSubmissionInterval), CirconusSubmissionURL: stringVal(c.Telemetry.CirconusSubmissionURL), DisableHostname: boolVal(c.Telemetry.DisableHostname), + DisablePerTenancyUsageMetrics: boolVal(c.Telemetry.DisablePerTenancyUsageMetrics), DogstatsdAddr: stringVal(c.Telemetry.DogstatsdAddr), DogstatsdTags: c.Telemetry.DogstatsdTags, RetryFailedConfiguration: boolVal(c.Telemetry.RetryFailedConfiguration), diff --git a/agent/config/config.go b/agent/config/config.go index 013f14dabf..3201f790a7 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -697,6 +697,7 @@ type Telemetry struct { CirconusSubmissionInterval *string `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"` CirconusSubmissionURL *string `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"` DisableHostname *bool `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"` + DisablePerTenancyUsageMetrics *bool `mapstructure:"disable_per_tenancy_usage_metrics" json:"disable_per_tenancy_usage_metrics,omitempty"` EnableHostMetrics *bool `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"` DogstatsdAddr *string `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"` DogstatsdTags []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"` diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index bc2222739c..257f320a55 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -6023,7 +6023,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { json: []string{` { "experiments": ["resource-apis"] - } + } `}, hcl: []string{`experiments=["resource-apis"]`}, expected: func(rt *RuntimeConfig) { @@ -6974,7 +6974,8 @@ func TestLoad_FullConfig(t *testing.T) { Expiration: 15 * time.Second, Name: "ftO6DySn", // notice this is the same as the metrics prefix }, - EnableHostMetrics: true, + EnableHostMetrics: true, + DisablePerTenancyUsageMetrics: true, }, TLS: tlsutil.Config{ InternalRPC: tlsutil.ProtocolConfig{ diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index da0eec7e24..5ead1e2f76 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -208,11 +208,11 @@ "ExposeMaxPort": 0, "ExposeMinPort": 0, "GRPCAddrs": [], + "GRPCKeepaliveInterval": "0s", + "GRPCKeepaliveTimeout": "0s", "GRPCPort": 0, "GRPCTLSAddrs": [], "GRPCTLSPort": 0, - "GRPCKeepaliveInterval": "0s", - "GRPCKeepaliveTimeout": "0s", "GossipLANGossipInterval": "0s", "GossipLANGossipNodes": 0, "GossipLANProbeInterval": "0s", @@ -472,6 +472,7 @@ "CirconusSubmissionURL": "", "Disable": false, "DisableHostname": false, + "DisablePerTenancyUsageMetrics": false, "DogstatsdAddr": "", "DogstatsdTags": [], "EnableHostMetrics": false, diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index 4c734265fd..bf31879627 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -718,6 +718,7 @@ telemetry { prometheus_retention_time = "15s" statsd_address = "drce87cy" statsite_address = "HpFwKB8R" + disable_per_tenancy_usage_metrics = true } tls { defaults { diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 30ede7dd18..5816a6713f 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -841,7 +841,8 @@ "metrics_prefix": "ftO6DySn", "prometheus_retention_time": "15s", "statsd_address": "drce87cy", - "statsite_address": "HpFwKB8R" + "statsite_address": "HpFwKB8R", + "disable_per_tenancy_usage_metrics": true }, "tls": { "defaults": { @@ -944,4 +945,4 @@ "xds": { "update_max_per_second": 9526.2 } -} \ No newline at end of file +} diff --git a/agent/consul/config.go b/agent/consul/config.go index a5ab21f731..03fb588eb0 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -404,6 +404,8 @@ type Config struct { // report usage metrics to the configured go-metrics Sinks. MetricsReportingInterval time.Duration + DisablePerTenancyUsageMetrics bool + // ConnectEnabled is whether to enable Connect features such as the CA. ConnectEnabled bool diff --git a/agent/consul/server.go b/agent/consul/server.go index 1aea2df125..cdd1904130 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -854,6 +854,7 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server, WithStateProvider(s.fsm). WithLogger(s.logger). WithDatacenter(s.config.Datacenter). + WithDisabledTenancyMetrics(s.config.DisablePerTenancyUsageMetrics). WithReportingInterval(s.config.MetricsReportingInterval). WithGetMembersFunc(func() []serf.Member { members, err := s.lanPoolAllMembers() diff --git a/agent/consul/usagemetrics/usagemetrics.go b/agent/consul/usagemetrics/usagemetrics.go index e55408f514..a0336d7e5d 100644 --- a/agent/consul/usagemetrics/usagemetrics.go +++ b/agent/consul/usagemetrics/usagemetrics.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/serf/serf" "github.com/hashicorp/consul/agent/consul/state" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/logging" "github.com/hashicorp/consul/version" ) @@ -76,6 +77,7 @@ type Config struct { stateProvider StateProvider tickerInterval time.Duration getMembersFunc getMembersFunc + excludeTenancy bool } // WithDatacenter adds the datacenter as a label to all metrics emitted by the @@ -85,6 +87,12 @@ func (c *Config) WithDatacenter(dc string) *Config { return c } +// WithDisabledTenancyMetrics opts the user out of specifying usage metrics for each tenancy. +func (c *Config) WithDisabledTenancyMetrics(disabled bool) *Config { + c.excludeTenancy = disabled + return c +} + // WithLogger takes a logger and creates a new, named sub-logger to use when // running func (c *Config) WithLogger(logger hclog.Logger) *Config { @@ -125,6 +133,9 @@ type UsageMetricsReporter struct { stateProvider StateProvider tickerInterval time.Duration getMembersFunc getMembersFunc + excludeTenancy bool + + usageReporter } func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) { @@ -151,8 +162,11 @@ func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) { metricLabels: cfg.metricLabels, tickerInterval: cfg.tickerInterval, getMembersFunc: cfg.getMembersFunc, + excludeTenancy: cfg.excludeTenancy, } + u.usageReporter = newTenancyUsageReporter(u) + return u, nil } @@ -190,7 +204,7 @@ func (u *UsageMetricsReporter) runOnce() { u.emitPeeringUsage(peeringUsage) - _, serviceUsage, err := state.ServiceUsage(nil, true) + _, serviceUsage, err := state.ServiceUsage(nil, !u.excludeTenancy) if err != nil { u.logger.Warn("failed to retrieve services from state store", "error", err) } @@ -259,3 +273,112 @@ func versionWithMetadata() string { return vsn } + +type usageReporter interface { + emitNodeUsage(nodeUsage state.NodeUsage) + emitPeeringUsage(peeringUsage state.PeeringUsage) + emitMemberUsage(members []serf.Member) + emitServiceUsage(serviceUsage structs.ServiceUsage) + emitKVUsage(kvUsage state.KVUsage) + emitConfigEntryUsage(configUsage state.ConfigEntryUsage) +} + +type baseUsageReporter struct { + metricLabels []metrics.Label +} + +var _ usageReporter = (*baseUsageReporter)(nil) + +func newBaseUsageReporter(u *UsageMetricsReporter) *baseUsageReporter { + return &baseUsageReporter{ + metricLabels: u.metricLabels, + } +} + +func (u *baseUsageReporter) emitNodeUsage(nodeUsage state.NodeUsage) { + metrics.SetGaugeWithLabels( + []string{"state", "nodes"}, + float32(nodeUsage.Nodes), + u.metricLabels, + ) +} + +func (u *baseUsageReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) { + metrics.SetGaugeWithLabels( + []string{"state", "peerings"}, + float32(peeringUsage.Peerings), + u.metricLabels, + ) +} + +func (u *baseUsageReporter) emitMemberUsage(members []serf.Member) { + var ( + servers int + clients int + ) + for _, m := range members { + switch m.Tags["role"] { + case "node": + clients++ + case "consul": + servers++ + } + } + + metrics.SetGaugeWithLabels( + []string{"members", "clients"}, + float32(clients), + u.metricLabels, + ) + + metrics.SetGaugeWithLabels( + []string{"members", "servers"}, + float32(servers), + u.metricLabels, + ) +} + +func (u *baseUsageReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) { + metrics.SetGaugeWithLabels( + []string{"state", "services"}, + float32(serviceUsage.Services), + u.metricLabels, + ) + + metrics.SetGaugeWithLabels( + []string{"state", "service_instances"}, + float32(serviceUsage.ServiceInstances), + u.metricLabels, + ) + metrics.SetGaugeWithLabels( + []string{"state", "billable_service_instances"}, + float32(serviceUsage.BillableServiceInstances), + u.metricLabels, + ) + + for k, i := range serviceUsage.ConnectServiceInstances { + metrics.SetGaugeWithLabels( + []string{"state", "connect_instances"}, + float32(i), + append(u.metricLabels, metrics.Label{Name: "kind", Value: k}), + ) + } +} + +func (u *baseUsageReporter) emitKVUsage(kvUsage state.KVUsage) { + metrics.SetGaugeWithLabels( + []string{"state", "kv_entries"}, + float32(kvUsage.KVCount), + u.metricLabels, + ) +} + +func (u *baseUsageReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) { + for k, i := range configUsage.ConfigByKind { + metrics.SetGaugeWithLabels( + []string{"state", "config_entries"}, + float32(i), + append(u.metricLabels, metrics.Label{Name: "kind", Value: k}), + ) + } +} diff --git a/agent/consul/usagemetrics/usagemetrics_ce.go b/agent/consul/usagemetrics/usagemetrics_ce.go index db11cb3353..519617b210 100644 --- a/agent/consul/usagemetrics/usagemetrics_ce.go +++ b/agent/consul/usagemetrics/usagemetrics_ce.go @@ -5,99 +5,6 @@ package usagemetrics -import ( - "github.com/armon/go-metrics" - - "github.com/hashicorp/serf/serf" - - "github.com/hashicorp/consul/agent/consul/state" - "github.com/hashicorp/consul/agent/structs" -) - -func (u *UsageMetricsReporter) emitNodeUsage(nodeUsage state.NodeUsage) { - metrics.SetGaugeWithLabels( - []string{"state", "nodes"}, - float32(nodeUsage.Nodes), - u.metricLabels, - ) -} - -func (u *UsageMetricsReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) { - metrics.SetGaugeWithLabels( - []string{"state", "peerings"}, - float32(peeringUsage.Peerings), - u.metricLabels, - ) -} - -func (u *UsageMetricsReporter) emitMemberUsage(members []serf.Member) { - var ( - servers int - clients int - ) - for _, m := range members { - switch m.Tags["role"] { - case "node": - clients++ - case "consul": - servers++ - } - } - - metrics.SetGaugeWithLabels( - []string{"members", "clients"}, - float32(clients), - u.metricLabels, - ) - - metrics.SetGaugeWithLabels( - []string{"members", "servers"}, - float32(servers), - u.metricLabels, - ) -} - -func (u *UsageMetricsReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) { - metrics.SetGaugeWithLabels( - []string{"state", "services"}, - float32(serviceUsage.Services), - u.metricLabels, - ) - - metrics.SetGaugeWithLabels( - []string{"state", "service_instances"}, - float32(serviceUsage.ServiceInstances), - u.metricLabels, - ) - metrics.SetGaugeWithLabels( - []string{"state", "billable_service_instances"}, - float32(serviceUsage.BillableServiceInstances), - u.metricLabels, - ) - - for k, i := range serviceUsage.ConnectServiceInstances { - metrics.SetGaugeWithLabels( - []string{"state", "connect_instances"}, - float32(i), - append(u.metricLabels, metrics.Label{Name: "kind", Value: k}), - ) - } -} - -func (u *UsageMetricsReporter) emitKVUsage(kvUsage state.KVUsage) { - metrics.SetGaugeWithLabels( - []string{"state", "kv_entries"}, - float32(kvUsage.KVCount), - u.metricLabels, - ) -} - -func (u *UsageMetricsReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) { - for k, i := range configUsage.ConfigByKind { - metrics.SetGaugeWithLabels( - []string{"state", "config_entries"}, - float32(i), - append(u.metricLabels, metrics.Label{Name: "kind", Value: k}), - ) - } +func newTenancyUsageReporter(u *UsageMetricsReporter) usageReporter { + return newBaseUsageReporter(u) } diff --git a/agent/consul/usagemetrics/usagemetrics_ce_test.go b/agent/consul/usagemetrics/usagemetrics_ce_test.go index 8048641d2e..1aa5867a5e 100644 --- a/agent/consul/usagemetrics/usagemetrics_ce_test.go +++ b/agent/consul/usagemetrics/usagemetrics_ce_test.go @@ -6,938 +6,46 @@ package usagemetrics import ( - "fmt" "testing" "time" "github.com/armon/go-metrics" - "github.com/hashicorp/serf/serf" "github.com/stretchr/testify/require" - "github.com/hashicorp/go-uuid" - - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/consul/state" - "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/consul/proto/private/pbpeering" "github.com/hashicorp/consul/sdk/testutil" - "github.com/hashicorp/consul/version" ) func newStateStore() (*state.Store, error) { return state.NewStateStore(nil), nil } -type testCase struct { - modfiyStateStore func(t *testing.T, s *state.Store) - getMembersFunc getMembersFunc - expectedGauges map[string]metrics.GaugeValue -} +func TestUsageReporter_CE(t *testing.T) { + getMetricsReporter := func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error) { + // Only have a single interval for the test + sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) + cfg := metrics.DefaultConfig("consul.usage.test") + cfg.EnableHostname = false + metrics.NewGlobal(cfg, sink) -var baseCases = map[string]testCase{ - "empty-state": { - expectedGauges: map[string]metrics.GaugeValue{ - // --- node --- - "consul.usage.test.state.nodes;datacenter=dc1": { - Name: "consul.usage.test.state.nodes", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- peering --- - "consul.usage.test.state.peerings;datacenter=dc1": { - Name: "consul.usage.test.state.peerings", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- member --- - "consul.usage.test.members.clients;datacenter=dc1": { - Name: "consul.usage.test.members.clients", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - "consul.usage.test.members.servers;datacenter=dc1": { - Name: "consul.usage.test.members.servers", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- service --- - "consul.usage.test.state.services;datacenter=dc1": { - Name: "consul.usage.test.state.services", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - "consul.usage.test.state.service_instances;datacenter=dc1": { - Name: "consul.usage.test.state.service_instances", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- service mesh --- - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-proxy"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "terminating-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "api-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "mesh-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-native"}, - }, - }, - "consul.usage.test.state.billable_service_instances;datacenter=dc1": { - Name: "consul.usage.test.state.billable_service_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - }, - }, - // --- kv --- - "consul.usage.test.state.kv_entries;datacenter=dc1": { - Name: "consul.usage.test.state.kv_entries", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- config entries --- - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-intentions"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-resolver"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-router"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-defaults"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-splitter"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "mesh"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "proxy-defaults"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "terminating-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "exported-services"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "sameness-group"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "api-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "bound-api-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "inline-certificate"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "http-route"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "tcp-route"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "jwt-provider"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "control-plane-request-limit"}, - }, - }, - // --- version --- - fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { - Name: "consul.usage.test.version", - Value: 1, - Labels: []metrics.Label{ - {Name: "version", Value: versionWithMetadata()}, - {Name: "pre_release", Value: version.VersionPrerelease}, - }, - }, - }, - getMembersFunc: func() []serf.Member { return []serf.Member{} }, - }, - "nodes": { - modfiyStateStore: func(t *testing.T, s *state.Store) { - require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"})) - require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"})) - }, - getMembersFunc: func() []serf.Member { - return []serf.Member{ - { - Name: "foo", - Tags: map[string]string{"role": "consul"}, - Status: serf.StatusAlive, - }, - { - Name: "bar", - Tags: map[string]string{"role": "consul"}, - Status: serf.StatusAlive, - }, - } - }, - expectedGauges: map[string]metrics.GaugeValue{ - // --- node --- - "consul.usage.test.state.nodes;datacenter=dc1": { - Name: "consul.usage.test.state.nodes", - Value: 2, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- peering --- - "consul.usage.test.state.peerings;datacenter=dc1": { - Name: "consul.usage.test.state.peerings", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- member --- - "consul.usage.test.members.servers;datacenter=dc1": { - Name: "consul.usage.test.members.servers", - Value: 2, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - "consul.usage.test.members.clients;datacenter=dc1": { - Name: "consul.usage.test.members.clients", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- service --- - "consul.usage.test.state.services;datacenter=dc1": { - Name: "consul.usage.test.state.services", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - "consul.usage.test.state.service_instances;datacenter=dc1": { - Name: "consul.usage.test.state.service_instances", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- service mesh --- - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-proxy"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "terminating-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "api-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "mesh-gateway"}, - }, - }, - "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": { - Name: "consul.usage.test.state.connect_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-native"}, - }, - }, - "consul.usage.test.state.billable_service_instances;datacenter=dc1": { - Name: "consul.usage.test.state.billable_service_instances", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - }, - }, - // --- kv --- - "consul.usage.test.state.kv_entries;datacenter=dc1": { - Name: "consul.usage.test.state.kv_entries", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - }, - // --- config entries --- - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-intentions"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-resolver"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-router"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-defaults"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "service-splitter"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "mesh"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "proxy-defaults"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "terminating-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "exported-services"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "sameness-group"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "api-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "bound-api-gateway"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "inline-certificate"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "http-route"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "tcp-route"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "jwt-provider"}, - }, - }, - "consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": { - Name: "consul.usage.test.state.config_entries", - Value: 0, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "control-plane-request-limit"}, - }, - }, - // --- version --- - fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { - Name: "consul.usage.test.version", - Value: 1, - Labels: []metrics.Label{ - {Name: "version", Value: versionWithMetadata()}, - {Name: "pre_release", Value: version.VersionPrerelease}, - }, - }, - }, - }, -} - -func TestUsageReporter_emitNodeUsage_CE(t *testing.T) { - cases := baseCases - - for name, tcase := range cases { - t.Run(name, func(t *testing.T) { - // Only have a single interval for the test - sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) - cfg := metrics.DefaultConfig("consul.usage.test") - cfg.EnableHostname = false - metrics.NewGlobal(cfg, sink) - - mockStateProvider := &mockStateProvider{} - s, err := newStateStore() - require.NoError(t, err) - if tcase.modfiyStateStore != nil { - tcase.modfiyStateStore(t, s) - } - mockStateProvider.On("State").Return(s) - - reporter, err := NewUsageMetricsReporter( - new(Config). - WithStateProvider(mockStateProvider). - WithLogger(testutil.Logger(t)). - WithDatacenter("dc1"). - WithGetMembersFunc(tcase.getMembersFunc), - ) - require.NoError(t, err) - - reporter.runOnce() - - intervals := sink.Data() - require.Len(t, intervals, 1) - intv := intervals[0] - - assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) - }) - } -} - -func TestUsageReporter_emitPeeringUsage_CE(t *testing.T) { - cases := make(map[string]testCase) - for k, v := range baseCases { - eg := make(map[string]metrics.GaugeValue) - for k, v := range v.expectedGauges { - eg[k] = v - } - cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg} - } - peeringsCase := cases["nodes"] - peeringsCase.modfiyStateStore = func(t *testing.T, s *state.Store) { - id, err := uuid.GenerateUUID() + mockStateProvider := &mockStateProvider{} + s, err := newStateStore() require.NoError(t, err) - require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}})) - id, err = uuid.GenerateUUID() - require.NoError(t, err) - require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}})) - id, err = uuid.GenerateUUID() - require.NoError(t, err) - require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}})) - } - peeringsCase.getMembersFunc = func() []serf.Member { - return []serf.Member{ - { - Name: "foo", - Tags: map[string]string{"role": "consul"}, - Status: serf.StatusAlive, - }, - { - Name: "bar", - Tags: map[string]string{"role": "consul"}, - Status: serf.StatusAlive, - }, + if tc.modifyStateStore != nil { + tc.modifyStateStore(t, s) } + mockStateProvider.On("State").Return(s) + + reporter, err := NewUsageMetricsReporter( + new(Config). + WithStateProvider(mockStateProvider). + WithLogger(testutil.Logger(t)). + WithDatacenter("dc1"). + WithGetMembersFunc(tc.getMembersFunc), + ) + + return reporter, sink, err } - peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.nodes", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.peerings", - Value: 3, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.members.clients", - Value: 0, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - cases["peerings"] = peeringsCase - delete(cases, "nodes") - for name, tcase := range cases { - t.Run(name, func(t *testing.T) { - // Only have a single interval for the test - sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) - cfg := metrics.DefaultConfig("consul.usage.test") - cfg.EnableHostname = false - metrics.NewGlobal(cfg, sink) - - mockStateProvider := &mockStateProvider{} - s, err := newStateStore() - require.NoError(t, err) - if tcase.modfiyStateStore != nil { - tcase.modfiyStateStore(t, s) - } - mockStateProvider.On("State").Return(s) - - reporter, err := NewUsageMetricsReporter( - new(Config). - WithStateProvider(mockStateProvider). - WithLogger(testutil.Logger(t)). - WithDatacenter("dc1"). - WithGetMembersFunc(tcase.getMembersFunc), - ) - require.NoError(t, err) - - reporter.runOnce() - - intervals := sink.Data() - require.Len(t, intervals, 1) - intv := intervals[0] - - assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) - }) - } -} - -func TestUsageReporter_emitServiceUsage_CE(t *testing.T) { - cases := make(map[string]testCase) - for k, v := range baseCases { - eg := make(map[string]metrics.GaugeValue) - for k, v := range v.expectedGauges { - eg[k] = v - } - cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg} - } - - nodesAndSvcsCase := cases["nodes"] - nodesAndSvcsCase.modfiyStateStore = func(t *testing.T, s *state.Store) { - require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"})) - require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"})) - require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"})) - require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"})) - - apigw := structs.TestNodeServiceAPIGateway(t) - apigw.ID = "api-gateway" - - mgw := structs.TestNodeServiceMeshGateway(t) - mgw.ID = "mesh-gateway" - - tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1") - tgw.ID = "terminating-gateway" - // Typical services and some consul services spread across two nodes - require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000})) - require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000})) - require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) - require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) - require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy})) - require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t))) - require.NoError(t, s.EnsureService(11, "foo", mgw)) - require.NoError(t, s.EnsureService(12, "foo", tgw)) - require.NoError(t, s.EnsureService(13, "foo", apigw)) - require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}})) - require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{ - Kind: structs.IngressGateway, - Name: "foo", - })) - require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{ - Kind: structs.IngressGateway, - Name: "bar", - })) - require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{ - Kind: structs.IngressGateway, - Name: "baz", - })) - } - baseCaseMembers := nodesAndSvcsCase.getMembersFunc() - nodesAndSvcsCase.getMembersFunc = func() []serf.Member { - baseCaseMembers = append(baseCaseMembers, serf.Member{ - Name: "baz", - Tags: map[string]string{"role": "node", "segment": "a"}, - Status: serf.StatusAlive, - }) - baseCaseMembers = append(baseCaseMembers, serf.Member{ - Name: "qux", - Tags: map[string]string{"role": "node", "segment": "b"}, - Status: serf.StatusAlive, - }) - return baseCaseMembers - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.nodes", - Value: 4, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.members.clients", - Value: 2, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.services", - Value: 8, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.service_instances", - Value: 10, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-proxy"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "terminating-gateway"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "api-gateway"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "mesh-gateway"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.connect_instances", - Value: 1, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "connect-native"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.billable_service_instances", - Value: 3, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - }, - } - nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.config_entries", - Value: 3, - Labels: []metrics.Label{ - {Name: "datacenter", Value: "dc1"}, - {Name: "kind", Value: "ingress-gateway"}, - }, - } - cases["nodes-and-services"] = nodesAndSvcsCase - delete(cases, "nodes") - - for name, tcase := range cases { - t.Run(name, func(t *testing.T) { - // Only have a single interval for the test - sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) - cfg := metrics.DefaultConfig("consul.usage.test") - cfg.EnableHostname = false - metrics.NewGlobal(cfg, sink) - - mockStateProvider := &mockStateProvider{} - s := state.NewStateStore(nil) - if tcase.modfiyStateStore != nil { - tcase.modfiyStateStore(t, s) - } - mockStateProvider.On("State").Return(s) - - reporter, err := NewUsageMetricsReporter( - new(Config). - WithStateProvider(mockStateProvider). - WithLogger(testutil.Logger(t)). - WithDatacenter("dc1"). - WithGetMembersFunc(tcase.getMembersFunc), - ) - require.NoError(t, err) - - reporter.runOnce() - - intervals := sink.Data() - require.Len(t, intervals, 1) - intv := intervals[0] - - assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) - }) - } -} - -func TestUsageReporter_emitKVUsage_CE(t *testing.T) { - cases := make(map[string]testCase) - for k, v := range baseCases { - eg := make(map[string]metrics.GaugeValue) - for k, v := range v.expectedGauges { - eg[k] = v - } - cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg} - } - - nodesCase := cases["nodes"] - mss := nodesCase.modfiyStateStore - nodesCase.modfiyStateStore = func(t *testing.T, s *state.Store) { - mss(t, s) - require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}})) - require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}})) - require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}})) - require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}})) - require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{})) - require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{})) - require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}})) - require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}})) - } - nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{ - Name: "consul.usage.test.state.kv_entries", - Value: 4, - Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, - } - cases["nodes"] = nodesCase - - for name, tcase := range cases { - t.Run(name, func(t *testing.T) { - // Only have a single interval for the test - sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute) - cfg := metrics.DefaultConfig("consul.usage.test") - cfg.EnableHostname = false - metrics.NewGlobal(cfg, sink) - - mockStateProvider := &mockStateProvider{} - s, err := newStateStore() - require.NoError(t, err) - if tcase.modfiyStateStore != nil { - tcase.modfiyStateStore(t, s) - } - mockStateProvider.On("State").Return(s) - - reporter, err := NewUsageMetricsReporter( - new(Config). - WithStateProvider(mockStateProvider). - WithLogger(testutil.Logger(t)). - WithDatacenter("dc1"). - WithGetMembersFunc(tcase.getMembersFunc), - ) - require.NoError(t, err) - - reporter.runOnce() - - intervals := sink.Data() - require.Len(t, intervals, 1) - intv := intervals[0] - - assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) - }) - } + testUsageReporter_Tenantless(t, getMetricsReporter) } diff --git a/agent/consul/usagemetrics/usagemetrics_test.go b/agent/consul/usagemetrics/usagemetrics_test.go index efe561acd0..b9a5669a6a 100644 --- a/agent/consul/usagemetrics/usagemetrics_test.go +++ b/agent/consul/usagemetrics/usagemetrics_test.go @@ -10,15 +10,19 @@ import ( "github.com/armon/go-metrics" "github.com/armon/go-metrics/prometheus" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-uuid" + "github.com/hashicorp/serf/serf" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/consul/state" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/lib" - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/serf/serf" + "github.com/hashicorp/consul/proto/private/pbpeering" + "github.com/hashicorp/consul/version" ) type mockStateProvider struct { @@ -119,3 +123,856 @@ type benchStateProvider func() *state.Store func (b benchStateProvider) State() *state.Store { return b() } + +type testCase struct { + modifyStateStore func(t *testing.T, s *state.Store) + getMembersFunc getMembersFunc + expectedGauges map[string]metrics.GaugeValue +} + +var baseCases = map[string]testCase{ + "empty-state": { + expectedGauges: map[string]metrics.GaugeValue{ + // --- node --- + "consul.usage.test.state.nodes;datacenter=dc1": { + Name: "consul.usage.test.state.nodes", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- peering --- + "consul.usage.test.state.peerings;datacenter=dc1": { + Name: "consul.usage.test.state.peerings", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- member --- + "consul.usage.test.members.clients;datacenter=dc1": { + Name: "consul.usage.test.members.clients", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.members.servers;datacenter=dc1": { + Name: "consul.usage.test.members.servers", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service --- + "consul.usage.test.state.services;datacenter=dc1": { + Name: "consul.usage.test.state.services", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.state.service_instances;datacenter=dc1": { + Name: "consul.usage.test.state.service_instances", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service mesh --- + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-proxy"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "terminating-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "mesh-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-native"}, + }, + }, + "consul.usage.test.state.billable_service_instances;datacenter=dc1": { + Name: "consul.usage.test.state.billable_service_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + }, + }, + // --- kv --- + "consul.usage.test.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- config entries --- + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-intentions"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-resolver"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-router"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-defaults"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-splitter"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "mesh"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "proxy-defaults"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "terminating-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "exported-services"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "sameness-group"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "bound-api-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "inline-certificate"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "http-route"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "tcp-route"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "jwt-provider"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "control-plane-request-limit"}, + }, + }, + // --- version --- + fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { + Name: "consul.usage.test.version", + Value: 1, + Labels: []metrics.Label{ + {Name: "version", Value: versionWithMetadata()}, + {Name: "pre_release", Value: version.VersionPrerelease}, + }, + }, + }, + getMembersFunc: func() []serf.Member { return []serf.Member{} }, + }, + "nodes": { + modifyStateStore: func(t *testing.T, s *state.Store) { + require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"})) + require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"})) + }, + getMembersFunc: func() []serf.Member { + return []serf.Member{ + { + Name: "foo", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + { + Name: "bar", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + } + }, + expectedGauges: map[string]metrics.GaugeValue{ + // --- node --- + "consul.usage.test.state.nodes;datacenter=dc1": { + Name: "consul.usage.test.state.nodes", + Value: 2, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- peering --- + "consul.usage.test.state.peerings;datacenter=dc1": { + Name: "consul.usage.test.state.peerings", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- member --- + "consul.usage.test.members.servers;datacenter=dc1": { + Name: "consul.usage.test.members.servers", + Value: 2, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.members.clients;datacenter=dc1": { + Name: "consul.usage.test.members.clients", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service --- + "consul.usage.test.state.services;datacenter=dc1": { + Name: "consul.usage.test.state.services", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + "consul.usage.test.state.service_instances;datacenter=dc1": { + Name: "consul.usage.test.state.service_instances", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- service mesh --- + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-proxy"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "terminating-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "mesh-gateway"}, + }, + }, + "consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": { + Name: "consul.usage.test.state.connect_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-native"}, + }, + }, + "consul.usage.test.state.billable_service_instances;datacenter=dc1": { + Name: "consul.usage.test.state.billable_service_instances", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + }, + }, + // --- kv --- + "consul.usage.test.state.kv_entries;datacenter=dc1": { + Name: "consul.usage.test.state.kv_entries", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + }, + // --- config entries --- + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-intentions"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-resolver"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-router"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-defaults"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "service-splitter"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "mesh"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "proxy-defaults"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "terminating-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "exported-services"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "sameness-group"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "bound-api-gateway"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "inline-certificate"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "http-route"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "tcp-route"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "jwt-provider"}, + }, + }, + "consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": { + Name: "consul.usage.test.state.config_entries", + Value: 0, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "control-plane-request-limit"}, + }, + }, + // --- version --- + fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): { + Name: "consul.usage.test.version", + Value: 1, + Labels: []metrics.Label{ + {Name: "version", Value: versionWithMetadata()}, + {Name: "pre_release", Value: version.VersionPrerelease}, + }, + }, + }, + }, +} + +func testUsageReporter_Tenantless(t *testing.T, getMetricsReporter func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) { + t.Run("emitNodeUsage", func(t *testing.T) { + testUsageReporter_emitNodeUsage_CE(t, getMetricsReporter) + }) + + t.Run("emitPeeringUsage", func(t *testing.T) { + testUsageReporter_emitPeeringUsage_CE(t, getMetricsReporter) + }) + + t.Run("emitServiceUsage", func(t *testing.T) { + testUsageReporter_emitServiceUsage_CE(t, getMetricsReporter) + }) + + t.Run("emitKVUsage", func(t *testing.T) { + testUsageReporter_emitKVUsage_CE(t, getMetricsReporter) + }) +} + +func testUsageReporter_emitNodeUsage_CE(t *testing.T, getReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) { + cases := baseCases + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + reporter, sink, err := getReporter(tcase) + require.NoError(t, err) + + reporter.runOnce() + + intervals := sink.Data() + require.Len(t, intervals, 1) + intv := intervals[0] + + assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) + }) + } +} + +func testUsageReporter_emitPeeringUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) { + cases := make(map[string]testCase) + for k, v := range baseCases { + eg := make(map[string]metrics.GaugeValue) + for k, v := range v.expectedGauges { + eg[k] = v + } + cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg} + } + peeringsCase := cases["nodes"] + peeringsCase.modifyStateStore = func(t *testing.T, s *state.Store) { + id, err := uuid.GenerateUUID() + require.NoError(t, err) + require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}})) + id, err = uuid.GenerateUUID() + require.NoError(t, err) + require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}})) + id, err = uuid.GenerateUUID() + require.NoError(t, err) + require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}})) + } + peeringsCase.getMembersFunc = func() []serf.Member { + return []serf.Member{ + { + Name: "foo", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + { + Name: "bar", + Tags: map[string]string{"role": "consul"}, + Status: serf.StatusAlive, + }, + } + } + peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.nodes", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.peerings", + Value: 3, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.members.clients", + Value: 0, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + cases["peerings"] = peeringsCase + delete(cases, "nodes") + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + reporter, sink, err := getMetricsReporter(tcase) + require.NoError(t, err) + + reporter.runOnce() + + intervals := sink.Data() + require.Len(t, intervals, 1) + intv := intervals[0] + + assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) + }) + } +} + +func testUsageReporter_emitServiceUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) { + cases := make(map[string]testCase) + for k, v := range baseCases { + eg := make(map[string]metrics.GaugeValue) + for k, v := range v.expectedGauges { + eg[k] = v + } + cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg} + } + + nodesAndSvcsCase := cases["nodes"] + nodesAndSvcsCase.modifyStateStore = func(t *testing.T, s *state.Store) { + require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"})) + require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"})) + require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"})) + require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"})) + + apigw := structs.TestNodeServiceAPIGateway(t) + apigw.ID = "api-gateway" + + mgw := structs.TestNodeServiceMeshGateway(t) + mgw.ID = "mesh-gateway" + + tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1") + tgw.ID = "terminating-gateway" + // Typical services and some consul services spread across two nodes + require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000})) + require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000})) + require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) + require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil})) + require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy})) + require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t))) + require.NoError(t, s.EnsureService(11, "foo", mgw)) + require.NoError(t, s.EnsureService(12, "foo", tgw)) + require.NoError(t, s.EnsureService(13, "foo", apigw)) + require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}})) + require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{ + Kind: structs.IngressGateway, + Name: "foo", + })) + require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{ + Kind: structs.IngressGateway, + Name: "bar", + })) + require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{ + Kind: structs.IngressGateway, + Name: "baz", + })) + } + baseCaseMembers := nodesAndSvcsCase.getMembersFunc() + nodesAndSvcsCase.getMembersFunc = func() []serf.Member { + baseCaseMembers = append(baseCaseMembers, serf.Member{ + Name: "baz", + Tags: map[string]string{"role": "node", "segment": "a"}, + Status: serf.StatusAlive, + }) + baseCaseMembers = append(baseCaseMembers, serf.Member{ + Name: "qux", + Tags: map[string]string{"role": "node", "segment": "b"}, + Status: serf.StatusAlive, + }) + return baseCaseMembers + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.nodes", + Value: 4, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.members.clients", + Value: 2, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.services", + Value: 8, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.service_instances", + Value: 10, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-proxy"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "terminating-gateway"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "api-gateway"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "mesh-gateway"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.connect_instances", + Value: 1, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "connect-native"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.billable_service_instances", + Value: 3, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + }, + } + nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.config_entries", + Value: 3, + Labels: []metrics.Label{ + {Name: "datacenter", Value: "dc1"}, + {Name: "kind", Value: "ingress-gateway"}, + }, + } + cases["nodes-and-services"] = nodesAndSvcsCase + delete(cases, "nodes") + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + reporter, sink, err := getMetricsReporter(tcase) + require.NoError(t, err) + + reporter.runOnce() + + intervals := sink.Data() + require.Len(t, intervals, 1) + intv := intervals[0] + + assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) + }) + } +} + +func testUsageReporter_emitKVUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) { + cases := make(map[string]testCase) + for k, v := range baseCases { + eg := make(map[string]metrics.GaugeValue) + for k, v := range v.expectedGauges { + eg[k] = v + } + cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg} + } + + nodesCase := cases["nodes"] + mss := nodesCase.modifyStateStore + nodesCase.modifyStateStore = func(t *testing.T, s *state.Store) { + mss(t, s) + require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}})) + require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}})) + require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}})) + require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}})) + require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{})) + require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{})) + require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}})) + require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}})) + } + nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{ + Name: "consul.usage.test.state.kv_entries", + Value: 4, + Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}}, + } + cases["nodes"] = nodesCase + + for name, tcase := range cases { + t.Run(name, func(t *testing.T) { + reporter, sink, err := getMetricsReporter(tcase) + require.NoError(t, err) + + reporter.runOnce() + + intervals := sink.Data() + require.Len(t, intervals, 1) + intv := intervals[0] + + assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges) + }) + } +} diff --git a/lib/telemetry.go b/lib/telemetry.go index bb6f4624ae..91b9d39a4d 100644 --- a/lib/telemetry.go +++ b/lib/telemetry.go @@ -146,9 +146,14 @@ type TelemetryConfig struct { // DisableHostname will disable hostname prefixing for all metrics. // - // hcl: telemetry { disable_hostname = (true|false) + // hcl: telemetry { disable_hostname = (true|false) } DisableHostname bool `json:"disable_hostname,omitempty" mapstructure:"disable_hostname"` + // DisablePerTenancyUsageMetrics will disable setting tenancy labels on usage metrics. + // + // hcl: telemetry { disable_per_tenancy_usage_metrics = (true|false) } + DisablePerTenancyUsageMetrics bool `json:"disable_per_tenancy_usage_metrics,omitempty" mapstructure:"disable_per_tenancy_usage_metrics"` + // DogStatsdAddr is the address of a dogstatsd instance. If provided, // metrics will be sent to that instance // diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 75775c7508..685512db46 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -1842,6 +1842,9 @@ subsystem that provides Consul's service mesh capabilities. - `disable_hostname` ((#telemetry-disable_hostname)) Set to `true` to stop prepending the machine's hostname to gauge-type metrics. Default is `false`. + - `disable_per_tenancy_usage_metrics` ((#telemetry-disable_per_tenancy_usage_metrics)) + Set to `true` to exclude tenancy labels from usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces. + - `dogstatsd_addr` ((#telemetry-dogstatsd_addr)) This provides the address of a DogStatsD instance in the format `host:port`. DogStatsD is a protocol-compatible flavor of statsd, with the added ability to decorate metrics with tags and event