Add an agent config option to diable per tenancy usage metrics. (#20976)

.changelog/_9142.txt

@@ -0,0 +1,3 @@
+```release-note: improvement
+telemetry: Add `telemetry.disable_per_tenancy_usage_metrics` in agent configuration to disable setting tenancy labels on usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces.
+```

agent/config/builder.go

@@ -947,6 +947,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 			CirconusSubmissionInterval:         stringVal(c.Telemetry.CirconusSubmissionInterval),
 			CirconusSubmissionURL:              stringVal(c.Telemetry.CirconusSubmissionURL),
 			DisableHostname:                    boolVal(c.Telemetry.DisableHostname),
+			DisablePerTenancyUsageMetrics:      boolVal(c.Telemetry.DisablePerTenancyUsageMetrics),
 			DogstatsdAddr:                      stringVal(c.Telemetry.DogstatsdAddr),
 			DogstatsdTags:                      c.Telemetry.DogstatsdTags,
 			RetryFailedConfiguration:           boolVal(c.Telemetry.RetryFailedConfiguration),

agent/config/config.go

@@ -697,6 +697,7 @@ type Telemetry struct {
 	CirconusSubmissionInterval         *string  `mapstructure:"circonus_submission_interval" json:"circonus_submission_interval,omitempty"`
 	CirconusSubmissionURL              *string  `mapstructure:"circonus_submission_url" json:"circonus_submission_url,omitempty"`
 	DisableHostname                    *bool    `mapstructure:"disable_hostname" json:"disable_hostname,omitempty"`
+	DisablePerTenancyUsageMetrics      *bool    `mapstructure:"disable_per_tenancy_usage_metrics" json:"disable_per_tenancy_usage_metrics,omitempty"`
 	EnableHostMetrics                  *bool    `mapstructure:"enable_host_metrics" json:"enable_host_metrics,omitempty"`
 	DogstatsdAddr                      *string  `mapstructure:"dogstatsd_addr" json:"dogstatsd_addr,omitempty"`
 	DogstatsdTags                      []string `mapstructure:"dogstatsd_tags" json:"dogstatsd_tags,omitempty"`

agent/config/runtime_test.go

@@ -6975,6 +6975,7 @@ func TestLoad_FullConfig(t *testing.T) {
 				Name:       "ftO6DySn", // notice this is the same as the metrics prefix
 			},
 			EnableHostMetrics:             true,
+			DisablePerTenancyUsageMetrics: true,
 		},
 		TLS: tlsutil.Config{
 			InternalRPC: tlsutil.ProtocolConfig{

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -208,11 +208,11 @@
     "ExposeMaxPort": 0,
     "ExposeMinPort": 0,
     "GRPCAddrs": [],
+    "GRPCKeepaliveInterval": "0s",
+    "GRPCKeepaliveTimeout": "0s",
     "GRPCPort": 0,
     "GRPCTLSAddrs": [],
     "GRPCTLSPort": 0,
-    "GRPCKeepaliveInterval": "0s",
-    "GRPCKeepaliveTimeout": "0s",
     "GossipLANGossipInterval": "0s",
     "GossipLANGossipNodes": 0,
     "GossipLANProbeInterval": "0s",
@@ -472,6 +472,7 @@
         "CirconusSubmissionURL": "",
         "Disable": false,
         "DisableHostname": false,
+        "DisablePerTenancyUsageMetrics": false,
         "DogstatsdAddr": "",
         "DogstatsdTags": [],
         "EnableHostMetrics": false,

agent/config/testdata/full-config.hcl

@@ -718,6 +718,7 @@ telemetry {
     prometheus_retention_time = "15s"
     statsd_address = "drce87cy"
     statsite_address = "HpFwKB8R"
+    disable_per_tenancy_usage_metrics = true
 }
 tls {
     defaults {

agent/config/testdata/full-config.json

@@ -841,7 +841,8 @@
     "metrics_prefix": "ftO6DySn",
     "prometheus_retention_time": "15s",
     "statsd_address": "drce87cy",
-    "statsite_address": "HpFwKB8R"
+    "statsite_address": "HpFwKB8R",
+    "disable_per_tenancy_usage_metrics": true
   },
   "tls": {
     "defaults": {

agent/consul/config.go

@@ -404,6 +404,8 @@ type Config struct {
 	// report usage metrics to the configured go-metrics Sinks.
 	MetricsReportingInterval time.Duration
 
+	DisablePerTenancyUsageMetrics bool
+
 	// ConnectEnabled is whether to enable Connect features such as the CA.
 	ConnectEnabled bool
 

agent/consul/server.go

@@ -854,6 +854,7 @@ func NewServer(config *Config, flat Deps, externalGRPCServer *grpc.Server,
 			WithStateProvider(s.fsm).
 			WithLogger(s.logger).
 			WithDatacenter(s.config.Datacenter).
+			WithDisabledTenancyMetrics(s.config.DisablePerTenancyUsageMetrics).
 			WithReportingInterval(s.config.MetricsReportingInterval).
 			WithGetMembersFunc(func() []serf.Member {
 				members, err := s.lanPoolAllMembers()

agent/consul/usagemetrics/usagemetrics.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/serf/serf"
 
 	"github.com/hashicorp/consul/agent/consul/state"
+	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/logging"
 	"github.com/hashicorp/consul/version"
 )
@@ -76,6 +77,7 @@ type Config struct {
 	stateProvider  StateProvider
 	tickerInterval time.Duration
 	getMembersFunc getMembersFunc
+	excludeTenancy bool
 }
 
 // WithDatacenter adds the datacenter as a label to all metrics emitted by the
@@ -85,6 +87,12 @@ func (c *Config) WithDatacenter(dc string) *Config {
 	return c
 }
 
+// WithDisabledTenancyMetrics opts the user out of specifying usage metrics for each tenancy.
+func (c *Config) WithDisabledTenancyMetrics(disabled bool) *Config {
+	c.excludeTenancy = disabled
+	return c
+}
+
 // WithLogger takes a logger and creates a new, named sub-logger to use when
 // running
 func (c *Config) WithLogger(logger hclog.Logger) *Config {
@@ -125,6 +133,9 @@ type UsageMetricsReporter struct {
 	stateProvider  StateProvider
 	tickerInterval time.Duration
 	getMembersFunc getMembersFunc
+	excludeTenancy bool
+
+	usageReporter
 }
 
 func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) {
@@ -151,8 +162,11 @@ func NewUsageMetricsReporter(cfg *Config) (*UsageMetricsReporter, error) {
 		metricLabels:   cfg.metricLabels,
 		tickerInterval: cfg.tickerInterval,
 		getMembersFunc: cfg.getMembersFunc,
+		excludeTenancy: cfg.excludeTenancy,
 	}
 
+	u.usageReporter = newTenancyUsageReporter(u)
+
 	return u, nil
 }
 
@@ -190,7 +204,7 @@ func (u *UsageMetricsReporter) runOnce() {
 
 	u.emitPeeringUsage(peeringUsage)
 
-	_, serviceUsage, err := state.ServiceUsage(nil, true)
+	_, serviceUsage, err := state.ServiceUsage(nil, !u.excludeTenancy)
 	if err != nil {
 		u.logger.Warn("failed to retrieve services from state store", "error", err)
 	}
@@ -259,3 +273,112 @@ func versionWithMetadata() string {
 
 	return vsn
 }
+
+type usageReporter interface {
+	emitNodeUsage(nodeUsage state.NodeUsage)
+	emitPeeringUsage(peeringUsage state.PeeringUsage)
+	emitMemberUsage(members []serf.Member)
+	emitServiceUsage(serviceUsage structs.ServiceUsage)
+	emitKVUsage(kvUsage state.KVUsage)
+	emitConfigEntryUsage(configUsage state.ConfigEntryUsage)
+}
+
+type baseUsageReporter struct {
+	metricLabels []metrics.Label
+}
+
+var _ usageReporter = (*baseUsageReporter)(nil)
+
+func newBaseUsageReporter(u *UsageMetricsReporter) *baseUsageReporter {
+	return &baseUsageReporter{
+		metricLabels: u.metricLabels,
+	}
+}
+
+func (u *baseUsageReporter) emitNodeUsage(nodeUsage state.NodeUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "nodes"},
+		float32(nodeUsage.Nodes),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "peerings"},
+		float32(peeringUsage.Peerings),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitMemberUsage(members []serf.Member) {
+	var (
+		servers int
+		clients int
+	)
+	for _, m := range members {
+		switch m.Tags["role"] {
+		case "node":
+			clients++
+		case "consul":
+			servers++
+		}
+	}
+
+	metrics.SetGaugeWithLabels(
+		[]string{"members", "clients"},
+		float32(clients),
+		u.metricLabels,
+	)
+
+	metrics.SetGaugeWithLabels(
+		[]string{"members", "servers"},
+		float32(servers),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "services"},
+		float32(serviceUsage.Services),
+		u.metricLabels,
+	)
+
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "service_instances"},
+		float32(serviceUsage.ServiceInstances),
+		u.metricLabels,
+	)
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "billable_service_instances"},
+		float32(serviceUsage.BillableServiceInstances),
+		u.metricLabels,
+	)
+
+	for k, i := range serviceUsage.ConnectServiceInstances {
+		metrics.SetGaugeWithLabels(
+			[]string{"state", "connect_instances"},
+			float32(i),
+			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
+		)
+	}
+}
+
+func (u *baseUsageReporter) emitKVUsage(kvUsage state.KVUsage) {
+	metrics.SetGaugeWithLabels(
+		[]string{"state", "kv_entries"},
+		float32(kvUsage.KVCount),
+		u.metricLabels,
+	)
+}
+
+func (u *baseUsageReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) {
+	for k, i := range configUsage.ConfigByKind {
+		metrics.SetGaugeWithLabels(
+			[]string{"state", "config_entries"},
+			float32(i),
+			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
+		)
+	}
+}

agent/consul/usagemetrics/usagemetrics_ce.go

@@ -5,99 +5,6 @@
 
 package usagemetrics
 
-import (
+func newTenancyUsageReporter(u *UsageMetricsReporter) usageReporter {
-	"github.com/armon/go-metrics"
+	return newBaseUsageReporter(u)
-
-	"github.com/hashicorp/serf/serf"
-
-	"github.com/hashicorp/consul/agent/consul/state"
-	"github.com/hashicorp/consul/agent/structs"
-)
-
-func (u *UsageMetricsReporter) emitNodeUsage(nodeUsage state.NodeUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "nodes"},
-		float32(nodeUsage.Nodes),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitPeeringUsage(peeringUsage state.PeeringUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "peerings"},
-		float32(peeringUsage.Peerings),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitMemberUsage(members []serf.Member) {
-	var (
-		servers int
-		clients int
-	)
-	for _, m := range members {
-		switch m.Tags["role"] {
-		case "node":
-			clients++
-		case "consul":
-			servers++
-		}
-	}
-
-	metrics.SetGaugeWithLabels(
-		[]string{"members", "clients"},
-		float32(clients),
-		u.metricLabels,
-	)
-
-	metrics.SetGaugeWithLabels(
-		[]string{"members", "servers"},
-		float32(servers),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitServiceUsage(serviceUsage structs.ServiceUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "services"},
-		float32(serviceUsage.Services),
-		u.metricLabels,
-	)
-
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "service_instances"},
-		float32(serviceUsage.ServiceInstances),
-		u.metricLabels,
-	)
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "billable_service_instances"},
-		float32(serviceUsage.BillableServiceInstances),
-		u.metricLabels,
-	)
-
-	for k, i := range serviceUsage.ConnectServiceInstances {
-		metrics.SetGaugeWithLabels(
-			[]string{"state", "connect_instances"},
-			float32(i),
-			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
-		)
-	}
-}
-
-func (u *UsageMetricsReporter) emitKVUsage(kvUsage state.KVUsage) {
-	metrics.SetGaugeWithLabels(
-		[]string{"state", "kv_entries"},
-		float32(kvUsage.KVCount),
-		u.metricLabels,
-	)
-}
-
-func (u *UsageMetricsReporter) emitConfigEntryUsage(configUsage state.ConfigEntryUsage) {
-	for k, i := range configUsage.ConfigByKind {
-		metrics.SetGaugeWithLabels(
-			[]string{"state", "config_entries"},
-			float32(i),
-			append(u.metricLabels, metrics.Label{Name: "kind", Value: k}),
-		)
-	}
 }

agent/consul/usagemetrics/usagemetrics_ce_test.go

@@ -6,569 +6,22 @@
 package usagemetrics
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
 	"github.com/armon/go-metrics"
-	"github.com/hashicorp/serf/serf"
 	"github.com/stretchr/testify/require"
 
-	"github.com/hashicorp/go-uuid"
-
-	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/consul/state"
-	"github.com/hashicorp/consul/agent/structs"
-	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/sdk/testutil"
-	"github.com/hashicorp/consul/version"
 )
 
 func newStateStore() (*state.Store, error) {
 	return state.NewStateStore(nil), nil
 }
 
-type testCase struct {
+func TestUsageReporter_CE(t *testing.T) {
-	modfiyStateStore func(t *testing.T, s *state.Store)
+	getMetricsReporter := func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error) {
-	getMembersFunc   getMembersFunc
-	expectedGauges   map[string]metrics.GaugeValue
-}
-
-var baseCases = map[string]testCase{
-	"empty-state": {
-		expectedGauges: map[string]metrics.GaugeValue{
-			// --- node ---
-			"consul.usage.test.state.nodes;datacenter=dc1": {
-				Name:   "consul.usage.test.state.nodes",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- peering ---
-			"consul.usage.test.state.peerings;datacenter=dc1": {
-				Name:   "consul.usage.test.state.peerings",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- member ---
-			"consul.usage.test.members.clients;datacenter=dc1": {
-				Name:   "consul.usage.test.members.clients",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.members.servers;datacenter=dc1": {
-				Name:   "consul.usage.test.members.servers",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service ---
-			"consul.usage.test.state.services;datacenter=dc1": {
-				Name:   "consul.usage.test.state.services",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.state.service_instances;datacenter=dc1": {
-				Name:   "consul.usage.test.state.service_instances",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service mesh ---
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-proxy"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-native"},
-				},
-			},
-			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
-				Name:  "consul.usage.test.state.billable_service_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-				},
-			},
-			// --- kv ---
-			"consul.usage.test.state.kv_entries;datacenter=dc1": {
-				Name:   "consul.usage.test.state.kv_entries",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- config entries ---
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-intentions"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-resolver"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-router"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-splitter"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "proxy-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "exported-services"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "sameness-group"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "bound-api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "inline-certificate"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "http-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "tcp-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "jwt-provider"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "control-plane-request-limit"},
-				},
-			},
-			// --- version ---
-			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
-				Name:  "consul.usage.test.version",
-				Value: 1,
-				Labels: []metrics.Label{
-					{Name: "version", Value: versionWithMetadata()},
-					{Name: "pre_release", Value: version.VersionPrerelease},
-				},
-			},
-		},
-		getMembersFunc: func() []serf.Member { return []serf.Member{} },
-	},
-	"nodes": {
-		modfiyStateStore: func(t *testing.T, s *state.Store) {
-			require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
-			require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
-		},
-		getMembersFunc: func() []serf.Member {
-			return []serf.Member{
-				{
-					Name:   "foo",
-					Tags:   map[string]string{"role": "consul"},
-					Status: serf.StatusAlive,
-				},
-				{
-					Name:   "bar",
-					Tags:   map[string]string{"role": "consul"},
-					Status: serf.StatusAlive,
-				},
-			}
-		},
-		expectedGauges: map[string]metrics.GaugeValue{
-			// --- node ---
-			"consul.usage.test.state.nodes;datacenter=dc1": {
-				Name:   "consul.usage.test.state.nodes",
-				Value:  2,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- peering ---
-			"consul.usage.test.state.peerings;datacenter=dc1": {
-				Name:   "consul.usage.test.state.peerings",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- member ---
-			"consul.usage.test.members.servers;datacenter=dc1": {
-				Name:   "consul.usage.test.members.servers",
-				Value:  2,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.members.clients;datacenter=dc1": {
-				Name:   "consul.usage.test.members.clients",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service ---
-			"consul.usage.test.state.services;datacenter=dc1": {
-				Name:   "consul.usage.test.state.services",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			"consul.usage.test.state.service_instances;datacenter=dc1": {
-				Name:   "consul.usage.test.state.service_instances",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- service mesh ---
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-proxy"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh-gateway"},
-				},
-			},
-			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
-				Name:  "consul.usage.test.state.connect_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "connect-native"},
-				},
-			},
-			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
-				Name:  "consul.usage.test.state.billable_service_instances",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-				},
-			},
-			// --- kv ---
-			"consul.usage.test.state.kv_entries;datacenter=dc1": {
-				Name:   "consul.usage.test.state.kv_entries",
-				Value:  0,
-				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-			},
-			// --- config entries ---
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-intentions"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-resolver"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-router"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "ingress-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "service-splitter"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "mesh"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "proxy-defaults"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "terminating-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "exported-services"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "sameness-group"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "bound-api-gateway"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "inline-certificate"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "http-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "tcp-route"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "jwt-provider"},
-				},
-			},
-			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
-				Name:  "consul.usage.test.state.config_entries",
-				Value: 0,
-				Labels: []metrics.Label{
-					{Name: "datacenter", Value: "dc1"},
-					{Name: "kind", Value: "control-plane-request-limit"},
-				},
-			},
-			// --- version ---
-			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
-				Name:  "consul.usage.test.version",
-				Value: 1,
-				Labels: []metrics.Label{
-					{Name: "version", Value: versionWithMetadata()},
-					{Name: "pre_release", Value: version.VersionPrerelease},
-				},
-			},
-		},
-	},
-}
-
-func TestUsageReporter_emitNodeUsage_CE(t *testing.T) {
-	cases := baseCases
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
 		// Only have a single interval for the test
 		sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
 		cfg := metrics.DefaultConfig("consul.usage.test")
@@ -578,8 +31,8 @@ func TestUsageReporter_emitNodeUsage_CE(t *testing.T) {
 		mockStateProvider := &mockStateProvider{}
 		s, err := newStateStore()
 		require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
+		if tc.modifyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
+			tc.modifyStateStore(t, s)
 		}
 		mockStateProvider.On("State").Return(s)
 
@@ -588,356 +41,11 @@ func TestUsageReporter_emitNodeUsage_CE(t *testing.T) {
 				WithStateProvider(mockStateProvider).
 				WithLogger(testutil.Logger(t)).
 				WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
+				WithGetMembersFunc(tc.getMembersFunc),
 		)
-			require.NoError(t, err)
 
-			reporter.runOnce()
+		return reporter, sink, err
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitPeeringUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-	peeringsCase := cases["nodes"]
-	peeringsCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		id, err := uuid.GenerateUUID()
-		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}}))
-		id, err = uuid.GenerateUUID()
-		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}}))
-		id, err = uuid.GenerateUUID()
-		require.NoError(t, err)
-		require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}}))
-	}
-	peeringsCase.getMembersFunc = func() []serf.Member {
-		return []serf.Member{
-			{
-				Name:   "foo",
-				Tags:   map[string]string{"role": "consul"},
-				Status: serf.StatusAlive,
-			},
-			{
-				Name:   "bar",
-				Tags:   map[string]string{"role": "consul"},
-				Status: serf.StatusAlive,
-			},
-		}
-	}
-	peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.nodes",
-		Value:  0,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.peerings",
-		Value:  3,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.members.clients",
-		Value:  0,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	cases["peerings"] = peeringsCase
-	delete(cases, "nodes")
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s, err := newStateStore()
-			require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitServiceUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-
-	nodesAndSvcsCase := cases["nodes"]
-	nodesAndSvcsCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
-		require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
-		require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"}))
-		require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"}))
-
-		apigw := structs.TestNodeServiceAPIGateway(t)
-		apigw.ID = "api-gateway"
-
-		mgw := structs.TestNodeServiceMeshGateway(t)
-		mgw.ID = "mesh-gateway"
-
-		tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1")
-		tgw.ID = "terminating-gateway"
-		// Typical services and some consul services spread across two nodes
-		require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000}))
-		require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000}))
-		require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
-		require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
-		require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy}))
-		require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t)))
-		require.NoError(t, s.EnsureService(11, "foo", mgw))
-		require.NoError(t, s.EnsureService(12, "foo", tgw))
-		require.NoError(t, s.EnsureService(13, "foo", apigw))
-		require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}}))
-		require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "foo",
-		}))
-		require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "bar",
-		}))
-		require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{
-			Kind: structs.IngressGateway,
-			Name: "baz",
-		}))
-	}
-	baseCaseMembers := nodesAndSvcsCase.getMembersFunc()
-	nodesAndSvcsCase.getMembersFunc = func() []serf.Member {
-		baseCaseMembers = append(baseCaseMembers, serf.Member{
-			Name:   "baz",
-			Tags:   map[string]string{"role": "node", "segment": "a"},
-			Status: serf.StatusAlive,
-		})
-		baseCaseMembers = append(baseCaseMembers, serf.Member{
-			Name:   "qux",
-			Tags:   map[string]string{"role": "node", "segment": "b"},
-			Status: serf.StatusAlive,
-		})
-		return baseCaseMembers
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.nodes",
-		Value:  4,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.members.clients",
-		Value:  2,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.services",
-		Value:  8,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.service_instances",
-		Value:  10,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "connect-proxy"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "terminating-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "ingress-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "api-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "mesh-gateway"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.connect_instances",
-		Value: 1,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "connect-native"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.billable_service_instances",
-		Value: 3,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-		},
-	}
-	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
-		Name:  "consul.usage.test.state.config_entries",
-		Value: 3,
-		Labels: []metrics.Label{
-			{Name: "datacenter", Value: "dc1"},
-			{Name: "kind", Value: "ingress-gateway"},
-		},
-	}
-	cases["nodes-and-services"] = nodesAndSvcsCase
-	delete(cases, "nodes")
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s := state.NewStateStore(nil)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
-	}
-}
-
-func TestUsageReporter_emitKVUsage_CE(t *testing.T) {
-	cases := make(map[string]testCase)
-	for k, v := range baseCases {
-		eg := make(map[string]metrics.GaugeValue)
-		for k, v := range v.expectedGauges {
-			eg[k] = v
-		}
-		cases[k] = testCase{v.modfiyStateStore, v.getMembersFunc, eg}
-	}
-
-	nodesCase := cases["nodes"]
-	mss := nodesCase.modfiyStateStore
-	nodesCase.modfiyStateStore = func(t *testing.T, s *state.Store) {
-		mss(t, s)
-		require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}}))
-		require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{}))
-		require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{}))
-		require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}}))
-		require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}}))
-	}
-	nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{
-		Name:   "consul.usage.test.state.kv_entries",
-		Value:  4,
-		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
-	}
-	cases["nodes"] = nodesCase
-
-	for name, tcase := range cases {
-		t.Run(name, func(t *testing.T) {
-			// Only have a single interval for the test
-			sink := metrics.NewInmemSink(1*time.Minute, 1*time.Minute)
-			cfg := metrics.DefaultConfig("consul.usage.test")
-			cfg.EnableHostname = false
-			metrics.NewGlobal(cfg, sink)
-
-			mockStateProvider := &mockStateProvider{}
-			s, err := newStateStore()
-			require.NoError(t, err)
-			if tcase.modfiyStateStore != nil {
-				tcase.modfiyStateStore(t, s)
-			}
-			mockStateProvider.On("State").Return(s)
-
-			reporter, err := NewUsageMetricsReporter(
-				new(Config).
-					WithStateProvider(mockStateProvider).
-					WithLogger(testutil.Logger(t)).
-					WithDatacenter("dc1").
-					WithGetMembersFunc(tcase.getMembersFunc),
-			)
-			require.NoError(t, err)
-
-			reporter.runOnce()
-
-			intervals := sink.Data()
-			require.Len(t, intervals, 1)
-			intv := intervals[0]
-
-			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
-		})
 	}
+
+	testUsageReporter_Tenantless(t, getMetricsReporter)
 }

agent/consul/usagemetrics/usagemetrics_test.go

@@ -10,15 +10,19 @@ import (
 
 	"github.com/armon/go-metrics"
 	"github.com/armon/go-metrics/prometheus"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/serf/serf"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
+	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/consul/state"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/lib"
-	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/consul/proto/private/pbpeering"
-	"github.com/hashicorp/serf/serf"
+	"github.com/hashicorp/consul/version"
 )
 
 type mockStateProvider struct {
@@ -119,3 +123,856 @@ type benchStateProvider func() *state.Store
 func (b benchStateProvider) State() *state.Store {
 	return b()
 }
+
+type testCase struct {
+	modifyStateStore func(t *testing.T, s *state.Store)
+	getMembersFunc   getMembersFunc
+	expectedGauges   map[string]metrics.GaugeValue
+}
+
+var baseCases = map[string]testCase{
+	"empty-state": {
+		expectedGauges: map[string]metrics.GaugeValue{
+			// --- node ---
+			"consul.usage.test.state.nodes;datacenter=dc1": {
+				Name:   "consul.usage.test.state.nodes",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- peering ---
+			"consul.usage.test.state.peerings;datacenter=dc1": {
+				Name:   "consul.usage.test.state.peerings",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- member ---
+			"consul.usage.test.members.clients;datacenter=dc1": {
+				Name:   "consul.usage.test.members.clients",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.members.servers;datacenter=dc1": {
+				Name:   "consul.usage.test.members.servers",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service ---
+			"consul.usage.test.state.services;datacenter=dc1": {
+				Name:   "consul.usage.test.state.services",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.state.service_instances;datacenter=dc1": {
+				Name:   "consul.usage.test.state.service_instances",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service mesh ---
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-proxy"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-native"},
+				},
+			},
+			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
+				Name:  "consul.usage.test.state.billable_service_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+				},
+			},
+			// --- kv ---
+			"consul.usage.test.state.kv_entries;datacenter=dc1": {
+				Name:   "consul.usage.test.state.kv_entries",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- config entries ---
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-intentions"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-resolver"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-router"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-splitter"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "proxy-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "exported-services"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "sameness-group"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "bound-api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "inline-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "http-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "tcp-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "jwt-provider"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "control-plane-request-limit"},
+				},
+			},
+			// --- version ---
+			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
+				Name:  "consul.usage.test.version",
+				Value: 1,
+				Labels: []metrics.Label{
+					{Name: "version", Value: versionWithMetadata()},
+					{Name: "pre_release", Value: version.VersionPrerelease},
+				},
+			},
+		},
+		getMembersFunc: func() []serf.Member { return []serf.Member{} },
+	},
+	"nodes": {
+		modifyStateStore: func(t *testing.T, s *state.Store) {
+			require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
+			require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
+		},
+		getMembersFunc: func() []serf.Member {
+			return []serf.Member{
+				{
+					Name:   "foo",
+					Tags:   map[string]string{"role": "consul"},
+					Status: serf.StatusAlive,
+				},
+				{
+					Name:   "bar",
+					Tags:   map[string]string{"role": "consul"},
+					Status: serf.StatusAlive,
+				},
+			}
+		},
+		expectedGauges: map[string]metrics.GaugeValue{
+			// --- node ---
+			"consul.usage.test.state.nodes;datacenter=dc1": {
+				Name:   "consul.usage.test.state.nodes",
+				Value:  2,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- peering ---
+			"consul.usage.test.state.peerings;datacenter=dc1": {
+				Name:   "consul.usage.test.state.peerings",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- member ---
+			"consul.usage.test.members.servers;datacenter=dc1": {
+				Name:   "consul.usage.test.members.servers",
+				Value:  2,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.members.clients;datacenter=dc1": {
+				Name:   "consul.usage.test.members.clients",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service ---
+			"consul.usage.test.state.services;datacenter=dc1": {
+				Name:   "consul.usage.test.state.services",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			"consul.usage.test.state.service_instances;datacenter=dc1": {
+				Name:   "consul.usage.test.state.service_instances",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- service mesh ---
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-proxy"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh-gateway"},
+				},
+			},
+			"consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native": {
+				Name:  "consul.usage.test.state.connect_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "connect-native"},
+				},
+			},
+			"consul.usage.test.state.billable_service_instances;datacenter=dc1": {
+				Name:  "consul.usage.test.state.billable_service_instances",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+				},
+			},
+			// --- kv ---
+			"consul.usage.test.state.kv_entries;datacenter=dc1": {
+				Name:   "consul.usage.test.state.kv_entries",
+				Value:  0,
+				Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+			},
+			// --- config entries ---
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-intentions": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-intentions"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-resolver": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-resolver"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-router": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-router"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "ingress-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=service-splitter": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "service-splitter"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=mesh": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "mesh"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=proxy-defaults": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "proxy-defaults"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=terminating-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "terminating-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=exported-services": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "exported-services"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=sameness-group": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "sameness-group"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=bound-api-gateway": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "bound-api-gateway"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=inline-certificate": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "inline-certificate"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=http-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "http-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=tcp-route": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "tcp-route"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=jwt-provider": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "jwt-provider"},
+				},
+			},
+			"consul.usage.test.state.config_entries;datacenter=dc1;kind=control-plane-request-limit": {
+				Name:  "consul.usage.test.state.config_entries",
+				Value: 0,
+				Labels: []metrics.Label{
+					{Name: "datacenter", Value: "dc1"},
+					{Name: "kind", Value: "control-plane-request-limit"},
+				},
+			},
+			// --- version ---
+			fmt.Sprintf("consul.usage.test.version;version=%s;pre_release=%s", versionWithMetadata(), version.VersionPrerelease): {
+				Name:  "consul.usage.test.version",
+				Value: 1,
+				Labels: []metrics.Label{
+					{Name: "version", Value: versionWithMetadata()},
+					{Name: "pre_release", Value: version.VersionPrerelease},
+				},
+			},
+		},
+	},
+}
+
+func testUsageReporter_Tenantless(t *testing.T, getMetricsReporter func(tc testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	t.Run("emitNodeUsage", func(t *testing.T) {
+		testUsageReporter_emitNodeUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitPeeringUsage", func(t *testing.T) {
+		testUsageReporter_emitPeeringUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitServiceUsage", func(t *testing.T) {
+		testUsageReporter_emitServiceUsage_CE(t, getMetricsReporter)
+	})
+
+	t.Run("emitKVUsage", func(t *testing.T) {
+		testUsageReporter_emitKVUsage_CE(t, getMetricsReporter)
+	})
+}
+
+func testUsageReporter_emitNodeUsage_CE(t *testing.T, getReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := baseCases
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitPeeringUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+	peeringsCase := cases["nodes"]
+	peeringsCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		id, err := uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(1, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "foo", ID: id}}))
+		id, err = uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(2, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "bar", ID: id}}))
+		id, err = uuid.GenerateUUID()
+		require.NoError(t, err)
+		require.NoError(t, s.PeeringWrite(3, &pbpeering.PeeringWriteRequest{Peering: &pbpeering.Peering{Name: "baz", ID: id}}))
+	}
+	peeringsCase.getMembersFunc = func() []serf.Member {
+		return []serf.Member{
+			{
+				Name:   "foo",
+				Tags:   map[string]string{"role": "consul"},
+				Status: serf.StatusAlive,
+			},
+			{
+				Name:   "bar",
+				Tags:   map[string]string{"role": "consul"},
+				Status: serf.StatusAlive,
+			},
+		}
+	}
+	peeringsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.nodes",
+		Value:  0,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	peeringsCase.expectedGauges["consul.usage.test.state.peerings;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.peerings",
+		Value:  3,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	peeringsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.members.clients",
+		Value:  0,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	cases["peerings"] = peeringsCase
+	delete(cases, "nodes")
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitServiceUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+
+	nodesAndSvcsCase := cases["nodes"]
+	nodesAndSvcsCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		require.NoError(t, s.EnsureNode(1, &structs.Node{Node: "foo", Address: "127.0.0.1"}))
+		require.NoError(t, s.EnsureNode(2, &structs.Node{Node: "bar", Address: "127.0.0.2"}))
+		require.NoError(t, s.EnsureNode(3, &structs.Node{Node: "baz", Address: "127.0.0.2"}))
+		require.NoError(t, s.EnsureNode(4, &structs.Node{Node: "qux", Address: "127.0.0.3"}))
+
+		apigw := structs.TestNodeServiceAPIGateway(t)
+		apigw.ID = "api-gateway"
+
+		mgw := structs.TestNodeServiceMeshGateway(t)
+		mgw.ID = "mesh-gateway"
+
+		tgw := structs.TestNodeServiceTerminatingGateway(t, "1.1.1.1")
+		tgw.ID = "terminating-gateway"
+		// Typical services and some consul services spread across two nodes
+		require.NoError(t, s.EnsureService(5, "foo", &structs.NodeService{ID: "db", Service: "db", Tags: nil, Address: "", Port: 5000}))
+		require.NoError(t, s.EnsureService(6, "bar", &structs.NodeService{ID: "api", Service: "api", Tags: nil, Address: "", Port: 5000}))
+		require.NoError(t, s.EnsureService(7, "foo", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
+		require.NoError(t, s.EnsureService(8, "bar", &structs.NodeService{ID: "consul", Service: "consul", Tags: nil}))
+		require.NoError(t, s.EnsureService(9, "foo", &structs.NodeService{ID: "db-connect-proxy", Service: "db-connect-proxy", Tags: nil, Address: "", Port: 5000, Kind: structs.ServiceKindConnectProxy}))
+		require.NoError(t, s.EnsureRegistration(10, structs.TestRegisterIngressGateway(t)))
+		require.NoError(t, s.EnsureService(11, "foo", mgw))
+		require.NoError(t, s.EnsureService(12, "foo", tgw))
+		require.NoError(t, s.EnsureService(13, "foo", apigw))
+		require.NoError(t, s.EnsureService(14, "bar", &structs.NodeService{ID: "db-native", Service: "db", Tags: nil, Address: "", Port: 5000, Connect: structs.ServiceConnect{Native: true}}))
+		require.NoError(t, s.EnsureConfigEntry(15, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "foo",
+		}))
+		require.NoError(t, s.EnsureConfigEntry(16, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "bar",
+		}))
+		require.NoError(t, s.EnsureConfigEntry(17, &structs.IngressGatewayConfigEntry{
+			Kind: structs.IngressGateway,
+			Name: "baz",
+		}))
+	}
+	baseCaseMembers := nodesAndSvcsCase.getMembersFunc()
+	nodesAndSvcsCase.getMembersFunc = func() []serf.Member {
+		baseCaseMembers = append(baseCaseMembers, serf.Member{
+			Name:   "baz",
+			Tags:   map[string]string{"role": "node", "segment": "a"},
+			Status: serf.StatusAlive,
+		})
+		baseCaseMembers = append(baseCaseMembers, serf.Member{
+			Name:   "qux",
+			Tags:   map[string]string{"role": "node", "segment": "b"},
+			Status: serf.StatusAlive,
+		})
+		return baseCaseMembers
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.nodes;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.nodes",
+		Value:  4,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.members.clients;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.members.clients",
+		Value:  2,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.services;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.services",
+		Value:  8,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.service_instances;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.service_instances",
+		Value:  10,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-proxy"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "connect-proxy"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=terminating-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "terminating-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "ingress-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=api-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "api-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=mesh-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "mesh-gateway"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.connect_instances;datacenter=dc1;kind=connect-native"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.connect_instances",
+		Value: 1,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "connect-native"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.billable_service_instances;datacenter=dc1"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.billable_service_instances",
+		Value: 3,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+		},
+	}
+	nodesAndSvcsCase.expectedGauges["consul.usage.test.state.config_entries;datacenter=dc1;kind=ingress-gateway"] = metrics.GaugeValue{
+		Name:  "consul.usage.test.state.config_entries",
+		Value: 3,
+		Labels: []metrics.Label{
+			{Name: "datacenter", Value: "dc1"},
+			{Name: "kind", Value: "ingress-gateway"},
+		},
+	}
+	cases["nodes-and-services"] = nodesAndSvcsCase
+	delete(cases, "nodes")
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}
+
+func testUsageReporter_emitKVUsage_CE(t *testing.T, getMetricsReporter func(testCase) (*UsageMetricsReporter, *metrics.InmemSink, error)) {
+	cases := make(map[string]testCase)
+	for k, v := range baseCases {
+		eg := make(map[string]metrics.GaugeValue)
+		for k, v := range v.expectedGauges {
+			eg[k] = v
+		}
+		cases[k] = testCase{v.modifyStateStore, v.getMembersFunc, eg}
+	}
+
+	nodesCase := cases["nodes"]
+	mss := nodesCase.modifyStateStore
+	nodesCase.modifyStateStore = func(t *testing.T, s *state.Store) {
+		mss(t, s)
+		require.NoError(t, s.KVSSet(4, &structs.DirEntry{Key: "a", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(5, &structs.DirEntry{Key: "b", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(6, &structs.DirEntry{Key: "c", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(7, &structs.DirEntry{Key: "d", Value: []byte{1}}))
+		require.NoError(t, s.KVSDelete(8, "d", &acl.EnterpriseMeta{}))
+		require.NoError(t, s.KVSDelete(9, "c", &acl.EnterpriseMeta{}))
+		require.NoError(t, s.KVSSet(10, &structs.DirEntry{Key: "e", Value: []byte{1}}))
+		require.NoError(t, s.KVSSet(11, &structs.DirEntry{Key: "f", Value: []byte{1}}))
+	}
+	nodesCase.expectedGauges["consul.usage.test.state.kv_entries;datacenter=dc1"] = metrics.GaugeValue{
+		Name:   "consul.usage.test.state.kv_entries",
+		Value:  4,
+		Labels: []metrics.Label{{Name: "datacenter", Value: "dc1"}},
+	}
+	cases["nodes"] = nodesCase
+
+	for name, tcase := range cases {
+		t.Run(name, func(t *testing.T) {
+			reporter, sink, err := getMetricsReporter(tcase)
+			require.NoError(t, err)
+
+			reporter.runOnce()
+
+			intervals := sink.Data()
+			require.Len(t, intervals, 1)
+			intv := intervals[0]
+
+			assertEqualGaugeMaps(t, tcase.expectedGauges, intv.Gauges)
+		})
+	}
+}

lib/telemetry.go

@@ -146,9 +146,14 @@ type TelemetryConfig struct {
 
 	// DisableHostname will disable hostname prefixing for all metrics.
 	//
-	// hcl: telemetry { disable_hostname = (true|false)
+	// hcl: telemetry { disable_hostname = (true|false) }
 	DisableHostname bool `json:"disable_hostname,omitempty" mapstructure:"disable_hostname"`
 
+	// DisablePerTenancyUsageMetrics will disable setting tenancy labels on usage metrics.
+	//
+	// hcl: telemetry { disable_per_tenancy_usage_metrics = (true|false) }
+	DisablePerTenancyUsageMetrics bool `json:"disable_per_tenancy_usage_metrics,omitempty" mapstructure:"disable_per_tenancy_usage_metrics"`
+
 	// DogStatsdAddr is the address of a dogstatsd instance. If provided,
 	// metrics will be sent to that instance
 	//

website/content/docs/agent/config/config-files.mdx

@@ -1842,6 +1842,9 @@ subsystem that provides Consul's service mesh capabilities.
   - `disable_hostname` ((#telemetry-disable_hostname))
     Set to `true` to stop prepending the machine's hostname to gauge-type metrics. Default is `false`.
 
+  - `disable_per_tenancy_usage_metrics` ((#telemetry-disable_per_tenancy_usage_metrics))
+    Set to `true` to exclude tenancy labels from usage metrics. This significantly decreases CPU utilization in clusters with many admin partitions or namespaces.
+
   - `dogstatsd_addr` ((#telemetry-dogstatsd_addr)) This provides the address
     of a DogStatsD instance in the format `host:port`. DogStatsD is a protocol-compatible
     flavor of statsd, with the added ability to decorate metrics with tags and event