diff --git a/.changelog/20945.txt b/.changelog/20945.txt new file mode 100644 index 0000000000..a0a0602156 --- /dev/null +++ b/.changelog/20945.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +gateways: service defaults configuration entries can now be used to set default upstream limits for mesh-gateways +``` \ No newline at end of file diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index c9fe60a892..3d8bcd43a9 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -127,6 +127,17 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er if err != nil { return snap, err } + // Watch for service default object that matches this mesh gateway's name + err = s.dataSources.ConfigEntry.Notify(ctx, &structs.ConfigEntryQuery{ + Kind: structs.ServiceDefaults, + Name: s.service, + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + EnterpriseMeta: s.proxyID.EnterpriseMeta, + }, serviceDefaultsWatchID, s.ch) + if err != nil { + return snap, err + } snap.MeshGateway.WatchedServices = make(map[structs.ServiceName]context.CancelFunc) snap.MeshGateway.WatchedGateways = make(map[string]context.CancelFunc) @@ -648,6 +659,25 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u UpdateEvent, sn } snap.MeshGateway.PeerServers = peerServers + case serviceDefaultsWatchID: + resp, ok := u.Result.(*structs.ConfigEntryResponse) + if !ok { + return fmt.Errorf("invalid type for config entry: %T", resp.Entry) + } + + if resp.Entry == nil { + return nil + } + serviceDefaults, ok := resp.Entry.(*structs.ServiceConfigEntry) + if !ok { + return fmt.Errorf("invalid type for config entry: %T", resp.Entry) + } + + if serviceDefaults.UpstreamConfig != nil && serviceDefaults.UpstreamConfig.Defaults != nil { + if serviceDefaults.UpstreamConfig.Defaults.Limits != nil { + snap.MeshGateway.Limits = serviceDefaults.UpstreamConfig.Defaults.Limits + } + } default: switch { diff --git a/agent/proxycfg/proxycfg.deepcopy.go b/agent/proxycfg/proxycfg.deepcopy.go index e144a036e5..669a519094 100644 --- a/agent/proxycfg/proxycfg.deepcopy.go +++ b/agent/proxycfg/proxycfg.deepcopy.go @@ -728,6 +728,22 @@ func (o *configSnapshotMeshGateway) DeepCopy() *configSnapshotMeshGateway { } } } + if o.Limits != nil { + cp.Limits = new(structs.UpstreamLimits) + *cp.Limits = *o.Limits + if o.Limits.MaxConnections != nil { + cp.Limits.MaxConnections = new(int) + *cp.Limits.MaxConnections = *o.Limits.MaxConnections + } + if o.Limits.MaxPendingRequests != nil { + cp.Limits.MaxPendingRequests = new(int) + *cp.Limits.MaxPendingRequests = *o.Limits.MaxPendingRequests + } + if o.Limits.MaxConcurrentRequests != nil { + cp.Limits.MaxConcurrentRequests = new(int) + *cp.Limits.MaxConcurrentRequests = *o.Limits.MaxConcurrentRequests + } + } return &cp } diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 01c8c158a6..bac6e8b6da 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -498,6 +498,9 @@ type configSnapshotMeshGateway struct { // PeeringTrustBundlesSet indicates that the watch on the peer trust // bundles has completed at least once. PeeringTrustBundlesSet bool + + // Limits + Limits *structs.UpstreamLimits } // MeshGatewayValidExportedServices ensures that the following data is present diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index f8bfe3b2c2..b6b9c78f32 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -34,6 +34,7 @@ const ( consulServerListWatchID = "consul-server-list" datacentersWatchID = "datacenters" serviceResolversWatchID = "service-resolvers" + serviceDefaultsWatchID = "service-defaults" gatewayServicesWatchID = "gateway-services" gatewayConfigWatchID = "gateway-config" apiGatewayConfigWatchID = "api-gateway-config" diff --git a/agent/proxycfg/testing_mesh_gateway.go b/agent/proxycfg/testing_mesh_gateway.go index 72a3b73730..80d37220cc 100644 --- a/agent/proxycfg/testing_mesh_gateway.go +++ b/agent/proxycfg/testing_mesh_gateway.go @@ -264,6 +264,25 @@ func TestConfigSnapshotMeshGateway(t testing.T, variant string, nsFn func(ns *st }, }, }) + case "limits-added": + extraUpdates = append(extraUpdates, UpdateEvent{ + CorrelationID: serviceDefaultsWatchID, + Result: &structs.ConfigEntryResponse{ + Entry: &structs.ServiceConfigEntry{ + Kind: structs.ServiceDefaults, + Name: "mesh-gateway", + UpstreamConfig: &structs.UpstreamConfiguration{ + Defaults: &structs.UpstreamConfig{ + Limits: &structs.UpstreamLimits{ + MaxConnections: pointerTo(1), + MaxPendingRequests: pointerTo(10), + MaxConcurrentRequests: pointerTo(100), + }, + }, + }, + }, + }, + }) default: t.Fatalf("unknown variant: %s", variant) return nil @@ -1124,3 +1143,7 @@ func TestConfigSnapshotPeeredMeshGateway(t testing.T, variant string, nsFn func( }, }, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates)) } + +func pointerTo[T any](v T) *T { + return &v +} diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index df59604583..f8abdc0e91 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -532,6 +532,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co name: connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain), hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()], isRemote: true, + limits: cfgSnap.MeshGateway.Limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) @@ -554,6 +555,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co name: cfgSnap.ServerSNIFn(key.Datacenter, ""), hostnameEndpoints: hostnameEndpoints, isRemote: !key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrDefault()), + limits: cfgSnap.MeshGateway.Limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) @@ -563,7 +565,8 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co servers, _ := cfgSnap.MeshGateway.WatchedLocalServers.Get(structs.ConsulServiceName) for _, srv := range servers { opts := clusterOpts{ - name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node), + name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node), + limits: cfgSnap.MeshGateway.Limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) @@ -579,14 +582,15 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co // We avoid routing to read replicas since they will never be Raft voters. if haveVoters(servers) { cluster := s.makeGatewayCluster(cfgSnap, clusterOpts{ - name: connect.PeeringServerSAN(cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain), + name: connect.PeeringServerSAN(cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain), + limits: cfgSnap.MeshGateway.Limits, }) clusters = append(clusters, cluster) } } // generate the per-service/subset clusters - c, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers) + c, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers, cfgSnap.MeshGateway.Limits) if err != nil { return nil, err } @@ -664,7 +668,7 @@ func (s *ResourceGenerator) makePeerServerClusters(cfgSnap *proxycfg.ConfigSnaps // for a terminating gateway. This will include 1 cluster per Destination associated with this terminating gateway. func (s *ResourceGenerator) clustersFromSnapshotTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { res := []proto.Message{} - gwClusters, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers) + gwClusters, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers, nil) if err != nil { return nil, err } @@ -683,6 +687,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( cfgSnap *proxycfg.ConfigSnapshot, services map[structs.ServiceName]structs.CheckServiceNodes, resolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry, + limits *structs.UpstreamLimits, ) ([]proto.Message, error) { var hostnameEndpoints structs.CheckServiceNodes @@ -724,6 +729,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( hostnameEndpoints: hostnameEndpoints, connectTimeout: resolver.ConnectTimeout, isRemote: isRemote, + limits: limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) @@ -763,6 +769,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( onlyPassing: subset.OnlyPassing, connectTimeout: resolver.ConnectTimeout, isRemote: isRemote, + limits: limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) @@ -812,6 +819,7 @@ func (s *ResourceGenerator) makeGatewayOutgoingClusterPeeringServiceClusters(cfg name: clusterName, isRemote: true, hostnameEndpoints: hostnameEndpoints, + limits: cfgSnap.MeshGateway.Limits, } cluster := s.makeGatewayCluster(cfgSnap, opts) @@ -1706,6 +1714,8 @@ type clusterOpts struct { // Corresponds to a valid address/port pairs to be routed externally // these addresses will be embedded in the cluster configuration and will never use EDS addresses []structs.ServiceAddress + + limits *structs.UpstreamLimits } // makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway @@ -1768,6 +1778,12 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op ) } + if opts.limits != nil { + cluster.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ + Thresholds: makeThresholdsIfNeeded(opts.limits), + } + } + return cluster } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index a6957d929d..a1afeaee07 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -743,6 +743,14 @@ func getMeshGatewayGoldenTestCases() []goldenTestCase { // TODO(proxystate): mesh gateway will come at a later time alsoRunTestForV2: false, }, + { + name: "mesh-gateway-with-limits", + create: func(t testinf.T) *proxycfg.ConfigSnapshot { + return proxycfg.TestConfigSnapshotMeshGateway(t, "limits-added", nil, nil) + }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, + }, { name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/clusters/mesh-gateway-with-limits.latest.golden b/agent/xds/testdata/clusters/mesh-gateway-with-limits.latest.golden new file mode 100644 index 0000000000..5be607cf8c --- /dev/null +++ b/agent/xds/testdata/clusters/mesh-gateway-with-limits.latest.golden @@ -0,0 +1,151 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "circuitBreakers": { + "thresholds": [ + { + "maxConnections": 1, + "maxPendingRequests": 10, + "maxRequests": 100 + } + ] + }, + "connectTimeout": "5s", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "outlierDetection": {}, + "type": "EDS" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "circuitBreakers": { + "thresholds": [ + { + "maxConnections": 1, + "maxPendingRequests": 10, + "maxRequests": 100 + } + ] + }, + "connectTimeout": "5s", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", + "outlierDetection": {}, + "type": "EDS" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "circuitBreakers": { + "thresholds": [ + { + "maxConnections": 1, + "maxPendingRequests": 10, + "maxRequests": 100 + } + ] + }, + "connectTimeout": "5s", + "dnsLookupFamily": "V4_ONLY", + "dnsRefreshRate": "10s", + "loadAssignment": { + "clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-west-2.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "outlierDetection": {}, + "type": "LOGICAL_DNS" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "circuitBreakers": { + "thresholds": [ + { + "maxConnections": 1, + "maxPendingRequests": 10, + "maxRequests": 100 + } + ] + }, + "connectTimeout": "5s", + "dnsLookupFamily": "V4_ONLY", + "dnsRefreshRate": "10s", + "loadAssignment": { + "clusterName": "dc6.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "123.us-east-1.elb.notaws.com", + "portValue": 443 + } + } + }, + "healthStatus": "UNHEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "name": "dc6.internal.11111111-2222-3333-4444-555555555555.consul", + "outlierDetection": {}, + "type": "LOGICAL_DNS" + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "circuitBreakers": { + "thresholds": [ + { + "maxConnections": 1, + "maxPendingRequests": 10, + "maxRequests": 100 + } + ] + }, + "connectTimeout": "5s", + "edsClusterConfig": { + "edsConfig": { + "ads": {}, + "resourceApiVersion": "V3" + } + }, + "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "outlierDetection": {}, + "type": "EDS" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/mesh-gateway-with-limits.latest.golden b/agent/xds/testdata/endpoints/mesh-gateway-with-limits.latest.golden new file mode 100644 index 0000000000..4853c9dc93 --- /dev/null +++ b/agent/xds/testdata/endpoints/mesh-gateway-with-limits.latest.golden @@ -0,0 +1,145 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.6", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.7", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.8", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.1", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.2", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.3", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.4", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.5", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.9", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/mesh-gateway-with-limits.latest.golden b/agent/xds/testdata/listeners/mesh-gateway-with-limits.latest.golden new file mode 100644 index 0000000000..3b77ccca9b --- /dev/null +++ b/agent/xds/testdata/listeners/mesh-gateway-with-limits.latest.golden @@ -0,0 +1,96 @@ +{ + "nonce": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8443 + } + }, + "filterChains": [ + { + "filterChainMatch": { + "serverNames": [ + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", + "statPrefix": "mesh_gateway_remote.default.dc2" + } + } + ] + }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc4.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "dc4.internal.11111111-2222-3333-4444-555555555555.consul", + "statPrefix": "mesh_gateway_remote.default.dc4" + } + } + ] + }, + { + "filterChainMatch": { + "serverNames": [ + "*.dc6.internal.11111111-2222-3333-4444-555555555555.consul" + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "dc6.internal.11111111-2222-3333-4444-555555555555.consul", + "statPrefix": "mesh_gateway_remote.default.dc6" + } + } + ] + }, + { + "filters": [ + { + "name": "envoy.filters.network.sni_cluster", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.sni_cluster.v3.SniCluster" + } + }, + { + "name": "envoy.filters.network.tcp_proxy", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "cluster": "", + "statPrefix": "mesh_gateway_local.default" + } + } + ] + } + ], + "listenerFilters": [ + { + "name": "envoy.filters.listener.tls_inspector", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector" + } + } + ], + "name": "default:1.2.3.4:8443" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/mesh-gateway-with-limits.latest.golden b/agent/xds/testdata/routes/mesh-gateway-with-limits.latest.golden new file mode 100644 index 0000000000..8b919343d2 --- /dev/null +++ b/agent/xds/testdata/routes/mesh-gateway-with-limits.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "versionInfo": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/secrets/mesh-gateway-with-limits.latest.golden b/agent/xds/testdata/secrets/mesh-gateway-with-limits.latest.golden new file mode 100644 index 0000000000..82e4565065 --- /dev/null +++ b/agent/xds/testdata/secrets/mesh-gateway-with-limits.latest.golden @@ -0,0 +1,5 @@ +{ + "nonce": "00000001", + "typeUrl": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + "versionInfo": "00000001" +} \ No newline at end of file