diff --git a/.changelog/13699.txt b/.changelog/13699.txt new file mode 100644 index 0000000000..64cde8d97c --- /dev/null +++ b/.changelog/13699.txt @@ -0,0 +1,3 @@ +```release-note:bug +xds: Fix a bug where terminating gateway upstream clusters weren't configured properly when the service protocol was `http2`. +``` diff --git a/agent/proxycfg/testing_terminating_gateway.go b/agent/proxycfg/testing_terminating_gateway.go index bac411fff8..00771433b5 100644 --- a/agent/proxycfg/testing_terminating_gateway.go +++ b/agent/proxycfg/testing_terminating_gateway.go @@ -633,6 +633,123 @@ func TestConfigSnapshotTerminatingGatewaySNI(t testing.T) *ConfigSnapshot { }) } +func TestConfigSnapshotTerminatingGatewayHTTP2(t testing.T) *ConfigSnapshot { + web := structs.NewServiceName("web", nil) + + return TestConfigSnapshotTerminatingGateway(t, false, nil, []UpdateEvent{ + { + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{ + Services: []*structs.GatewayService{ + { + Service: web, + CAFile: "ca.cert.pem", + }, + }, + }, + }, + { + CorrelationID: serviceConfigIDPrefix + web.String(), + Result: &structs.ServiceConfigResponse{ + ProxyConfig: map[string]interface{}{"protocol": "http2"}, + }, + }, + { + CorrelationID: externalServiceIDPrefix + web.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: []structs.CheckServiceNode{ + { + Node: &structs.Node{ + ID: "external", + Node: "external", + Address: "web.external.service", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "web", + Port: 9090, + }, + }, + }, + }, + }, + }) +} + +func TestConfigSnapshotTerminatingGatewaySubsetsHTTP2(t testing.T) *ConfigSnapshot { + web := structs.NewServiceName("web", nil) + + return TestConfigSnapshotTerminatingGateway(t, false, nil, []UpdateEvent{ + { + CorrelationID: serviceResolverIDPrefix + web.String(), + Result: &structs.ConfigEntryResponse{ + Entry: &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "web", + Subsets: map[string]structs.ServiceResolverSubset{ + "v1": { + Filter: "Service.Meta.version == 1", + }, + "v2": { + Filter: "Service.Meta.version == 2", + }, + }, + }, + }, + }, + { + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{ + Services: []*structs.GatewayService{ + { + Service: web, + CAFile: "ca.cert.pem", + }, + }, + }, + }, + { + CorrelationID: serviceConfigIDPrefix + web.String(), + Result: &structs.ServiceConfigResponse{ + ProxyConfig: map[string]interface{}{"protocol": "http2"}, + }, + }, + { + CorrelationID: externalServiceIDPrefix + web.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: []structs.CheckServiceNode{ + { + Node: &structs.Node{ + ID: "external", + Node: "external", + Address: "web.external.service", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "web", + Port: 9090, + Meta: map[string]string{"version": "1"}, + }, + }, + { + Node: &structs.Node{ + ID: "external2", + Node: "external2", + Address: "web.external2.service", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Service: "web", + Port: 9091, + Meta: map[string]string{"version": "2"}, + }, + }, + }, + }, + }, + }) +} + func TestConfigSnapshotTerminatingGatewayHostnameSubsets(t testing.T) *ConfigSnapshot { var ( api = structs.NewServiceName("api", nil) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 99cae180ca..b00fcfeeb6 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -425,6 +425,24 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( } clusters = append(clusters, cluster) + svcConfig, ok := cfgSnap.TerminatingGateway.ServiceConfigs[svc] + isHTTP2 := false + if ok { + upstreamCfg, err := structs.ParseUpstreamConfig(svcConfig.ProxyConfig) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse", "upstream", svc, "error", err) + } + isHTTP2 = upstreamCfg.Protocol == "http2" || upstreamCfg.Protocol == "grpc" + } + + if isHTTP2 { + if err := s.setHttp2ProtocolOptions(cluster); err != nil { + return nil, err + } + } + // If there is a service-resolver for this service then also setup a cluster for each subset for name, subset := range resolver.Subsets { subsetHostnameEndpoints, err := s.filterSubsetEndpoints(&subset, hostnameEndpoints) @@ -444,6 +462,11 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( if err := s.injectGatewayServiceAddons(cfgSnap, cluster, svc, loadBalancer); err != nil { return nil, err } + if isHTTP2 { + if err := s.setHttp2ProtocolOptions(cluster); err != nil { + return nil, err + } + } clusters = append(clusters, cluster) } } diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index b3e85486b7..49d333750c 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -589,6 +589,14 @@ func TestClustersFromSnapshot(t *testing.T) { name: "terminating-gateway-sni", create: proxycfg.TestConfigSnapshotTerminatingGatewaySNI, }, + { + name: "terminating-gateway-http2-upstream", + create: proxycfg.TestConfigSnapshotTerminatingGatewayHTTP2, + }, + { + name: "terminating-gateway-http2-upstream-subsets", + create: proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2, + }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGatewayIgnoreExtraResolvers, diff --git a/agent/xds/testdata/clusters/terminating-gateway-http2-upstream-subsets.latest.golden b/agent/xds/testdata/clusters/terminating-gateway-http2-upstream-subsets.latest.golden new file mode 100644 index 0000000000..953a207e68 --- /dev/null +++ b/agent/xds/testdata/clusters/terminating-gateway-http2-upstream-subsets.latest.golden @@ -0,0 +1,181 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "v1.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "web.external.service", + "portValue": 9090 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": { + + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "v2.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "v2.web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "web.external2.service", + "portValue": 9091 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": { + + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "web.external.service", + "portValue": 9090 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": { + + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/terminating-gateway-http2-upstream.latest.golden b/agent/xds/testdata/clusters/terminating-gateway-http2-upstream.latest.golden new file mode 100644 index 0000000000..9a8e2bdf43 --- /dev/null +++ b/agent/xds/testdata/clusters/terminating-gateway-http2-upstream.latest.golden @@ -0,0 +1,65 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "LOGICAL_DNS", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "web.external.service", + "portValue": 9090 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": { + + } + } + } + }, + "dnsRefreshRate": "10s", + "dnsLookupFamily": "V4_ONLY", + "outlierDetection": { + + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "validationContext": { + "trustedCa": { + "filename": "ca.cert.pem" + } + } + } + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file