xds: Use downstream protocol when connecting to local app (#18573)

Configure Envoy to use the same HTTP protocol version used by the
downstream caller when forwarding requests to a local application that
is configured with the protocol set to either `http2` or `grpc`.

This allows upstream applications that support both HTTP/1.1 and
HTTP/2 on a single port to receive requests using either protocol. This
is beneficial when the application primarily communicates using HTTP/2,
but also needs to support HTTP/1.1, such as to respond to Kubernetes
HTTP readiness/liveness probes.

Co-authored-by: Derek Menteer <derek.menteer@hashicorp.com>

.changelog/18573.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+xds: Use downstream protocol when connecting to local app
+```

agent/xds/clusters.go

@@ -1096,10 +1096,16 @@ func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, nam
 		protocol = cfg.Protocol
 	}
 	if protocol == "http2" || protocol == "grpc" {
+		if name == xdscommon.LocalAppClusterName {
+			if err := s.setLocalAppHttpProtocolOptions(c); err != nil {
+				return c, err
+			}
+		} else {
 			if err := s.setHttp2ProtocolOptions(c); err != nil {
 				return c, err
 			}
 		}
+	}
 	if cfg.MaxInboundConnections > 0 {
 		c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{
 			Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{
@@ -2016,6 +2022,29 @@ func (s *ResourceGenerator) setHttp2ProtocolOptions(c *envoy_cluster_v3.Cluster)
 	return nil
 }
 
+// Allows forwarding either HTTP/1.1 or HTTP/2 traffic to the local application.
+// The protocol used depends on the protocol received from the downstream service
+// on the public listener.
+func (s *ResourceGenerator) setLocalAppHttpProtocolOptions(c *envoy_cluster_v3.Cluster) error {
+	cfg := &envoy_upstreams_v3.HttpProtocolOptions{
+		UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamProtocolConfig{
+			UseDownstreamProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamHttpConfig{
+				HttpProtocolOptions:  &envoy_core_v3.Http1ProtocolOptions{},
+				Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
+			},
+		},
+	}
+	any, err := anypb.New(cfg)
+	if err != nil {
+		return err
+	}
+	c.TypedExtensionProtocolOptions = map[string]*anypb.Any{
+		"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any,
+	}
+
+	return nil
+}
+
 // generatePeeredClusterName returns an SNI-like cluster name which mimics PeeredServiceSNI
 // but excludes partition information which could be ambiguous (local vs remote partition).
 func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTrustBundle) string {

agent/xds/testdata/clusters/expose-paths-grpc-new-cluster-http1.latest.golden

@@ -53,8 +53,9 @@
       "typedExtensionProtocolOptions": {
         "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
           "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
-          "explicitHttpConfig": {
-            "http2ProtocolOptions": {}
+          "useDownstreamProtocolConfig": {
+            "http2ProtocolOptions": {},
+            "httpProtocolOptions": {}
           }
         }
       }

agent/xdsv2/cluster_resources.go

@@ -164,7 +164,13 @@ func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, s
 	if ok {
 		cluster.LoadAssignment = makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints)
 	}
-	err := addHttpProtocolOptions(protocol, cluster)
+
+	var err error
+	if name == xdscommon.LocalAppClusterName {
+		err = addLocalAppHttpProtocolOptions(protocol, cluster)
+	} else {
+		err = addHttpProtocolOptions(protocol, cluster)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -243,6 +249,30 @@ func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string
 	return clusters, nil
 }
 
+func addLocalAppHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error {
+	if !(protocol == "http2" || protocol == "grpc") {
+		// do not error.  returning nil means it won't get set.
+		return nil
+	}
+	cfg := &envoy_upstreams_v3.HttpProtocolOptions{
+		UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamProtocolConfig{
+			UseDownstreamProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_UseDownstreamHttpConfig{
+				HttpProtocolOptions:  &envoy_core_v3.Http1ProtocolOptions{},
+				Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
+			},
+		},
+	}
+	any, err := anypb.New(cfg)
+	if err != nil {
+		return err
+	}
+	c.TypedExtensionProtocolOptions = map[string]*anypb.Any{
+		"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any,
+	}
+
+	return nil
+}
+
 func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error {
 	if !(protocol == "http2" || protocol == "grpc") {
 		// do not error.  returning nil means it won't get set.