docs: document how envoy escape hatches work with the discovery chain (#6350)

- Bootstrap escape hatches are OK.
- Public listener/cluster escape hatches are OK.
- Upstream listener/cluster escape hatches are not supported.

If an unsupported escape hatch is configured and the discovery chain is
activated log a warning and act like it was not configured.

Fixes #6160

agent/xds/clusters.go

@@ -235,6 +235,10 @@ func (s *Server) makeUpstreamClustersForDiscoveryChain(
 		s.Logger.Printf("[WARN] envoy: failed to parse Upstream[%s].Config: %s",
 			upstream.Identifier(), err)
 	}
+	if cfg.ClusterJSON != "" {
+		s.Logger.Printf("[WARN] envoy: ignoring escape hatch setting Upstream[%s].Config[%s] because a discovery chain for %q is configured",
+			upstream.Identifier(), "envoy_cluster_json", chain.ServiceName)
+	}
 
 	if chain == nil {
 		panic("chain must be provided")

agent/xds/config.go

@@ -17,10 +17,14 @@ type ProxyConfig struct {
 	// validation context will be injected overriding any TLS settings present. An
 	// AuthZ filter will also be prepended to each filterChain provided to enforce
 	// Connect's access control.
+	//
+	// Note: This escape hatch is compatible with the discovery chain.
 	PublicListenerJSON string `mapstructure:"envoy_public_listener_json"`
 
 	// LocalClusterJSON is a complete override ("escape hatch") for the
 	// local application cluster.
+	//
+	// Note: This escape hatch is compatible with the discovery chain.
 	LocalClusterJSON string `mapstructure:"envoy_local_cluster_json"`
 
 	// LocalConnectTimeoutMs is the number of milliseconds to timeout making a new
@@ -105,11 +109,17 @@ func ParseMeshGatewayConfig(m map[string]interface{}) (MeshGatewayConfig, error)
 type UpstreamConfig struct {
 	// ListenerJSON is a complete override ("escape hatch") for the upstream's
 	// listener.
+	//
+	// Note: This escape hatch is NOT compatible with the discovery chain and
+	// will be ignored if a discovery chain is active.
 	ListenerJSON string `mapstructure:"envoy_listener_json"`
 
 	// ClusterJSON is a complete override ("escape hatch") for the upstream's
 	// cluster. The Connect client TLS certificate and context will be injected
 	// overriding any TLS settings present.
+	//
+	// Note: This escape hatch is NOT compatible with the discovery chain and
+	// will be ignored if a discovery chain is active.
 	ClusterJSON string `mapstructure:"envoy_cluster_json"`
 
 	// Protocol describes the upstream's service protocol. Valid values are "tcp",

agent/xds/listeners.go

@@ -378,7 +378,6 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain(
 	chain *structs.CompiledDiscoveryChain,
 	cfgSnap *proxycfg.ConfigSnapshot,
 ) (proto.Message, error) {
-	// TODO(rb): make the listener escape hatch work again
 	cfg, err := ParseUpstreamConfigNoDefaults(u.Config)
 	if err != nil {
 		// Don't hard fail on a config typo, just warn. The parse func returns
@@ -386,6 +385,10 @@ func (s *Server) makeUpstreamListenerForDiscoveryChain(
 		s.Logger.Printf("[WARN] envoy: failed to parse Upstream[%s].Config: %s",
 			u.Identifier(), err)
 	}
+	if cfg.ListenerJSON != "" {
+		s.Logger.Printf("[WARN] envoy: ignoring escape hatch setting Upstream[%s].Config[%s] because a discovery chain for %q is configured",
+			u.Identifier(), "envoy_listener_json", chain.ServiceName)
+	}
 
 	addr := u.LocalBindAddress
 	if addr == "" {

website/source/docs/connect/l7-traffic-management.html.md

@@ -9,8 +9,9 @@ description: |-
 # L7 Traffic Management <sup>(beta)</sup>
 
 -> **Note:** This feature is not compatible with the 
-[built-in proxy](/docs/connect/proxies/built-in.html)
+[built-in proxy](/docs/connect/proxies/built-in.html),
-or [native proxies](/docs/connect/native.html).
+[native proxies](/docs/connect/native.html),
+and some [Envoy proxy escape hatches](/docs/connect/proxies/envoy.html#escape-hatch-overrides).
 
 Layer 7 traffic management allows operators to divide L7 traffic between
 different

website/source/docs/connect/proxies/envoy.md

@@ -18,13 +18,13 @@ Consul can configure Envoy sidecars to proxy http/1.1, http2 or gRPC traffic at
 L7 or any other tcp-based protocol at L4. Prior to Consul 1.5.0 Envoy proxies
 could only proxy tcp at L4.
 
-Currently configuration of additional L7 features is limited, however we have
+Configuration of some [L7 features](/docs/connect/l7-traffic-management.html)
-plans to support a wider range of features in the next major release
+is possible via [configuration entries](/docs/agent/config_entries.html). If
-cycle.
+you wish to use an Envoy feature not currently exposed through these config
-
+entries as an interim solution, you can add [custom Envoy
-As an interim solution, you can add [custom Envoy configuration](#custom-configuration)
+configuration](#advanced-configuration) in the [proxy service
-in the [proxy service definition](/docs/connect/registration/service-registration.html) allowing
+definition](/docs/connect/registration/service-registration.html) allowing you
-you to use the more powerful features of Envoy.
+to use the more powerful features of Envoy.
 
 ~> **Note:** When using Envoy with Consul and not using the [`consul connect envoy` command](/docs/commands/connect/envoy.html)
    Envoy must be run with the `--max-obj-name-len` option set to `256` or greater.
@@ -180,6 +180,7 @@ defaults that are inherited by all services.
 
 - `protocol` - The protocol the service speaks. Connect's Envoy integration
   currently supports the following `protocol` values:
+
   - `tcp` - Unless otherwise specified this is the default, which causes Envoy
     to proxy at L4. This provides all the security benefits of Connect's mTLS
     and works for any TCP-based protocol. Load-balancing and metrics are
@@ -199,10 +200,22 @@ defaults that are inherited by all services.
     filter](https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/http_filters/grpc_http1_bridge_filter#config-http-filters-grpc-bridge)
     that translates HTTP/1.1 calls into gRPC, and instruments
     metrics with `gRPC-status` trailer codes.
+
+    ~> **Note:** The protocol of a service should ideally be configured via the
+    [`protocol`](/docs/agent/config-entries/service-defaults.html#protocol)
+    field of a
+    [`service-defaults`](/docs/agent/config-entries/service-defaults.html)
+    config entry for the service. Configuring it in a
+    proxy config will not fully enable some [L7
+    features](/docs/connect/l7-traffic-management.html).
+    It is supported here for backwards compatibility with Consul versions prior to 1.6.0.
+
 - `bind_address` - Override the address Envoy's public listener binds to. By
   default Envoy will bind to the service address or 0.0.0.0 if there is not explicit address on the service registration.
+
 - `bind_port` - Override the port Envoy's public listener binds to. By default
   Envoy will bind to the service port.
+
 - `local_connect_timeout_ms` - The number of milliseconds allowed to make
   connections to the local application instance before timing out. Defaults to 5000
   (5 seconds).
@@ -216,10 +229,29 @@ definition](/docs/connect/registration/service-registration.html) or
 
 - `protocol` - Same as above in main config but affects the listener setup for
   the upstream.
+
+    ~> **Note:** The protocol of a service should ideally be configured via the
+    [`protocol`](/docs/agent/config-entries/service-defaults.html#protocol)
+    field of a
+    [`service-defaults`](/docs/agent/config-entries/service-defaults.html)
+    config entry for the upstream destination service. Configuring it in a
+    proxy upstream config will not fully enable some [L7
+    features](/docs/connect/l7-traffic-management.html).
+    It is supported here for backwards compatibility with Consul versions prior to 1.6.0.
+
 - `connect_timeout_ms` - The number of milliseconds to allow when making upstream
   connections before timing out. Defaults to 5000
   (5 seconds).
 
+    ~> **Note:** The connection timeout for a service should ideally be
+    configured via the
+    [`connect_timeout`](/docs/agent/config-entries/service-resolver.html#connecttimeout)
+    field of a
+    [`service-resolver`](/docs/agent/config-entries/service-resolver.html)
+    config entry for the upstream destination service. Configuring it in a
+    proxy upstream config will override any values defined in config entries.
+    It is supported here for backwards compatibility with Consul versions prior to 1.6.0.
+
 ### Mesh Gateway Options <sup>(beta)</sup>
 
 These fields may also be overridden explicitly in the [proxy service
@@ -359,6 +391,13 @@ The following configuration items may be overridden directly in the
 definition](/docs/connect/registration/service-registration.html) or
 [`sidecar_service`](/docs/connect/registration/sidecar-service.html) block.
 
+~> **Note:** - When a
+[`service-router`](/docs/agent/config-entries/service-router.html),
+[`service-splitter`](/docs/agent/config-entries/service-splitter.html), or
+[`service-resolver`](/docs/agent/config-entries/service-resolver.html) config
+entry exists for a service the below escape hatches are ignored and will log a
+warning.
+
 - `envoy_listener_json` - Specifies a complete
   [Listener](https://www.envoyproxy.io/docs/envoy/v1.10.0/api-v2/api/v2/lds.proto)
   to be delivered in place of the upstream listener that the proxy exposes to