Merge pull request #14294 from hashicorp/derekm/split-grpc-ports

Add separate grpc tls port.

.changelog/14294.txt

@@ -0,0 +1,6 @@
+```release-note:breaking-change
+config: Add new `ports.grpc_tls` configuration option.
+Introduce a new port to better separate TLS config from the existing `ports.grpc` config.
+The new `ports.grpc_tls` only supports TLS encrypted communication.
+The existing `ports.grpc` currently supports both plain-text and tls communication, but tls support will be removed in a future release.
+```

agent/agent.go

@@ -213,7 +213,7 @@ type Agent struct {
 	// depending on the configuration
 	delegate delegate
 
-	// externalGRPCServer is the gRPC server exposed on the dedicated gRPC port (as
+	// externalGRPCServer is the gRPC server exposed on dedicated gRPC ports (as
 	// opposed to the multiplexed "server" port).
 	externalGRPCServer *grpc.Server
 
@@ -384,18 +384,18 @@ type Agent struct {
 
 // New process the desired options and creates a new Agent.
 // This process will
-//   * parse the config given the config Flags
+//   - parse the config given the config Flags
-//   * setup logging
+//   - setup logging
-//      * using predefined logger given in an option
+//   - using predefined logger given in an option
-//        OR
+//     OR
-//      * initialize a new logger from the configuration
+//   - initialize a new logger from the configuration
-//        including setting up gRPC logging
+//     including setting up gRPC logging
-//   * initialize telemetry
+//   - initialize telemetry
-//   * create a TLS Configurator
+//   - create a TLS Configurator
-//   * build a shared connection pool
+//   - build a shared connection pool
-//   * create the ServiceManager
+//   - create the ServiceManager
-//   * setup the NodeID if one isn't provided in the configuration
+//   - setup the NodeID if one isn't provided in the configuration
-//   * create the AutoConfig object for future use in fully
+//   - create the AutoConfig object for future use in fully
 //     resolving the configuration
 func New(bd BaseDeps) (*Agent, error) {
 	a := Agent{
@@ -539,7 +539,7 @@ func (a *Agent) Start(ctx context.Context) error {
 
 	// This needs to happen after the initial auto-config is loaded, because TLS
 	// can only be configured on the gRPC server at the point of creation.
-	a.buildExternalGRPCServer()
+	a.externalGRPCServer = external.NewServer(a.logger.Named("grpc.external"))
 
 	if err := a.startLicenseManager(ctx); err != nil {
 		return err
@@ -702,7 +702,7 @@ func (a *Agent) Start(ctx context.Context) error {
 		a.apiServers.Start(srv)
 	}
 
-	// Start gRPC server.
+	// Start grpc and grpc_tls servers.
 	if err := a.listenAndServeGRPC(); err != nil {
 		return err
 	}
@@ -760,15 +760,10 @@ func (a *Agent) Failed() <-chan struct{} {
 	return a.apiServers.failed
 }
 
-func (a *Agent) buildExternalGRPCServer() {
-	a.externalGRPCServer = external.NewServer(a.logger.Named("grpc.external"), a.tlsConfigurator)
-}
-
 func (a *Agent) listenAndServeGRPC() error {
-	if len(a.config.GRPCAddrs) < 1 {
+	if len(a.config.GRPCAddrs) < 1 && len(a.config.GRPCTLSAddrs) < 1 {
 		return nil
 	}
-
 	// TODO(agentless): rather than asserting the concrete type of delegate, we
 	// should add a method to the Delegate interface to build a ConfigSource.
 	var cfg xds.ProxyConfigSource = localproxycfg.NewConfigSource(a.proxyConfig)
@@ -787,7 +782,6 @@ func (a *Agent) listenAndServeGRPC() error {
 		}()
 		cfg = catalogCfg
 	}
-
 	a.xdsServer = xds.NewServer(
 		a.config.NodeName,
 		a.logger.Named(logging.Envoy),
@@ -800,22 +794,61 @@ func (a *Agent) listenAndServeGRPC() error {
 	)
 	a.xdsServer.Register(a.externalGRPCServer)
 
-	ln, err := a.startListeners(a.config.GRPCAddrs)
+	// Attempt to spawn listeners
-	if err != nil {
+	var listeners []net.Listener
-		return err
+	start := func(port_name string, addrs []net.Addr, tlsConf *tls.Config) error {
+		if len(addrs) < 1 {
+			return nil
+		}
+
+		ln, err := a.startListeners(addrs)
+		if err != nil {
+			return err
+		}
+		for i := range ln {
+			// Wrap with TLS, if provided.
+			if tlsConf != nil {
+				ln[i] = tls.NewListener(ln[i], tlsConf)
+			}
+			listeners = append(listeners, ln[i])
+		}
+
+		for _, l := range ln {
+			go func(innerL net.Listener) {
+				a.logger.Info("Started gRPC listeners",
+					"port_name", port_name,
+					"address", innerL.Addr().String(),
+					"network", innerL.Addr().Network(),
+				)
+				err := a.externalGRPCServer.Serve(innerL)
+				if err != nil {
+					a.logger.Error("gRPC server failed", "port_name", port_name, "error", err)
+				}
+			}(l)
+		}
+		return nil
 	}
 
-	for _, l := range ln {
+	// The original grpc port may spawn in either plain-text or TLS mode (for backwards compatibility).
-		go func(innerL net.Listener) {
+	// TODO: Simplify this block to only spawn plain-text after 1.14 when deprecated TLS support is removed.
-			a.logger.Info("Started gRPC server",
+	if a.config.GRPCPort > 0 {
-				"address", innerL.Addr().String(),
+		// Only allow the grpc port to spawn TLS connections if the other grpc_tls port is NOT defined.
-				"network", innerL.Addr().Network(),
+		var tlsConf *tls.Config = nil
-			)
+		if a.config.GRPCTLSPort <= 0 && a.tlsConfigurator.GRPCServerUseTLS() {
-			err := a.externalGRPCServer.Serve(innerL)
+			a.logger.Warn("deprecated gRPC TLS configuration detected. Consider using `ports.grpc_tls` instead")
-			if err != nil {
+			tlsConf = a.tlsConfigurator.IncomingGRPCConfig()
-				a.logger.Error("gRPC server failed", "error", err)
+		}
-			}
+		if err := start("grpc", a.config.GRPCAddrs, tlsConf); err != nil {
-		}(l)
+			closeListeners(listeners)
+			return err
+		}
+	}
+	// Only allow grpc_tls to spawn with a TLS listener.
+	if a.config.GRPCTLSPort > 0 {
+		if err := start("grpc_tls", a.config.GRPCTLSAddrs, a.tlsConfigurator.IncomingGRPCConfig()); err != nil {
+			closeListeners(listeners)
+			return err
+		}
 	}
 	return nil
 }
@@ -1203,6 +1236,7 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co
 	cfg.RPCAdvertise = runtimeCfg.RPCAdvertiseAddr
 
 	cfg.GRPCPort = runtimeCfg.GRPCPort
+	cfg.GRPCTLSPort = runtimeCfg.GRPCTLSPort
 
 	cfg.Segment = runtimeCfg.SegmentName
 	if len(runtimeCfg.Segments) > 0 {
@@ -1506,7 +1540,9 @@ func (a *Agent) ShutdownAgent() error {
 	}
 
 	// Stop gRPC
-	a.externalGRPCServer.Stop()
+	if a.externalGRPCServer != nil {
+		a.externalGRPCServer.Stop()
+	}
 
 	// Stop the proxy config manager
 	if a.proxyConfig != nil {

agent/agent_endpoint.go

@@ -45,7 +45,19 @@ type Self struct {
 
 type XDSSelf struct {
 	SupportedProxies map[string][]string
-	Port             int
+	// Port could be used for either TLS or plain-text communication
+	// up through version 1.14. In order to maintain backwards-compatibility,
+	// Port will now default to TLS and fallback to the standard port value.
+	// DEPRECATED: Use Ports field instead
+	Port  int
+	Ports GRPCPorts
+}
+
+// GRPCPorts is used to hold the external GRPC server's port numbers.
+type GRPCPorts struct {
+	// Technically, this port is not always plain-text as of 1.14, but will be in a future release.
+	Plaintext int
+	TLS       int
 }
 
 func (s *HTTPHandlers) AgentSelf(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
@@ -78,7 +90,16 @@ func (s *HTTPHandlers) AgentSelf(resp http.ResponseWriter, req *http.Request) (i
 			SupportedProxies: map[string][]string{
 				"envoy": proxysupport.EnvoyVersions,
 			},
-			Port: s.agent.config.GRPCPort,
+			// Prefer the TLS port. See comment on the XDSSelf struct for details.
+			Port: s.agent.config.GRPCTLSPort,
+			Ports: GRPCPorts{
+				Plaintext: s.agent.config.GRPCPort,
+				TLS:       s.agent.config.GRPCTLSPort,
+			},
+		}
+		// Fallback to standard port if TLS is not enabled.
+		if s.agent.config.GRPCTLSPort <= 0 {
+			xds.Port = s.agent.config.GRPCPort
 		}
 	}
 

agent/agent_endpoint_test.go

@@ -1434,15 +1434,8 @@ func TestAgent_Self(t *testing.T) {
 	cases := map[string]struct {
 		hcl       string
 		expectXDS bool
+		grpcTLS   bool
 	}{
-		"normal": {
-			hcl: `
-			node_meta {
-				somekey = "somevalue"
-			}
-			`,
-			expectXDS: true,
-		},
 		"no grpc": {
 			hcl: `
 			node_meta {
@@ -1453,13 +1446,35 @@ func TestAgent_Self(t *testing.T) {
 			}
 			`,
 			expectXDS: false,
+			grpcTLS:   false,
+		},
+		"plaintext grpc": {
+			hcl: `
+			node_meta {
+				somekey = "somevalue"
+			}
+			`,
+			expectXDS: true,
+			grpcTLS:   false,
+		},
+		"tls grpc": {
+			hcl: `
+				node_meta {
+					somekey = "somevalue"
+				}
+				`,
+			expectXDS: true,
+			grpcTLS:   true,
 		},
 	}
 
 	for name, tc := range cases {
 		tc := tc
 		t.Run(name, func(t *testing.T) {
-			a := NewTestAgent(t, tc.hcl)
+			a := StartTestAgent(t, TestAgent{
+				HCL:        tc.hcl,
+				UseGRPCTLS: tc.grpcTLS,
+			})
 			defer a.Shutdown()
 
 			testrpc.WaitForTestAgent(t, a.RPC, "dc1")
@@ -1487,6 +1502,13 @@ func TestAgent_Self(t *testing.T) {
 					map[string][]string{"envoy": proxysupport.EnvoyVersions},
 					val.XDS.SupportedProxies,
 				)
+				require.Equal(t, a.Config.GRPCTLSPort, val.XDS.Ports.TLS)
+				require.Equal(t, a.Config.GRPCPort, val.XDS.Ports.Plaintext)
+				if tc.grpcTLS {
+					require.Equal(t, a.Config.GRPCTLSPort, val.XDS.Port)
+				} else {
+					require.Equal(t, a.Config.GRPCPort, val.XDS.Port)
+				}
 
 			} else {
 				require.Nil(t, val.XDS, "xds component should be missing when gRPC is disabled")

agent/agent_test.go

@@ -2095,7 +2095,7 @@ func TestAgent_HTTPCheck_EnableAgentTLSForChecks(t *testing.T) {
 
 	run := func(t *testing.T, ca string) {
 		a := StartTestAgent(t, TestAgent{
-			UseTLS: true,
+			UseHTTPS: true,
 			HCL: `
 				enable_agent_tls_for_checks = true
 
@@ -3873,7 +3873,7 @@ func TestAgent_reloadWatchesHTTPS(t *testing.T) {
 	}
 
 	t.Parallel()
-	a := TestAgent{UseTLS: true}
+	a := TestAgent{UseHTTPS: true}
 	if err := a.Start(t); err != nil {
 		t.Fatal(err)
 	}
@@ -5220,7 +5220,7 @@ func TestAgent_AutoEncrypt(t *testing.T) {
 			server = ` + strconv.Itoa(srv.Config.RPCBindAddr.Port) + `
 		}
 		retry_join = ["` + srv.Config.SerfBindAddrLAN.String() + `"]`,
-		UseTLS: true,
+		UseHTTPS: true,
 	})
 
 	defer client.Shutdown()

agent/config/builder.go

@@ -125,10 +125,10 @@ type LoadResult struct {
 //
 // The sources are merged in the following order:
 //
-//  * default configuration
+//   - default configuration
-//  * config files in alphabetical order
+//   - config files in alphabetical order
-//  * command line arguments
+//   - command line arguments
-//  * overrides
+//   - overrides
 //
 // The config sources are merged sequentially and later values overwrite
 // previously set values. Slice values are merged by concatenating the two slices.
@@ -433,6 +433,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 	httpsPort := b.portVal("ports.https", c.Ports.HTTPS)
 	serverPort := b.portVal("ports.server", c.Ports.Server)
 	grpcPort := b.portVal("ports.grpc", c.Ports.GRPC)
+	grpcTlsPort := b.portVal("ports.grpc_tls", c.Ports.GRPCTLS)
 	serfPortLAN := b.portVal("ports.serf_lan", c.Ports.SerfLAN)
 	serfPortWAN := b.portVal("ports.serf_wan", c.Ports.SerfWAN)
 	proxyMinPort := b.portVal("ports.proxy_min_port", c.Ports.ProxyMinPort)
@@ -563,6 +564,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 	httpAddrs := b.makeAddrs(b.expandAddrs("addresses.http", c.Addresses.HTTP), clientAddrs, httpPort)
 	httpsAddrs := b.makeAddrs(b.expandAddrs("addresses.https", c.Addresses.HTTPS), clientAddrs, httpsPort)
 	grpcAddrs := b.makeAddrs(b.expandAddrs("addresses.grpc", c.Addresses.GRPC), clientAddrs, grpcPort)
+	grpcTlsAddrs := b.makeAddrs(b.expandAddrs("addresses.grpc_tls", c.Addresses.GRPCTLS), clientAddrs, grpcTlsPort)
 
 	for _, a := range dnsAddrs {
 		if x, ok := a.(*net.TCPAddr); ok {
@@ -987,8 +989,10 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		EnableRemoteScriptChecks:   enableRemoteScriptChecks,
 		EnableLocalScriptChecks:    enableLocalScriptChecks,
 		EncryptKey:                 stringVal(c.EncryptKey),
-		GRPCPort:                   grpcPort,
 		GRPCAddrs:                  grpcAddrs,
+		GRPCPort:                   grpcPort,
+		GRPCTLSAddrs:               grpcTlsAddrs,
+		GRPCTLSPort:                grpcTlsPort,
 		HTTPMaxConnsPerClient:      intVal(c.Limits.HTTPMaxConnsPerClient),
 		HTTPSHandshakeTimeout:      b.durationVal("limits.https_handshake_timeout", c.Limits.HTTPSHandshakeTimeout),
 		KVMaxValueSize:             uint64Val(c.Limits.KVMaxValueSize),

agent/config/config.go

@@ -332,10 +332,11 @@ type Consul struct {
 }
 
 type Addresses struct {
-	DNS   *string `mapstructure:"dns"`
+	DNS     *string `mapstructure:"dns"`
-	HTTP  *string `mapstructure:"http"`
+	HTTP    *string `mapstructure:"http"`
-	HTTPS *string `mapstructure:"https"`
+	HTTPS   *string `mapstructure:"https"`
-	GRPC  *string `mapstructure:"grpc"`
+	GRPC    *string `mapstructure:"grpc"`
+	GRPCTLS *string `mapstructure:"grpc_tls"`
 }
 
 type AdvertiseAddrsConfig struct {
@@ -694,6 +695,7 @@ type Ports struct {
 	SerfWAN        *int `mapstructure:"serf_wan"`
 	Server         *int `mapstructure:"server"`
 	GRPC           *int `mapstructure:"grpc"`
+	GRPCTLS        *int `mapstructure:"grpc_tls"`
 	ProxyMinPort   *int `mapstructure:"proxy_min_port"`
 	ProxyMaxPort   *int `mapstructure:"proxy_max_port"`
 	SidecarMinPort *int `mapstructure:"sidecar_min_port"`

agent/config/flags.go

@@ -53,7 +53,8 @@ func AddFlags(fs *flag.FlagSet, f *LoadOpts) {
 	add(&f.FlagValues.EnableLocalScriptChecks, "enable-local-script-checks", "Enables health check scripts from configuration file.")
 	add(&f.FlagValues.HTTPConfig.AllowWriteHTTPFrom, "allow-write-http-from", "Only allow write endpoint calls from given network. CIDR format, can be specified multiple times.")
 	add(&f.FlagValues.EncryptKey, "encrypt", "Provides the gossip encryption key.")
-	add(&f.FlagValues.Ports.GRPC, "grpc-port", "Sets the gRPC API port to listen on (currently needed for Envoy xDS only).")
+	add(&f.FlagValues.Ports.GRPC, "grpc-port", "Sets the gRPC API port to listen on.")
+	add(&f.FlagValues.Ports.GRPCTLS, "grpc-tls-port", "Sets the gRPC-TLS API port to listen on.")
 	add(&f.FlagValues.Ports.HTTP, "http-port", "Sets the HTTP API port to listen on.")
 	add(&f.FlagValues.Ports.HTTPS, "https-port", "Sets the HTTPS API port to listen on.")
 	add(&f.FlagValues.StartJoinAddrsLAN, "join", "Address of an agent to join at start time. Can be specified multiple times.")

agent/config/runtime.go

@@ -670,13 +670,18 @@ type RuntimeConfig struct {
 	// flag: -encrypt string
 	EncryptKey string
 
-	// GRPCPort is the port the gRPC server listens on. Currently this only
+	// GRPCPort is the port the gRPC server listens on. It is disabled by default.
-	// exposes the xDS and ext_authz APIs for Envoy and it is disabled by default.
 	//
 	// hcl: ports { grpc = int }
 	// flags: -grpc-port int
 	GRPCPort int
 
+	// GRPCTLSPort is the port the gRPC server listens on. It is disabled by default.
+	//
+	// hcl: ports { grpc_tls = int }
+	// flags: -grpc-tls-port int
+	GRPCTLSPort int
+
 	// GRPCAddrs contains the list of TCP addresses and UNIX sockets the gRPC
 	// server will bind to. If the gRPC endpoint is disabled (ports.grpc <= 0)
 	// the list is empty.
@@ -692,6 +697,21 @@ type RuntimeConfig struct {
 	// hcl: client_addr = string addresses { grpc = string } ports { grpc = int }
 	GRPCAddrs []net.Addr
 
+	// GRPCTLSAddrs contains the list of TCP addresses and UNIX sockets the gRPC
+	// server will bind to. If the gRPC endpoint is disabled (ports.grpc <= 0)
+	// the list is empty.
+	//
+	// The addresses are taken from 'addresses.grpc_tls' which should contain a
+	// space separated list of ip addresses, UNIX socket paths and/or
+	// go-sockaddr templates. UNIX socket paths must be written as
+	// 'unix://<full path>', e.g. 'unix:///var/run/consul-grpc.sock'.
+	//
+	// If 'addresses.grpc_tls' was not provided the 'client_addr' addresses are
+	// used.
+	//
+	// hcl: client_addr = string addresses { grpc_tls = string } ports { grpc_tls = int }
+	GRPCTLSAddrs []net.Addr
+
 	// HTTPAddrs contains the list of TCP addresses and UNIX sockets the HTTP
 	// server will bind to. If the HTTP endpoint is disabled (ports.http <= 0)
 	// the list is empty.

agent/config/runtime_test.go

@@ -6016,6 +6016,8 @@ func TestLoad_FullConfig(t *testing.T) {
 
 		GRPCPort:              4881,
 		GRPCAddrs:             []net.Addr{tcpAddr("32.31.61.91:4881")},
+		GRPCTLSPort:           5201,
+		GRPCTLSAddrs:          []net.Addr{tcpAddr("23.14.88.19:5201")},
 		HTTPAddrs:             []net.Addr{tcpAddr("83.39.91.39:7999")},
 		HTTPBlockEndpoints:    []string{"RBvAFcGD", "fWOWFznh"},
 		AllowWriteHTTPFrom:    []*net.IPNet{cidr("127.0.0.0/8"), cidr("22.33.44.55/32"), cidr("0.0.0.0/0")},

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -192,6 +192,8 @@
     "ExposeMinPort": 0,
     "GRPCAddrs": [],
     "GRPCPort": 0,
+    "GRPCTLSAddrs": [],
+    "GRPCTLSPort": 0,
     "GossipLANGossipInterval": "0s",
     "GossipLANGossipNodes": 0,
     "GossipLANProbeInterval": "0s",

agent/config/testdata/full-config.hcl

@@ -44,6 +44,7 @@ addresses = {
     http = "83.39.91.39"
     https = "95.17.17.19"
     grpc = "32.31.61.91"
+    grpc_tls = "23.14.88.19"
 }
 advertise_addr = "17.99.29.16"
 advertise_addr_wan = "78.63.37.19"
@@ -320,6 +321,7 @@ ports {
     https = 15127
     server = 3757
     grpc = 4881
+    grpc_tls = 5201
     proxy_min_port = 2000
     proxy_max_port = 3000
     sidecar_min_port = 8888

agent/config/testdata/full-config.json

@@ -44,7 +44,8 @@
     "dns": "93.95.95.81",
     "http": "83.39.91.39",
     "https": "95.17.17.19",
-    "grpc": "32.31.61.91"
+    "grpc": "32.31.61.91",
+    "grpc_tls": "23.14.88.19"
   },
   "advertise_addr": "17.99.29.16",
   "advertise_addr_wan": "78.63.37.19",
@@ -320,6 +321,7 @@
     "https": 15127,
     "server": 3757,
     "grpc": 4881,
+    "grpc_tls": 5201,
     "sidecar_min_port": 8888,
     "sidecar_max_port": 9999,
     "expose_min_port": 1111,

agent/consul/config.go

@@ -133,6 +133,9 @@ type Config struct {
 	// GRPCPort is the port the public gRPC server listens on.
 	GRPCPort int
 
+	// GRPCTLSPort is the port the public gRPC TLS server listens on.
+	GRPCTLSPort int
+
 	// (Enterprise-only) The network segment this agent is part of.
 	Segment string
 

agent/consul/leader.go

@@ -1073,9 +1073,11 @@ func (s *Server) handleAliveMember(member serf.Member, nodeEntMeta *acl.Enterpri
 			},
 		}
 
-		grpcPortStr := member.Tags["grpc_port"]
+		if parts.ExternalGRPCPort > 0 {
-		if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 {
+			service.Meta["grpc_port"] = strconv.Itoa(parts.ExternalGRPCPort)
-			service.Meta["grpc_port"] = grpcPortStr
+		}
+		if parts.ExternalGRPCTLSPort > 0 {
+			service.Meta["grpc_tls_port"] = strconv.Itoa(parts.ExternalGRPCTLSPort)
 		}
 
 		// Attempt to join the consul server

agent/consul/leader_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -19,6 +20,7 @@ import (
 	"github.com/hashicorp/consul/agent/structs"
 	tokenStore "github.com/hashicorp/consul/agent/token"
 	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/freeport"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/consul/testrpc"
@@ -355,8 +357,10 @@ func TestLeader_CheckServersMeta(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
-
 	t.Parallel()
+
+	ports := freeport.GetN(t, 2) // s3 grpc, s3 grpc_tls
+
 	dir1, s1 := testServerWithConfig(t, func(c *Config) {
 		c.PrimaryDatacenter = "dc1"
 		c.ACLsEnabled = true
@@ -383,6 +387,8 @@ func TestLeader_CheckServersMeta(t *testing.T) {
 		c.ACLInitialManagementToken = "root"
 		c.ACLResolverSettings.ACLDefaultPolicy = "allow"
 		c.Bootstrap = false
+		c.GRPCPort = ports[0]
+		c.GRPCTLSPort = ports[1]
 	})
 	defer os.RemoveAll(dir3)
 	defer s3.Shutdown()
@@ -456,6 +462,14 @@ func TestLeader_CheckServersMeta(t *testing.T) {
 		if newVersion != versionToExpect {
 			r.Fatalf("Expected version to be updated to %s, was %s", versionToExpect, newVersion)
 		}
+		grpcPort := service.Meta["grpc_port"]
+		if grpcPort != strconv.Itoa(ports[0]) {
+			r.Fatalf("Expected grpc port to be %d, was %s", ports[0], grpcPort)
+		}
+		grpcTLSPort := service.Meta["grpc_tls_port"]
+		if grpcTLSPort != strconv.Itoa(ports[1]) {
+			r.Fatalf("Expected grpc tls port to be %d, was %s", ports[1], grpcTLSPort)
+		}
 	})
 }
 

agent/consul/peering_backend.go

@@ -66,11 +66,19 @@ func (b *PeeringBackend) GetServerAddresses() ([]string, error) {
 	}
 	var addrs []string
 	for _, node := range nodes {
-		grpcPortStr := node.ServiceMeta["grpc_port"]
+		// Prefer the TLS port if it is defined.
-		if v, err := strconv.Atoi(grpcPortStr); err != nil || v < 1 {
+		grpcPortStr := node.ServiceMeta["grpc_tls_port"]
-			continue // skip server that isn't exporting public gRPC properly
+		if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 {
+			addrs = append(addrs, node.Address+":"+grpcPortStr)
+			continue
 		}
-		addrs = append(addrs, node.Address+":"+grpcPortStr)
+		// Fallback to the standard port if TLS is not defined.
+		grpcPortStr = node.ServiceMeta["grpc_port"]
+		if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 {
+			addrs = append(addrs, node.Address+":"+grpcPortStr)
+			continue
+		}
+		// Skip node if neither defined.
 	}
 	if len(addrs) == 0 {
 		return nil, fmt.Errorf("a grpc bind port must be specified in the configuration for all servers")

agent/consul/server.go

@@ -253,7 +253,7 @@ type Server struct {
 	// enable RPC forwarding.
 	externalConnectCAServer *connectca.Server
 
-	// externalGRPCServer is the gRPC server exposed on the dedicated gRPC port, as
+	// externalGRPCServer has a gRPC server exposed on the dedicated gRPC ports, as
 	// opposed to the multiplexed "server" port which is served by grpcHandler.
 	externalGRPCServer *grpc.Server
 
@@ -377,7 +377,6 @@ type Server struct {
 	// embedded struct to hold all the enterprise specific data
 	EnterpriseServer
 }
-
 type connHandler interface {
 	Run() error
 	Handle(conn net.Conn)

agent/consul/server_serf.go

@@ -107,6 +107,9 @@ func (s *Server) setupSerfConfig(opts setupSerfOptions) (*serf.Config, error) {
 	if s.config.GRPCPort > 0 {
 		conf.Tags["grpc_port"] = fmt.Sprintf("%d", s.config.GRPCPort)
 	}
+	if s.config.GRPCTLSPort > 0 {
+		conf.Tags["grpc_tls_port"] = fmt.Sprintf("%d", s.config.GRPCTLSPort)
+	}
 	if s.config.Bootstrap {
 		conf.Tags["bootstrap"] = "1"
 	}

agent/consul/server_test.go

@@ -1,6 +1,7 @@
 package consul
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"fmt"
 	"net"
@@ -218,9 +219,10 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S
 	var dir string
 	var srv *Server
 
+	var config *Config
+	var deps Deps
 	// Retry added to avoid cases where bind addr is already in use
 	retry.RunWith(retry.ThreeTimes(), t, func(r *retry.R) {
-		var config *Config
 		dir, config = testServerConfig(t)
 		for _, fn := range configOpts {
 			fn(config)
@@ -234,7 +236,8 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S
 		config.ACLResolverSettings.EnterpriseMeta = *config.AgentEnterpriseMeta()
 
 		var err error
-		srv, err = newServer(t, config)
+		deps = newDefaultDeps(t, config)
+		srv, err = newServerWithDeps(t, config, deps)
 		if err != nil {
 			r.Fatalf("err: %v", err)
 		}
@@ -245,9 +248,14 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S
 		// Normally the gRPC server listener is created at the agent level and
 		// passed down into the Server creation.
 		externalGRPCAddr := fmt.Sprintf("127.0.0.1:%d", srv.config.GRPCPort)
-
 		ln, err := net.Listen("tcp", externalGRPCAddr)
 		require.NoError(t, err)
+
+		// Wrap the listener with TLS
+		if deps.TLSConfigurator.GRPCServerUseTLS() {
+			ln = tls.NewListener(ln, deps.TLSConfigurator.IncomingGRPCConfig())
+		}
+
 		go func() {
 			_ = srv.externalGRPCServer.Serve(ln)
 		}()
@@ -300,8 +308,8 @@ func newServerWithDeps(t *testing.T, c *Config, deps Deps) (*Server, error) {
 			oldNotify()
 		}
 	}
-
+	grpcServer := external.NewServer(deps.Logger.Named("grpc.external"))
-	srv, err := NewServer(c, deps, external.NewServer(deps.Logger.Named("grpc.external"), deps.TLSConfigurator))
+	srv, err := NewServer(c, deps, grpcServer)
 	if err != nil {
 		return nil, err
 	}

agent/grpc-external/server.go

@@ -6,16 +6,14 @@ import (
 	middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
 
 	agentmiddleware "github.com/hashicorp/consul/agent/grpc-middleware"
-	"github.com/hashicorp/consul/tlsutil"
 )
 
 // NewServer constructs a gRPC server for the external gRPC port, to which
 // handlers can be registered.
-func NewServer(logger agentmiddleware.Logger, tls *tlsutil.Configurator) *grpc.Server {
+func NewServer(logger agentmiddleware.Logger) *grpc.Server {
 	recoveryOpts := agentmiddleware.PanicHandlerMiddlewareOpts(logger)
 
 	opts := []grpc.ServerOption{
@@ -35,9 +33,5 @@ func NewServer(logger agentmiddleware.Logger, tls *tlsutil.Configurator) *grpc.S
 			MinTime: 15 * time.Second,
 		}),
 	}
-	if tls != nil && tls.GRPCServerUseTLS() {
-		creds := credentials.NewTLS(tls.IncomingGRPCConfig())
-		opts = append(opts, grpc.Creds(creds))
-	}
 	return grpc.NewServer(opts...)
 }

agent/http_test.go

@@ -148,7 +148,7 @@ func TestSetupHTTPServer_HTTP2(t *testing.T) {
 
 	// Fire up an agent with TLS enabled.
 	a := StartTestAgent(t, TestAgent{
-		UseTLS: true,
+		UseHTTPS: true,
 		HCL: `
 			key_file = "../test/client_certs/server.key"
 			cert_file = "../test/client_certs/server.crt"
@@ -1549,7 +1549,7 @@ func TestHTTPServer_HandshakeTimeout(t *testing.T) {
 
 	// Fire up an agent with TLS enabled.
 	a := StartTestAgent(t, TestAgent{
-		UseTLS: true,
+		UseHTTPS: true,
 		HCL: `
 			key_file = "../test/client_certs/server.key"
 			cert_file = "../test/client_certs/server.crt"
@@ -1621,7 +1621,7 @@ func TestRPC_HTTPSMaxConnsPerClient(t *testing.T) {
 
 			// Fire up an agent with TLS enabled.
 			a := StartTestAgent(t, TestAgent{
-				UseTLS: tc.tlsEnabled,
+				UseHTTPS: tc.tlsEnabled,
 				HCL: hclPrefix + `
 					limits {
 						http_max_conns_per_client = 2

agent/metadata/server.go

@@ -23,26 +23,27 @@ func (k *Key) Equal(x *Key) bool {
 
 // Server is used to return details of a consul server
 type Server struct {
-	Name             string // <node>.<dc>
+	Name                string // <node>.<dc>
-	ShortName        string // <node>
+	ShortName           string // <node>
-	ID               string
+	ID                  string
-	Datacenter       string
+	Datacenter          string
-	Segment          string
+	Segment             string
-	Port             int
+	Port                int
-	SegmentAddrs     map[string]string
+	SegmentAddrs        map[string]string
-	SegmentPorts     map[string]int
+	SegmentPorts        map[string]int
-	WanJoinPort      int
+	WanJoinPort         int
-	LanJoinPort      int
+	LanJoinPort         int
-	ExternalGRPCPort int
+	ExternalGRPCPort    int
-	Bootstrap        bool
+	ExternalGRPCTLSPort int
-	Expect           int
+	Bootstrap           bool
-	Build            version.Version
+	Expect              int
-	Version          int
+	Build               version.Version
-	RaftVersion      int
+	Version             int
-	Addr             net.Addr
+	RaftVersion         int
-	Status           serf.MemberStatus
+	Addr                net.Addr
-	ReadReplica      bool
+	Status              serf.MemberStatus
-	FeatureFlags     map[string]int
+	ReadReplica         bool
+	FeatureFlags        map[string]int
 
 	// If true, use TLS when connecting to this server
 	UseTLS bool
@@ -137,14 +138,18 @@ func IsConsulServer(m serf.Member) (bool, *Server) {
 		}
 	}
 
-	externalGRPCPort := 0
+	var externalGRPCPort, externalGRPCTLSPort int
-	externalGRPCPortStr, ok := m.Tags["grpc_port"]
+	externalGRPCPortStr, foundGRPC := m.Tags["grpc_port"]
-	if ok {
+	externalGRPCTLSPortStr, foundGRPCTLS := m.Tags["grpc_tls_port"]
-		externalGRPCPort, err = strconv.Atoi(externalGRPCPortStr)
+	if foundGRPC {
-		if err != nil {
+		externalGRPCPort, _ = strconv.Atoi(externalGRPCPortStr)
-			return false, nil
+	}
-		}
+	if foundGRPCTLS {
-		if externalGRPCPort < 1 {
+		externalGRPCTLSPort, _ = strconv.Atoi(externalGRPCTLSPortStr)
+	}
+	// If either port tag was found, check to ensure that at least one port was valid.
+	if foundGRPC || foundGRPCTLS {
+		if externalGRPCPort < 1 && externalGRPCTLSPort < 1 {
 			return false, nil
 		}
 	}
@@ -173,25 +178,26 @@ func IsConsulServer(m serf.Member) (bool, *Server) {
 	addr := &net.TCPAddr{IP: m.Addr, Port: port}
 
 	parts := &Server{
-		Name:             m.Name,
+		Name:                m.Name,
-		ShortName:        strings.TrimSuffix(m.Name, "."+datacenter),
+		ShortName:           strings.TrimSuffix(m.Name, "."+datacenter),
-		ID:               m.Tags["id"],
+		ID:                  m.Tags["id"],
-		Datacenter:       datacenter,
+		Datacenter:          datacenter,
-		Segment:          segment,
+		Segment:             segment,
-		Port:             port,
+		Port:                port,
-		SegmentAddrs:     segmentAddrs,
+		SegmentAddrs:        segmentAddrs,
-		SegmentPorts:     segmentPorts,
+		SegmentPorts:        segmentPorts,
-		WanJoinPort:      wanJoinPort,
+		WanJoinPort:         wanJoinPort,
-		LanJoinPort:      int(m.Port),
+		LanJoinPort:         int(m.Port),
-		ExternalGRPCPort: externalGRPCPort,
+		ExternalGRPCPort:    externalGRPCPort,
-		Bootstrap:        bootstrap,
+		ExternalGRPCTLSPort: externalGRPCTLSPort,
-		Expect:           expect,
+		Bootstrap:           bootstrap,
-		Addr:             addr,
+		Expect:              expect,
-		Build:            *buildVersion,
+		Addr:                addr,
-		Version:          vsn,
+		Build:               *buildVersion,
-		RaftVersion:      raftVsn,
+		Version:             vsn,
-		Status:           m.Status,
+		RaftVersion:         raftVsn,
-		UseTLS:           useTLS,
+		Status:              m.Status,
+		UseTLS:              useTLS,
 		// DEPRECATED - remove nonVoter check once support for that tag is removed
 		ReadReplica:  nonVoter || readReplica,
 		FeatureFlags: featureFlags,

agent/metadata/server_test.go

@@ -73,6 +73,7 @@ func TestIsConsulServer(t *testing.T) {
 				"build":         "0.8.0",
 				"wan_join_port": "1234",
 				"grpc_port":     "9876",
+				"grpc_tls_port": "9877",
 				"vsn":           "1",
 				"expect":        "3",
 				"raft_vsn":      "3",
@@ -82,19 +83,20 @@ func TestIsConsulServer(t *testing.T) {
 		}
 
 		expected := &metadata.Server{
-			Name:             "foo",
+			Name:                "foo",
-			ShortName:        "foo",
+			ShortName:           "foo",
-			ID:               "asdf",
+			ID:                  "asdf",
-			Datacenter:       "east-aws",
+			Datacenter:          "east-aws",
-			Segment:          "",
+			Segment:             "",
-			Port:             10000,
+			Port:                10000,
-			SegmentAddrs:     map[string]string{},
+			SegmentAddrs:        map[string]string{},
-			SegmentPorts:     map[string]int{},
+			SegmentPorts:        map[string]int{},
-			WanJoinPort:      1234,
+			WanJoinPort:         1234,
-			LanJoinPort:      5454,
+			LanJoinPort:         5454,
-			ExternalGRPCPort: 9876,
+			ExternalGRPCPort:    9876,
-			Bootstrap:        false,
+			ExternalGRPCTLSPort: 9877,
-			Expect:           3,
+			Bootstrap:           false,
+			Expect:              3,
 			Addr: &net.TCPAddr{
 				IP:   net.IP([]byte{127, 0, 0, 1}),
 				Port: 10000,
@@ -137,15 +139,41 @@ func TestIsConsulServer(t *testing.T) {
 		case "feature-namespaces":
 			m.Tags["ft_ns"] = "1"
 			expected.FeatureFlags = map[string]int{"ns": 1}
-			//
-		case "bad-grpc-port":
-			m.Tags["grpc_port"] = "three"
-		case "negative-grpc-port":
-			m.Tags["grpc_port"] = "-1"
-		case "zero-grpc-port":
-			m.Tags["grpc_port"] = "0"
 		case "no-role":
 			delete(m.Tags, "role")
+			//
+		case "missing-grpc-port":
+			delete(m.Tags, "grpc_port")
+			expected.ExternalGRPCPort = 0
+		case "missing-grpc-tls-port":
+			delete(m.Tags, "grpc_tls_port")
+			expected.ExternalGRPCTLSPort = 0
+		case "missing-both-grpc-ports":
+			delete(m.Tags, "grpc_port")
+			delete(m.Tags, "grpc_tls_port")
+			expected.ExternalGRPCPort = 0
+			expected.ExternalGRPCTLSPort = 0
+		case "bad-both-grpc-ports":
+			m.Tags["grpc_port"] = ""
+			m.Tags["grpc_tls_port"] = ""
+		case "bad-grpc-port":
+			m.Tags["grpc_port"] = "three"
+			m.Tags["grpc_tls_port"] = ""
+		case "bad-grpc-tls-port":
+			m.Tags["grpc_port"] = ""
+			m.Tags["grpc_tls_port"] = "three"
+		case "negative-grpc-port":
+			m.Tags["grpc_port"] = "-1"
+			m.Tags["grpc_tls_port"] = ""
+		case "negative-grpc-tls-port":
+			m.Tags["grpc_port"] = ""
+			m.Tags["grpc_tls_port"] = "-1"
+		case "zero-grpc-port":
+			m.Tags["grpc_port"] = "0"
+			m.Tags["grpc_tls_port"] = ""
+		case "zero-grpc-tls-port":
+			m.Tags["grpc_port"] = ""
+			m.Tags["grpc_tls_port"] = "0"
 		default:
 			t.Fatalf("unhandled variant: %s", variant)
 		}
@@ -174,11 +202,18 @@ func TestIsConsulServer(t *testing.T) {
 		"bootstrapped":       true,
 		"optionals":          true,
 		"feature-namespaces": true,
-		//
 		"no-role":            false,
-		"bad-grpc-port":      false,
+		//
-		"negative-grpc-port": false,
+		"missing-grpc-port":       true,
-		"zero-grpc-port":     false,
+		"missing-grpc-tls-port":   true,
+		"missing-both-grpc-ports": true,
+		"bad-both-grpc-ports":     false,
+		"bad-grpc-port":           false,
+		"negative-grpc-port":      false,
+		"zero-grpc-port":          false,
+		"bad-grpc-tls-port":       false,
+		"negative-grpc-tls-port":  false,
+		"zero-grpc-tls-port":      false,
 	}
 
 	for variant, expectOK := range cases {

agent/testagent.go

@@ -66,9 +66,13 @@ type TestAgent struct {
 	// and the directory will be removed once the test ends.
 	DataDir string
 
-	// UseTLS, if true, will disable the HTTP port and enable the HTTPS
+	// UseHTTPS, if true, will disable the HTTP port and enable the HTTPS
 	// one.
-	UseTLS bool
+	UseHTTPS bool
+
+	// UseGRPCTLS, if true, will disable the GRPC port and enable the GRPC+TLS
+	// one.
+	UseGRPCTLS bool
 
 	// dns is a reference to the first started DNS endpoint.
 	// It is valid after Start().
@@ -183,7 +187,7 @@ func (a *TestAgent) Start(t *testing.T) error {
 		Name:       name,
 	})
 
-	portsConfig := randomPortsSource(t, a.UseTLS)
+	portsConfig := randomPortsSource(t, a.UseHTTPS, a.UseGRPCTLS)
 
 	// Create NodeID outside the closure, so that it does not change
 	testHCLConfig := TestConfigHCL(NodeID())
@@ -401,11 +405,11 @@ func (a *TestAgent) consulConfig() *consul.Config {
 // chance of port conflicts for concurrently executed test binaries.
 // Instead of relying on one set of ports to be sufficient we retry
 // starting the agent with different ports on port conflict.
-func randomPortsSource(t *testing.T, tls bool) string {
+func randomPortsSource(t *testing.T, useHTTPS bool, useGRPCTLS bool) string {
-	ports := freeport.GetN(t, 7)
+	ports := freeport.GetN(t, 8)
 
 	var http, https int
-	if tls {
+	if useHTTPS {
 		http = -1
 		https = ports[2]
 	} else {
@@ -413,6 +417,15 @@ func randomPortsSource(t *testing.T, tls bool) string {
 		https = -1
 	}
 
+	var grpc, grpcTLS int
+	if useGRPCTLS {
+		grpc = -1
+		grpcTLS = ports[7]
+	} else {
+		grpc = ports[6]
+		grpcTLS = -1
+	}
+
 	return `
 		ports = {
 			dns = ` + strconv.Itoa(ports[0]) + `
@@ -421,7 +434,8 @@ func randomPortsSource(t *testing.T, tls bool) string {
 			serf_lan = ` + strconv.Itoa(ports[3]) + `
 			serf_wan = ` + strconv.Itoa(ports[4]) + `
 			server = ` + strconv.Itoa(ports[5]) + `
-			grpc = ` + strconv.Itoa(ports[6]) + `
+			grpc = ` + strconv.Itoa(grpc) + `
+			grpc_tls = ` + strconv.Itoa(grpcTLS) + `
 		}
 	`
 }

command/agent/agent.go

@@ -213,8 +213,8 @@ func (c *cmd) run(args []string) int {
 	}
 	ui.Info(fmt.Sprintf("       Datacenter: '%s' (Segment: '%s')", config.Datacenter, segment))
 	ui.Info(fmt.Sprintf("           Server: %v (Bootstrap: %v)", config.ServerMode, config.Bootstrap))
-	ui.Info(fmt.Sprintf("      Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, DNS: %d)", config.ClientAddrs,
+	ui.Info(fmt.Sprintf("      Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, gRPC-TLS: %d, DNS: %d)", config.ClientAddrs,
-		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.DNSPort))
+		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.GRPCTLSPort, config.DNSPort))
 	ui.Info(fmt.Sprintf("     Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
 		config.SerfPortLAN, config.SerfPortWAN))
 	ui.Info(fmt.Sprintf("Gossip Encryption: %t", config.EncryptKey != ""))

command/connect/envoy/envoy.go

@@ -639,7 +639,7 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) {
 
 	addr := c.grpcAddr
 	if addr == "" {
-		port, err := c.lookupXDSPort()
+		port, protocol, err := c.lookupXDSPort()
 		if err != nil {
 			c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
 		}
@@ -648,7 +648,7 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) {
 			// enabled.
 			port = 8502
 		}
-		addr = fmt.Sprintf("localhost:%v", port)
+		addr = fmt.Sprintf("%vlocalhost:%v", protocol, port)
 	}
 
 	// TODO: parse addr as a url instead of strings.HasPrefix/TrimPrefix
@@ -697,39 +697,48 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) {
 	return g, nil
 }
 
-func (c *cmd) lookupXDSPort() (int, error) {
+func (c *cmd) lookupXDSPort() (int, string, error) {
 	self, err := c.client.Agent().Self()
 	if err != nil {
-		return 0, err
+		return 0, "", err
 	}
 
 	type response struct {
 		XDS struct {
-			Port int
+			Ports struct {
+				Plaintext int
+				TLS       int
+			}
 		}
 	}
 
 	var resp response
-	if err := mapstructure.Decode(self, &resp); err == nil && resp.XDS.Port != 0 {
+	if err := mapstructure.Decode(self, &resp); err == nil {
-		return resp.XDS.Port, nil
+		if resp.XDS.Ports.TLS > 0 {
+			return resp.XDS.Ports.TLS, "https://", nil
+		}
+		if resp.XDS.Ports.Plaintext > 0 {
+			return resp.XDS.Ports.Plaintext, "http://", nil
+		}
 	}
 
 	// Fallback to old API for the case where a new consul CLI is being used with
 	// an older API version.
 	cfg, ok := self["DebugConfig"]
 	if !ok {
-		return 0, fmt.Errorf("unexpected agent response: no debug config")
+		return 0, "", fmt.Errorf("unexpected agent response: no debug config")
 	}
+	// TODO what does this mean? What did the old API look like? How does this affect compatibility?
 	port, ok := cfg["GRPCPort"]
 	if !ok {
-		return 0, fmt.Errorf("agent does not have grpc port enabled")
+		return 0, "", fmt.Errorf("agent does not have grpc port enabled")
 	}
 	portN, ok := port.(float64)
 	if !ok {
-		return 0, fmt.Errorf("invalid grpc port in agent response")
+		return 0, "", fmt.Errorf("invalid grpc port in agent response")
 	}
 
-	return int(portN), nil
+	return int(portN), "", nil
 }
 
 func (c *cmd) Synopsis() string {

command/connect/envoy/envoy_test.go

@@ -117,8 +117,8 @@ type generateConfigTestCase struct {
 	Files             map[string]string
 	ProxyConfig       map[string]interface{}
 	NamespacesEnabled bool
-	XDSPort           int  // only used for testing custom-configured grpc port
+	XDSPorts          agent.GRPCPorts // only used for testing custom-configured grpc port
-	AgentSelf110      bool // fake the agent API from versions v1.10 and earlier
+	AgentSelf110      bool            // fake the agent API from versions v1.10 and earlier
 	WantArgs          BootstrapTplArgs
 	WantErr           string
 }
@@ -447,9 +447,9 @@ func TestGenerateConfig(t *testing.T) {
 			},
 		},
 		{
-			Name:    "xds-addr-config",
+			Name:     "xds-addr-config",
-			Flags:   []string{"-proxy-id", "test-proxy"},
+			Flags:    []string{"-proxy-id", "test-proxy"},
-			XDSPort: 9999,
+			XDSPorts: agent.GRPCPorts{Plaintext: 9999, TLS: 0},
 			WantArgs: BootstrapTplArgs{
 				ProxyCluster: "test-proxy",
 				ProxyID:      "test-proxy",
@@ -470,10 +470,36 @@ func TestGenerateConfig(t *testing.T) {
 				PrometheusScrapePath:  "/metrics",
 			},
 		},
+		{
+			Name:         "grpc-tls-addr-config",
+			Flags:        []string{"-proxy-id", "test-proxy"},
+			XDSPorts:     agent.GRPCPorts{Plaintext: 9997, TLS: 9998},
+			AgentSelf110: false,
+			WantArgs: BootstrapTplArgs{
+				ProxyCluster: "test-proxy",
+				ProxyID:      "test-proxy",
+				// We don't know this til after the lookup so it will be empty in the
+				// initial args call we are testing here.
+				ProxySourceService: "",
+				// Should resolve IP, note this might not resolve the same way
+				// everywhere which might make this test brittle but not sure what else
+				// to do.
+				GRPC: GRPC{
+					AgentAddress: "127.0.0.1",
+					AgentPort:    "9998",
+					AgentTLS:     true,
+				},
+				AdminAccessLogPath:    "/dev/null",
+				AdminBindAddress:      "127.0.0.1",
+				AdminBindPort:         "19000",
+				LocalAgentClusterName: xds.LocalAgentClusterName,
+				PrometheusScrapePath:  "/metrics",
+			},
+		},
 		{
 			Name:         "deprecated-grpc-addr-config",
 			Flags:        []string{"-proxy-id", "test-proxy"},
-			XDSPort:      9999,
+			XDSPorts:     agent.GRPCPorts{Plaintext: 9999, TLS: 0},
 			AgentSelf110: true,
 			WantArgs: BootstrapTplArgs{
 				ProxyCluster: "test-proxy",
@@ -1138,7 +1164,7 @@ func testMockAgent(tc generateConfigTestCase) http.HandlerFunc {
 		case strings.Contains(r.URL.Path, "/agent/service"):
 			testMockAgentProxyConfig(tc.ProxyConfig, tc.NamespacesEnabled)(w, r)
 		case strings.Contains(r.URL.Path, "/agent/self"):
-			testMockAgentSelf(tc.XDSPort, tc.AgentSelf110)(w, r)
+			testMockAgentSelf(tc.XDSPorts, tc.AgentSelf110)(w, r)
 		case strings.Contains(r.URL.Path, "/catalog/node-services"):
 			testMockCatalogNodeServiceList()(w, r)
 		default:
@@ -1378,7 +1404,7 @@ func TestEnvoyCommand_canBindInternal(t *testing.T) {
 
 // testMockAgentSelf returns an empty /v1/agent/self response except GRPC
 // port is filled in to match the given wantXDSPort argument.
-func testMockAgentSelf(wantXDSPort int, agentSelf110 bool) http.HandlerFunc {
+func testMockAgentSelf(wantXDSPorts agent.GRPCPorts, agentSelf110 bool) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		resp := agent.Self{
 			Config: map[string]interface{}{
@@ -1388,10 +1414,17 @@ func testMockAgentSelf(wantXDSPort int, agentSelf110 bool) http.HandlerFunc {
 
 		if agentSelf110 {
 			resp.DebugConfig = map[string]interface{}{
-				"GRPCPort": wantXDSPort,
+				"GRPCPort": wantXDSPorts.Plaintext,
 			}
 		} else {
-			resp.XDS = &agent.XDSSelf{Port: wantXDSPort}
+			resp.XDS = &agent.XDSSelf{
+				// The deprecated Port field should default to TLS if it's available.
+				Port:  wantXDSPorts.TLS,
+				Ports: wantXDSPorts,
+			}
+			if wantXDSPorts.TLS <= 0 {
+				resp.XDS.Port = wantXDSPorts.Plaintext
+			}
 		}
 
 		selfJSON, err := json.Marshal(resp)

command/connect/envoy/testdata/grpc-tls-addr-config.golden

@@ -0,0 +1,223 @@
+{
+  "admin": {
+    "access_log_path": "/dev/null",
+    "address": {
+      "socket_address": {
+        "address": "127.0.0.1",
+        "port_value": 19000
+      }
+    }
+  },
+  "node": {
+    "cluster": "test",
+    "id": "test-proxy",
+    "metadata": {
+      "namespace": "default",
+      "partition": "default"
+    }
+  },
+  "layered_runtime": {
+    "layers": [
+      {
+        "name": "base",
+        "static_layer": {
+          "re2.max_program_size.error_level": 1048576
+        }
+      }
+    ]
+  },
+  "static_resources": {
+    "clusters": [
+      {
+        "name": "local_agent",
+        "ignore_health_on_host_removal": false,
+        "connect_timeout": "1s",
+        "type": "STATIC",
+        "transport_socket": {
+          "name": "tls",
+          "typed_config": {
+            "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+            "common_tls_context": {
+              "validation_context": {
+                "trusted_ca": {
+                  "inline_string": ""
+                }
+              }
+            }
+          }
+        },
+        "http2_protocol_options": {},
+        "loadAssignment": {
+          "clusterName": "local_agent",
+          "endpoints": [
+            {
+              "lbEndpoints": [
+                {
+                  "endpoint": {
+                    "address": {
+                      "socket_address": {
+                        "address": "127.0.0.1",
+                        "port_value": 9998
+                      }
+                    }
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "stats_config": {
+    "stats_tags": [
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.custom_hash"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.service_subset"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.service"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.namespace"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.partition"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.datacenter"
+      },
+      {
+        "regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.peer"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.routing_type"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
+        "tag_name": "consul.destination.trust_domain"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.destination.target"
+      },
+      {
+        "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
+        "tag_name": "consul.destination.full_target"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)",
+        "tag_name": "consul.upstream.service"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)",
+        "tag_name": "consul.upstream.datacenter"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)",
+        "tag_name": "consul.upstream.peer"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)",
+        "tag_name": "consul.upstream.namespace"
+      },
+      {
+        "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)",
+        "tag_name": "consul.upstream.partition"
+      },
+      {
+        "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.custom_hash"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.service_subset"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.service"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.namespace"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.datacenter"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.routing_type"
+      },
+      {
+        "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)",
+        "tag_name": "consul.trust_domain"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)",
+        "tag_name": "consul.target"
+      },
+      {
+        "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)",
+        "tag_name": "consul.full_target"
+      },
+      {
+        "tag_name": "local_cluster",
+        "fixed_value": "test"
+      },
+      {
+        "tag_name": "consul.source.service",
+        "fixed_value": "test"
+      },
+      {
+        "tag_name": "consul.source.namespace",
+        "fixed_value": "default"
+      },
+      {
+        "tag_name": "consul.source.partition",
+        "fixed_value": "default"
+      },
+      {
+        "tag_name": "consul.source.datacenter",
+        "fixed_value": "dc1"
+      }
+    ],
+    "use_all_default_tags": true
+  },
+  "dynamic_resources": {
+    "lds_config": {
+      "ads": {},
+      "resource_api_version": "V3"
+    },
+    "cds_config": {
+      "ads": {},
+      "resource_api_version": "V3"
+    },
+    "ads_config": {
+      "api_type": "DELTA_GRPC",
+      "transport_api_version": "V3",
+      "grpc_services": {
+        "initial_metadata": [
+          {
+            "key": "x-consul-token",
+            "value": ""
+          }
+        ],
+        "envoy_grpc": {
+          "cluster_name": "local_agent"
+        }
+      }
+    }
+  }
+}
+

website/content/docs/agent/config/config-files.mdx

@@ -608,6 +608,10 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'."
     automatically with this. This is set to `8502` by default when the agent runs
     in `-dev` mode. Currently gRPC is only used to expose Envoy xDS API to Envoy
     proxies.
+  - `grpc_tls` ((#grpc_tls_port)) - The gRPC API with TLS connections, -1 to disable. Default -1 (disabled).
+    **We recommend using `8502`** for `grpc_tls` by convention as some tooling will work
+    automatically with this. Usually only one of the `grpc_tls` or `grpc` ports should be defined.
+    Currently gRPC is only used to expose Envoy xDS API to Envoy proxies.
   - `serf_lan` ((#serf_lan_port)) - The Serf LAN port. Default 8301. TCP
     and UDP. Equivalent to the [`-serf-lan-port` command line flag](/docs/agent/config/cli-flags#_serf_lan_port).
   - `serf_wan` ((#serf_wan_port)) - The Serf WAN port. Default 8302.
@@ -2003,7 +2007,7 @@ specially crafted certificate signed by the CA can be used to gain full access t
       interface.
 
   - `grpc` ((#tls_grpc)) Provides settings for the gRPC/xDS interface. To enable
-    the gRPC interface you must define a port via [`ports.grpc`](#grpc_port).
+    the gRPC interface you must define a port via [`ports.grpc_tls`](#grpc_tls_port).
 
     - `ca_file` ((#tls_grpc_ca_file)) Overrides [`tls.defaults.ca_file`](#tls_defaults_ca_file).
 

website/content/docs/upgrading/upgrade-specific.mdx

@@ -14,6 +14,32 @@ provided for their upgrades as a result of new features or changed behavior.
 This page is used to document those details separately from the standard
 upgrade flow.
 
+## Consul 1.14.x
+
+### Service Mesh Compatibility
+
+##### Changes to gRPC TLS configuration
+
+**Configuration changes should be made** if using sidecar proxies or gateways
+in conjunction with any of the following:
+1. [`ports.https`](/docs/agent/config/config-files#https_port) - Encrypts gRPC in Consul 1.12 and prior
+2. [`auto_encrypt`](/docs/agent/config/config-files#auto_encrypt) - Encrypts gRPC in Consul 1.13 and prior
+3. [`auto_config`](/docs/agent/config/config-files#auto_config) - Encrypts gRPC in Consul 1.13 and prior
+
+Prior to Consul 1.14, it was possible for communication between Consul and Envoy over `ports.grpc`
+to be encrypted by one of these features.
+
+In Consul 1.14, a new [`ports.grpc_tls`](/docs/agent/config/config-files#grpc_tls_port) configuration
+is introduced. The existing [`ports.grpc`](/docs/agent/config/config-files#grpc_port) configuration
+**will stop supporting encryption in a future release**. Now, the recommended way to encrypt gRPC
+traffic is only via `ports.grpc_tls`.
+
+For most environments, the Envoy communication to Consul is loop-back only and does not benefit from encryption.
+
+If you would like to continue utilizing encryption for gRPC, change the existing `ports.grpc` to `ports.grpc_tls` in
+your configuration during the upgrade to ensure compatibility with future releases.
+
+
 ## Consul 1.13.x
 
 ### Service Mesh Compatibility