diff --git a/.changelog/14294.txt b/.changelog/14294.txt new file mode 100644 index 0000000000..7fcb497b1e --- /dev/null +++ b/.changelog/14294.txt @@ -0,0 +1,6 @@ +```release-note:breaking-change +config: Add new `ports.grpc_tls` configuration option. +Introduce a new port to better separate TLS config from the existing `ports.grpc` config. +The new `ports.grpc_tls` only supports TLS encrypted communication. +The existing `ports.grpc` currently supports both plain-text and tls communication, but tls support will be removed in a future release. +``` diff --git a/agent/agent.go b/agent/agent.go index 1bcb6c1480..43e1a6ea8f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -213,7 +213,7 @@ type Agent struct { // depending on the configuration delegate delegate - // externalGRPCServer is the gRPC server exposed on the dedicated gRPC port (as + // externalGRPCServer is the gRPC server exposed on dedicated gRPC ports (as // opposed to the multiplexed "server" port). externalGRPCServer *grpc.Server @@ -384,18 +384,18 @@ type Agent struct { // New process the desired options and creates a new Agent. // This process will -// * parse the config given the config Flags -// * setup logging -// * using predefined logger given in an option -// OR -// * initialize a new logger from the configuration -// including setting up gRPC logging -// * initialize telemetry -// * create a TLS Configurator -// * build a shared connection pool -// * create the ServiceManager -// * setup the NodeID if one isn't provided in the configuration -// * create the AutoConfig object for future use in fully +// - parse the config given the config Flags +// - setup logging +// - using predefined logger given in an option +// OR +// - initialize a new logger from the configuration +// including setting up gRPC logging +// - initialize telemetry +// - create a TLS Configurator +// - build a shared connection pool +// - create the ServiceManager +// - setup the NodeID if one isn't provided in the configuration +// - create the AutoConfig object for future use in fully // resolving the configuration func New(bd BaseDeps) (*Agent, error) { a := Agent{ @@ -539,7 +539,7 @@ func (a *Agent) Start(ctx context.Context) error { // This needs to happen after the initial auto-config is loaded, because TLS // can only be configured on the gRPC server at the point of creation. - a.buildExternalGRPCServer() + a.externalGRPCServer = external.NewServer(a.logger.Named("grpc.external")) if err := a.startLicenseManager(ctx); err != nil { return err @@ -702,7 +702,7 @@ func (a *Agent) Start(ctx context.Context) error { a.apiServers.Start(srv) } - // Start gRPC server. + // Start grpc and grpc_tls servers. if err := a.listenAndServeGRPC(); err != nil { return err } @@ -760,15 +760,10 @@ func (a *Agent) Failed() <-chan struct{} { return a.apiServers.failed } -func (a *Agent) buildExternalGRPCServer() { - a.externalGRPCServer = external.NewServer(a.logger.Named("grpc.external"), a.tlsConfigurator) -} - func (a *Agent) listenAndServeGRPC() error { - if len(a.config.GRPCAddrs) < 1 { + if len(a.config.GRPCAddrs) < 1 && len(a.config.GRPCTLSAddrs) < 1 { return nil } - // TODO(agentless): rather than asserting the concrete type of delegate, we // should add a method to the Delegate interface to build a ConfigSource. var cfg xds.ProxyConfigSource = localproxycfg.NewConfigSource(a.proxyConfig) @@ -787,7 +782,6 @@ func (a *Agent) listenAndServeGRPC() error { }() cfg = catalogCfg } - a.xdsServer = xds.NewServer( a.config.NodeName, a.logger.Named(logging.Envoy), @@ -800,22 +794,61 @@ func (a *Agent) listenAndServeGRPC() error { ) a.xdsServer.Register(a.externalGRPCServer) - ln, err := a.startListeners(a.config.GRPCAddrs) - if err != nil { - return err + // Attempt to spawn listeners + var listeners []net.Listener + start := func(port_name string, addrs []net.Addr, tlsConf *tls.Config) error { + if len(addrs) < 1 { + return nil + } + + ln, err := a.startListeners(addrs) + if err != nil { + return err + } + for i := range ln { + // Wrap with TLS, if provided. + if tlsConf != nil { + ln[i] = tls.NewListener(ln[i], tlsConf) + } + listeners = append(listeners, ln[i]) + } + + for _, l := range ln { + go func(innerL net.Listener) { + a.logger.Info("Started gRPC listeners", + "port_name", port_name, + "address", innerL.Addr().String(), + "network", innerL.Addr().Network(), + ) + err := a.externalGRPCServer.Serve(innerL) + if err != nil { + a.logger.Error("gRPC server failed", "port_name", port_name, "error", err) + } + }(l) + } + return nil } - for _, l := range ln { - go func(innerL net.Listener) { - a.logger.Info("Started gRPC server", - "address", innerL.Addr().String(), - "network", innerL.Addr().Network(), - ) - err := a.externalGRPCServer.Serve(innerL) - if err != nil { - a.logger.Error("gRPC server failed", "error", err) - } - }(l) + // The original grpc port may spawn in either plain-text or TLS mode (for backwards compatibility). + // TODO: Simplify this block to only spawn plain-text after 1.14 when deprecated TLS support is removed. + if a.config.GRPCPort > 0 { + // Only allow the grpc port to spawn TLS connections if the other grpc_tls port is NOT defined. + var tlsConf *tls.Config = nil + if a.config.GRPCTLSPort <= 0 && a.tlsConfigurator.GRPCServerUseTLS() { + a.logger.Warn("deprecated gRPC TLS configuration detected. Consider using `ports.grpc_tls` instead") + tlsConf = a.tlsConfigurator.IncomingGRPCConfig() + } + if err := start("grpc", a.config.GRPCAddrs, tlsConf); err != nil { + closeListeners(listeners) + return err + } + } + // Only allow grpc_tls to spawn with a TLS listener. + if a.config.GRPCTLSPort > 0 { + if err := start("grpc_tls", a.config.GRPCTLSAddrs, a.tlsConfigurator.IncomingGRPCConfig()); err != nil { + closeListeners(listeners) + return err + } } return nil } @@ -1203,6 +1236,7 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co cfg.RPCAdvertise = runtimeCfg.RPCAdvertiseAddr cfg.GRPCPort = runtimeCfg.GRPCPort + cfg.GRPCTLSPort = runtimeCfg.GRPCTLSPort cfg.Segment = runtimeCfg.SegmentName if len(runtimeCfg.Segments) > 0 { @@ -1506,7 +1540,9 @@ func (a *Agent) ShutdownAgent() error { } // Stop gRPC - a.externalGRPCServer.Stop() + if a.externalGRPCServer != nil { + a.externalGRPCServer.Stop() + } // Stop the proxy config manager if a.proxyConfig != nil { diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index e758d4bbf3..08324c2c43 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -45,7 +45,19 @@ type Self struct { type XDSSelf struct { SupportedProxies map[string][]string - Port int + // Port could be used for either TLS or plain-text communication + // up through version 1.14. In order to maintain backwards-compatibility, + // Port will now default to TLS and fallback to the standard port value. + // DEPRECATED: Use Ports field instead + Port int + Ports GRPCPorts +} + +// GRPCPorts is used to hold the external GRPC server's port numbers. +type GRPCPorts struct { + // Technically, this port is not always plain-text as of 1.14, but will be in a future release. + Plaintext int + TLS int } func (s *HTTPHandlers) AgentSelf(resp http.ResponseWriter, req *http.Request) (interface{}, error) { @@ -78,7 +90,16 @@ func (s *HTTPHandlers) AgentSelf(resp http.ResponseWriter, req *http.Request) (i SupportedProxies: map[string][]string{ "envoy": proxysupport.EnvoyVersions, }, - Port: s.agent.config.GRPCPort, + // Prefer the TLS port. See comment on the XDSSelf struct for details. + Port: s.agent.config.GRPCTLSPort, + Ports: GRPCPorts{ + Plaintext: s.agent.config.GRPCPort, + TLS: s.agent.config.GRPCTLSPort, + }, + } + // Fallback to standard port if TLS is not enabled. + if s.agent.config.GRPCTLSPort <= 0 { + xds.Port = s.agent.config.GRPCPort } } diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index d380d0d939..4f8f188821 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -1434,15 +1434,8 @@ func TestAgent_Self(t *testing.T) { cases := map[string]struct { hcl string expectXDS bool + grpcTLS bool }{ - "normal": { - hcl: ` - node_meta { - somekey = "somevalue" - } - `, - expectXDS: true, - }, "no grpc": { hcl: ` node_meta { @@ -1453,13 +1446,35 @@ func TestAgent_Self(t *testing.T) { } `, expectXDS: false, + grpcTLS: false, + }, + "plaintext grpc": { + hcl: ` + node_meta { + somekey = "somevalue" + } + `, + expectXDS: true, + grpcTLS: false, + }, + "tls grpc": { + hcl: ` + node_meta { + somekey = "somevalue" + } + `, + expectXDS: true, + grpcTLS: true, }, } for name, tc := range cases { tc := tc t.Run(name, func(t *testing.T) { - a := NewTestAgent(t, tc.hcl) + a := StartTestAgent(t, TestAgent{ + HCL: tc.hcl, + UseGRPCTLS: tc.grpcTLS, + }) defer a.Shutdown() testrpc.WaitForTestAgent(t, a.RPC, "dc1") @@ -1487,6 +1502,13 @@ func TestAgent_Self(t *testing.T) { map[string][]string{"envoy": proxysupport.EnvoyVersions}, val.XDS.SupportedProxies, ) + require.Equal(t, a.Config.GRPCTLSPort, val.XDS.Ports.TLS) + require.Equal(t, a.Config.GRPCPort, val.XDS.Ports.Plaintext) + if tc.grpcTLS { + require.Equal(t, a.Config.GRPCTLSPort, val.XDS.Port) + } else { + require.Equal(t, a.Config.GRPCPort, val.XDS.Port) + } } else { require.Nil(t, val.XDS, "xds component should be missing when gRPC is disabled") diff --git a/agent/agent_test.go b/agent/agent_test.go index b0da44d6e7..efd983e788 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2095,7 +2095,7 @@ func TestAgent_HTTPCheck_EnableAgentTLSForChecks(t *testing.T) { run := func(t *testing.T, ca string) { a := StartTestAgent(t, TestAgent{ - UseTLS: true, + UseHTTPS: true, HCL: ` enable_agent_tls_for_checks = true @@ -3873,7 +3873,7 @@ func TestAgent_reloadWatchesHTTPS(t *testing.T) { } t.Parallel() - a := TestAgent{UseTLS: true} + a := TestAgent{UseHTTPS: true} if err := a.Start(t); err != nil { t.Fatal(err) } @@ -5220,7 +5220,7 @@ func TestAgent_AutoEncrypt(t *testing.T) { server = ` + strconv.Itoa(srv.Config.RPCBindAddr.Port) + ` } retry_join = ["` + srv.Config.SerfBindAddrLAN.String() + `"]`, - UseTLS: true, + UseHTTPS: true, }) defer client.Shutdown() diff --git a/agent/config/builder.go b/agent/config/builder.go index 960d86ea43..721fd05ea7 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -125,10 +125,10 @@ type LoadResult struct { // // The sources are merged in the following order: // -// * default configuration -// * config files in alphabetical order -// * command line arguments -// * overrides +// - default configuration +// - config files in alphabetical order +// - command line arguments +// - overrides // // The config sources are merged sequentially and later values overwrite // previously set values. Slice values are merged by concatenating the two slices. @@ -433,6 +433,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { httpsPort := b.portVal("ports.https", c.Ports.HTTPS) serverPort := b.portVal("ports.server", c.Ports.Server) grpcPort := b.portVal("ports.grpc", c.Ports.GRPC) + grpcTlsPort := b.portVal("ports.grpc_tls", c.Ports.GRPCTLS) serfPortLAN := b.portVal("ports.serf_lan", c.Ports.SerfLAN) serfPortWAN := b.portVal("ports.serf_wan", c.Ports.SerfWAN) proxyMinPort := b.portVal("ports.proxy_min_port", c.Ports.ProxyMinPort) @@ -563,6 +564,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { httpAddrs := b.makeAddrs(b.expandAddrs("addresses.http", c.Addresses.HTTP), clientAddrs, httpPort) httpsAddrs := b.makeAddrs(b.expandAddrs("addresses.https", c.Addresses.HTTPS), clientAddrs, httpsPort) grpcAddrs := b.makeAddrs(b.expandAddrs("addresses.grpc", c.Addresses.GRPC), clientAddrs, grpcPort) + grpcTlsAddrs := b.makeAddrs(b.expandAddrs("addresses.grpc_tls", c.Addresses.GRPCTLS), clientAddrs, grpcTlsPort) for _, a := range dnsAddrs { if x, ok := a.(*net.TCPAddr); ok { @@ -987,8 +989,10 @@ func (b *builder) build() (rt RuntimeConfig, err error) { EnableRemoteScriptChecks: enableRemoteScriptChecks, EnableLocalScriptChecks: enableLocalScriptChecks, EncryptKey: stringVal(c.EncryptKey), - GRPCPort: grpcPort, GRPCAddrs: grpcAddrs, + GRPCPort: grpcPort, + GRPCTLSAddrs: grpcTlsAddrs, + GRPCTLSPort: grpcTlsPort, HTTPMaxConnsPerClient: intVal(c.Limits.HTTPMaxConnsPerClient), HTTPSHandshakeTimeout: b.durationVal("limits.https_handshake_timeout", c.Limits.HTTPSHandshakeTimeout), KVMaxValueSize: uint64Val(c.Limits.KVMaxValueSize), diff --git a/agent/config/config.go b/agent/config/config.go index 2d21e75dae..de82d98769 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -332,10 +332,11 @@ type Consul struct { } type Addresses struct { - DNS *string `mapstructure:"dns"` - HTTP *string `mapstructure:"http"` - HTTPS *string `mapstructure:"https"` - GRPC *string `mapstructure:"grpc"` + DNS *string `mapstructure:"dns"` + HTTP *string `mapstructure:"http"` + HTTPS *string `mapstructure:"https"` + GRPC *string `mapstructure:"grpc"` + GRPCTLS *string `mapstructure:"grpc_tls"` } type AdvertiseAddrsConfig struct { @@ -694,6 +695,7 @@ type Ports struct { SerfWAN *int `mapstructure:"serf_wan"` Server *int `mapstructure:"server"` GRPC *int `mapstructure:"grpc"` + GRPCTLS *int `mapstructure:"grpc_tls"` ProxyMinPort *int `mapstructure:"proxy_min_port"` ProxyMaxPort *int `mapstructure:"proxy_max_port"` SidecarMinPort *int `mapstructure:"sidecar_min_port"` diff --git a/agent/config/flags.go b/agent/config/flags.go index b2e3c35ba6..44554bd5ea 100644 --- a/agent/config/flags.go +++ b/agent/config/flags.go @@ -53,7 +53,8 @@ func AddFlags(fs *flag.FlagSet, f *LoadOpts) { add(&f.FlagValues.EnableLocalScriptChecks, "enable-local-script-checks", "Enables health check scripts from configuration file.") add(&f.FlagValues.HTTPConfig.AllowWriteHTTPFrom, "allow-write-http-from", "Only allow write endpoint calls from given network. CIDR format, can be specified multiple times.") add(&f.FlagValues.EncryptKey, "encrypt", "Provides the gossip encryption key.") - add(&f.FlagValues.Ports.GRPC, "grpc-port", "Sets the gRPC API port to listen on (currently needed for Envoy xDS only).") + add(&f.FlagValues.Ports.GRPC, "grpc-port", "Sets the gRPC API port to listen on.") + add(&f.FlagValues.Ports.GRPCTLS, "grpc-tls-port", "Sets the gRPC-TLS API port to listen on.") add(&f.FlagValues.Ports.HTTP, "http-port", "Sets the HTTP API port to listen on.") add(&f.FlagValues.Ports.HTTPS, "https-port", "Sets the HTTPS API port to listen on.") add(&f.FlagValues.StartJoinAddrsLAN, "join", "Address of an agent to join at start time. Can be specified multiple times.") diff --git a/agent/config/runtime.go b/agent/config/runtime.go index e607efcf39..ee82ea477e 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -670,13 +670,18 @@ type RuntimeConfig struct { // flag: -encrypt string EncryptKey string - // GRPCPort is the port the gRPC server listens on. Currently this only - // exposes the xDS and ext_authz APIs for Envoy and it is disabled by default. + // GRPCPort is the port the gRPC server listens on. It is disabled by default. // // hcl: ports { grpc = int } // flags: -grpc-port int GRPCPort int + // GRPCTLSPort is the port the gRPC server listens on. It is disabled by default. + // + // hcl: ports { grpc_tls = int } + // flags: -grpc-tls-port int + GRPCTLSPort int + // GRPCAddrs contains the list of TCP addresses and UNIX sockets the gRPC // server will bind to. If the gRPC endpoint is disabled (ports.grpc <= 0) // the list is empty. @@ -692,6 +697,21 @@ type RuntimeConfig struct { // hcl: client_addr = string addresses { grpc = string } ports { grpc = int } GRPCAddrs []net.Addr + // GRPCTLSAddrs contains the list of TCP addresses and UNIX sockets the gRPC + // server will bind to. If the gRPC endpoint is disabled (ports.grpc <= 0) + // the list is empty. + // + // The addresses are taken from 'addresses.grpc_tls' which should contain a + // space separated list of ip addresses, UNIX socket paths and/or + // go-sockaddr templates. UNIX socket paths must be written as + // 'unix://', e.g. 'unix:///var/run/consul-grpc.sock'. + // + // If 'addresses.grpc_tls' was not provided the 'client_addr' addresses are + // used. + // + // hcl: client_addr = string addresses { grpc_tls = string } ports { grpc_tls = int } + GRPCTLSAddrs []net.Addr + // HTTPAddrs contains the list of TCP addresses and UNIX sockets the HTTP // server will bind to. If the HTTP endpoint is disabled (ports.http <= 0) // the list is empty. diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index f5e9bd3352..597899886b 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -6016,6 +6016,8 @@ func TestLoad_FullConfig(t *testing.T) { GRPCPort: 4881, GRPCAddrs: []net.Addr{tcpAddr("32.31.61.91:4881")}, + GRPCTLSPort: 5201, + GRPCTLSAddrs: []net.Addr{tcpAddr("23.14.88.19:5201")}, HTTPAddrs: []net.Addr{tcpAddr("83.39.91.39:7999")}, HTTPBlockEndpoints: []string{"RBvAFcGD", "fWOWFznh"}, AllowWriteHTTPFrom: []*net.IPNet{cidr("127.0.0.0/8"), cidr("22.33.44.55/32"), cidr("0.0.0.0/0")}, diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index 8f91743dba..a48d909712 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -192,6 +192,8 @@ "ExposeMinPort": 0, "GRPCAddrs": [], "GRPCPort": 0, + "GRPCTLSAddrs": [], + "GRPCTLSPort": 0, "GossipLANGossipInterval": "0s", "GossipLANGossipNodes": 0, "GossipLANProbeInterval": "0s", diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index 305df9b89e..09e9aabd58 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -44,6 +44,7 @@ addresses = { http = "83.39.91.39" https = "95.17.17.19" grpc = "32.31.61.91" + grpc_tls = "23.14.88.19" } advertise_addr = "17.99.29.16" advertise_addr_wan = "78.63.37.19" @@ -320,6 +321,7 @@ ports { https = 15127 server = 3757 grpc = 4881 + grpc_tls = 5201 proxy_min_port = 2000 proxy_max_port = 3000 sidecar_min_port = 8888 diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index bc72c2955e..946b27e73c 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -44,7 +44,8 @@ "dns": "93.95.95.81", "http": "83.39.91.39", "https": "95.17.17.19", - "grpc": "32.31.61.91" + "grpc": "32.31.61.91", + "grpc_tls": "23.14.88.19" }, "advertise_addr": "17.99.29.16", "advertise_addr_wan": "78.63.37.19", @@ -320,6 +321,7 @@ "https": 15127, "server": 3757, "grpc": 4881, + "grpc_tls": 5201, "sidecar_min_port": 8888, "sidecar_max_port": 9999, "expose_min_port": 1111, diff --git a/agent/consul/config.go b/agent/consul/config.go index 38063f808a..69d4fddee5 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -133,6 +133,9 @@ type Config struct { // GRPCPort is the port the public gRPC server listens on. GRPCPort int + // GRPCTLSPort is the port the public gRPC TLS server listens on. + GRPCTLSPort int + // (Enterprise-only) The network segment this agent is part of. Segment string diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 389b790569..29cd216c9f 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -1073,9 +1073,11 @@ func (s *Server) handleAliveMember(member serf.Member, nodeEntMeta *acl.Enterpri }, } - grpcPortStr := member.Tags["grpc_port"] - if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 { - service.Meta["grpc_port"] = grpcPortStr + if parts.ExternalGRPCPort > 0 { + service.Meta["grpc_port"] = strconv.Itoa(parts.ExternalGRPCPort) + } + if parts.ExternalGRPCTLSPort > 0 { + service.Meta["grpc_tls_port"] = strconv.Itoa(parts.ExternalGRPCTLSPort) } // Attempt to join the consul server diff --git a/agent/consul/leader_test.go b/agent/consul/leader_test.go index b041d2f925..1edaa88a3c 100644 --- a/agent/consul/leader_test.go +++ b/agent/consul/leader_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "testing" "time" @@ -19,6 +20,7 @@ import ( "github.com/hashicorp/consul/agent/structs" tokenStore "github.com/hashicorp/consul/agent/token" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/freeport" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul/testrpc" @@ -355,8 +357,10 @@ func TestLeader_CheckServersMeta(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") } - t.Parallel() + + ports := freeport.GetN(t, 2) // s3 grpc, s3 grpc_tls + dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.ACLsEnabled = true @@ -383,6 +387,8 @@ func TestLeader_CheckServersMeta(t *testing.T) { c.ACLInitialManagementToken = "root" c.ACLResolverSettings.ACLDefaultPolicy = "allow" c.Bootstrap = false + c.GRPCPort = ports[0] + c.GRPCTLSPort = ports[1] }) defer os.RemoveAll(dir3) defer s3.Shutdown() @@ -456,6 +462,14 @@ func TestLeader_CheckServersMeta(t *testing.T) { if newVersion != versionToExpect { r.Fatalf("Expected version to be updated to %s, was %s", versionToExpect, newVersion) } + grpcPort := service.Meta["grpc_port"] + if grpcPort != strconv.Itoa(ports[0]) { + r.Fatalf("Expected grpc port to be %d, was %s", ports[0], grpcPort) + } + grpcTLSPort := service.Meta["grpc_tls_port"] + if grpcTLSPort != strconv.Itoa(ports[1]) { + r.Fatalf("Expected grpc tls port to be %d, was %s", ports[1], grpcTLSPort) + } }) } diff --git a/agent/consul/peering_backend.go b/agent/consul/peering_backend.go index 0f8b009e9c..5b01b9d040 100644 --- a/agent/consul/peering_backend.go +++ b/agent/consul/peering_backend.go @@ -66,11 +66,19 @@ func (b *PeeringBackend) GetServerAddresses() ([]string, error) { } var addrs []string for _, node := range nodes { - grpcPortStr := node.ServiceMeta["grpc_port"] - if v, err := strconv.Atoi(grpcPortStr); err != nil || v < 1 { - continue // skip server that isn't exporting public gRPC properly + // Prefer the TLS port if it is defined. + grpcPortStr := node.ServiceMeta["grpc_tls_port"] + if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 { + addrs = append(addrs, node.Address+":"+grpcPortStr) + continue } - addrs = append(addrs, node.Address+":"+grpcPortStr) + // Fallback to the standard port if TLS is not defined. + grpcPortStr = node.ServiceMeta["grpc_port"] + if v, err := strconv.Atoi(grpcPortStr); err == nil && v > 0 { + addrs = append(addrs, node.Address+":"+grpcPortStr) + continue + } + // Skip node if neither defined. } if len(addrs) == 0 { return nil, fmt.Errorf("a grpc bind port must be specified in the configuration for all servers") diff --git a/agent/consul/server.go b/agent/consul/server.go index 94048d06f2..b6e77a4a17 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -253,7 +253,7 @@ type Server struct { // enable RPC forwarding. externalConnectCAServer *connectca.Server - // externalGRPCServer is the gRPC server exposed on the dedicated gRPC port, as + // externalGRPCServer has a gRPC server exposed on the dedicated gRPC ports, as // opposed to the multiplexed "server" port which is served by grpcHandler. externalGRPCServer *grpc.Server @@ -377,7 +377,6 @@ type Server struct { // embedded struct to hold all the enterprise specific data EnterpriseServer } - type connHandler interface { Run() error Handle(conn net.Conn) diff --git a/agent/consul/server_serf.go b/agent/consul/server_serf.go index 569bc977ff..80f44aedc2 100644 --- a/agent/consul/server_serf.go +++ b/agent/consul/server_serf.go @@ -107,6 +107,9 @@ func (s *Server) setupSerfConfig(opts setupSerfOptions) (*serf.Config, error) { if s.config.GRPCPort > 0 { conf.Tags["grpc_port"] = fmt.Sprintf("%d", s.config.GRPCPort) } + if s.config.GRPCTLSPort > 0 { + conf.Tags["grpc_tls_port"] = fmt.Sprintf("%d", s.config.GRPCTLSPort) + } if s.config.Bootstrap { conf.Tags["bootstrap"] = "1" } diff --git a/agent/consul/server_test.go b/agent/consul/server_test.go index 35bbe720e2..5a3a2dea16 100644 --- a/agent/consul/server_test.go +++ b/agent/consul/server_test.go @@ -1,6 +1,7 @@ package consul import ( + "crypto/tls" "crypto/x509" "fmt" "net" @@ -218,9 +219,10 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S var dir string var srv *Server + var config *Config + var deps Deps // Retry added to avoid cases where bind addr is already in use retry.RunWith(retry.ThreeTimes(), t, func(r *retry.R) { - var config *Config dir, config = testServerConfig(t) for _, fn := range configOpts { fn(config) @@ -234,7 +236,8 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S config.ACLResolverSettings.EnterpriseMeta = *config.AgentEnterpriseMeta() var err error - srv, err = newServer(t, config) + deps = newDefaultDeps(t, config) + srv, err = newServerWithDeps(t, config, deps) if err != nil { r.Fatalf("err: %v", err) } @@ -245,9 +248,14 @@ func testServerWithConfig(t *testing.T, configOpts ...func(*Config)) (string, *S // Normally the gRPC server listener is created at the agent level and // passed down into the Server creation. externalGRPCAddr := fmt.Sprintf("127.0.0.1:%d", srv.config.GRPCPort) - ln, err := net.Listen("tcp", externalGRPCAddr) require.NoError(t, err) + + // Wrap the listener with TLS + if deps.TLSConfigurator.GRPCServerUseTLS() { + ln = tls.NewListener(ln, deps.TLSConfigurator.IncomingGRPCConfig()) + } + go func() { _ = srv.externalGRPCServer.Serve(ln) }() @@ -300,8 +308,8 @@ func newServerWithDeps(t *testing.T, c *Config, deps Deps) (*Server, error) { oldNotify() } } - - srv, err := NewServer(c, deps, external.NewServer(deps.Logger.Named("grpc.external"), deps.TLSConfigurator)) + grpcServer := external.NewServer(deps.Logger.Named("grpc.external")) + srv, err := NewServer(c, deps, grpcServer) if err != nil { return nil, err } diff --git a/agent/grpc-external/server.go b/agent/grpc-external/server.go index 4ae8c6d652..7ab72d77ca 100644 --- a/agent/grpc-external/server.go +++ b/agent/grpc-external/server.go @@ -6,16 +6,14 @@ import ( middleware "github.com/grpc-ecosystem/go-grpc-middleware" recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" agentmiddleware "github.com/hashicorp/consul/agent/grpc-middleware" - "github.com/hashicorp/consul/tlsutil" ) // NewServer constructs a gRPC server for the external gRPC port, to which // handlers can be registered. -func NewServer(logger agentmiddleware.Logger, tls *tlsutil.Configurator) *grpc.Server { +func NewServer(logger agentmiddleware.Logger) *grpc.Server { recoveryOpts := agentmiddleware.PanicHandlerMiddlewareOpts(logger) opts := []grpc.ServerOption{ @@ -35,9 +33,5 @@ func NewServer(logger agentmiddleware.Logger, tls *tlsutil.Configurator) *grpc.S MinTime: 15 * time.Second, }), } - if tls != nil && tls.GRPCServerUseTLS() { - creds := credentials.NewTLS(tls.IncomingGRPCConfig()) - opts = append(opts, grpc.Creds(creds)) - } return grpc.NewServer(opts...) } diff --git a/agent/http_test.go b/agent/http_test.go index a42a932303..60b0bd97f7 100644 --- a/agent/http_test.go +++ b/agent/http_test.go @@ -148,7 +148,7 @@ func TestSetupHTTPServer_HTTP2(t *testing.T) { // Fire up an agent with TLS enabled. a := StartTestAgent(t, TestAgent{ - UseTLS: true, + UseHTTPS: true, HCL: ` key_file = "../test/client_certs/server.key" cert_file = "../test/client_certs/server.crt" @@ -1549,7 +1549,7 @@ func TestHTTPServer_HandshakeTimeout(t *testing.T) { // Fire up an agent with TLS enabled. a := StartTestAgent(t, TestAgent{ - UseTLS: true, + UseHTTPS: true, HCL: ` key_file = "../test/client_certs/server.key" cert_file = "../test/client_certs/server.crt" @@ -1621,7 +1621,7 @@ func TestRPC_HTTPSMaxConnsPerClient(t *testing.T) { // Fire up an agent with TLS enabled. a := StartTestAgent(t, TestAgent{ - UseTLS: tc.tlsEnabled, + UseHTTPS: tc.tlsEnabled, HCL: hclPrefix + ` limits { http_max_conns_per_client = 2 diff --git a/agent/metadata/server.go b/agent/metadata/server.go index 83997f7cd1..bb0c263364 100644 --- a/agent/metadata/server.go +++ b/agent/metadata/server.go @@ -23,26 +23,27 @@ func (k *Key) Equal(x *Key) bool { // Server is used to return details of a consul server type Server struct { - Name string // . - ShortName string // - ID string - Datacenter string - Segment string - Port int - SegmentAddrs map[string]string - SegmentPorts map[string]int - WanJoinPort int - LanJoinPort int - ExternalGRPCPort int - Bootstrap bool - Expect int - Build version.Version - Version int - RaftVersion int - Addr net.Addr - Status serf.MemberStatus - ReadReplica bool - FeatureFlags map[string]int + Name string // . + ShortName string // + ID string + Datacenter string + Segment string + Port int + SegmentAddrs map[string]string + SegmentPorts map[string]int + WanJoinPort int + LanJoinPort int + ExternalGRPCPort int + ExternalGRPCTLSPort int + Bootstrap bool + Expect int + Build version.Version + Version int + RaftVersion int + Addr net.Addr + Status serf.MemberStatus + ReadReplica bool + FeatureFlags map[string]int // If true, use TLS when connecting to this server UseTLS bool @@ -137,14 +138,18 @@ func IsConsulServer(m serf.Member) (bool, *Server) { } } - externalGRPCPort := 0 - externalGRPCPortStr, ok := m.Tags["grpc_port"] - if ok { - externalGRPCPort, err = strconv.Atoi(externalGRPCPortStr) - if err != nil { - return false, nil - } - if externalGRPCPort < 1 { + var externalGRPCPort, externalGRPCTLSPort int + externalGRPCPortStr, foundGRPC := m.Tags["grpc_port"] + externalGRPCTLSPortStr, foundGRPCTLS := m.Tags["grpc_tls_port"] + if foundGRPC { + externalGRPCPort, _ = strconv.Atoi(externalGRPCPortStr) + } + if foundGRPCTLS { + externalGRPCTLSPort, _ = strconv.Atoi(externalGRPCTLSPortStr) + } + // If either port tag was found, check to ensure that at least one port was valid. + if foundGRPC || foundGRPCTLS { + if externalGRPCPort < 1 && externalGRPCTLSPort < 1 { return false, nil } } @@ -173,25 +178,26 @@ func IsConsulServer(m serf.Member) (bool, *Server) { addr := &net.TCPAddr{IP: m.Addr, Port: port} parts := &Server{ - Name: m.Name, - ShortName: strings.TrimSuffix(m.Name, "."+datacenter), - ID: m.Tags["id"], - Datacenter: datacenter, - Segment: segment, - Port: port, - SegmentAddrs: segmentAddrs, - SegmentPorts: segmentPorts, - WanJoinPort: wanJoinPort, - LanJoinPort: int(m.Port), - ExternalGRPCPort: externalGRPCPort, - Bootstrap: bootstrap, - Expect: expect, - Addr: addr, - Build: *buildVersion, - Version: vsn, - RaftVersion: raftVsn, - Status: m.Status, - UseTLS: useTLS, + Name: m.Name, + ShortName: strings.TrimSuffix(m.Name, "."+datacenter), + ID: m.Tags["id"], + Datacenter: datacenter, + Segment: segment, + Port: port, + SegmentAddrs: segmentAddrs, + SegmentPorts: segmentPorts, + WanJoinPort: wanJoinPort, + LanJoinPort: int(m.Port), + ExternalGRPCPort: externalGRPCPort, + ExternalGRPCTLSPort: externalGRPCTLSPort, + Bootstrap: bootstrap, + Expect: expect, + Addr: addr, + Build: *buildVersion, + Version: vsn, + RaftVersion: raftVsn, + Status: m.Status, + UseTLS: useTLS, // DEPRECATED - remove nonVoter check once support for that tag is removed ReadReplica: nonVoter || readReplica, FeatureFlags: featureFlags, diff --git a/agent/metadata/server_test.go b/agent/metadata/server_test.go index 2f56bd7fd4..6913021224 100644 --- a/agent/metadata/server_test.go +++ b/agent/metadata/server_test.go @@ -73,6 +73,7 @@ func TestIsConsulServer(t *testing.T) { "build": "0.8.0", "wan_join_port": "1234", "grpc_port": "9876", + "grpc_tls_port": "9877", "vsn": "1", "expect": "3", "raft_vsn": "3", @@ -82,19 +83,20 @@ func TestIsConsulServer(t *testing.T) { } expected := &metadata.Server{ - Name: "foo", - ShortName: "foo", - ID: "asdf", - Datacenter: "east-aws", - Segment: "", - Port: 10000, - SegmentAddrs: map[string]string{}, - SegmentPorts: map[string]int{}, - WanJoinPort: 1234, - LanJoinPort: 5454, - ExternalGRPCPort: 9876, - Bootstrap: false, - Expect: 3, + Name: "foo", + ShortName: "foo", + ID: "asdf", + Datacenter: "east-aws", + Segment: "", + Port: 10000, + SegmentAddrs: map[string]string{}, + SegmentPorts: map[string]int{}, + WanJoinPort: 1234, + LanJoinPort: 5454, + ExternalGRPCPort: 9876, + ExternalGRPCTLSPort: 9877, + Bootstrap: false, + Expect: 3, Addr: &net.TCPAddr{ IP: net.IP([]byte{127, 0, 0, 1}), Port: 10000, @@ -137,15 +139,41 @@ func TestIsConsulServer(t *testing.T) { case "feature-namespaces": m.Tags["ft_ns"] = "1" expected.FeatureFlags = map[string]int{"ns": 1} - // - case "bad-grpc-port": - m.Tags["grpc_port"] = "three" - case "negative-grpc-port": - m.Tags["grpc_port"] = "-1" - case "zero-grpc-port": - m.Tags["grpc_port"] = "0" case "no-role": delete(m.Tags, "role") + // + case "missing-grpc-port": + delete(m.Tags, "grpc_port") + expected.ExternalGRPCPort = 0 + case "missing-grpc-tls-port": + delete(m.Tags, "grpc_tls_port") + expected.ExternalGRPCTLSPort = 0 + case "missing-both-grpc-ports": + delete(m.Tags, "grpc_port") + delete(m.Tags, "grpc_tls_port") + expected.ExternalGRPCPort = 0 + expected.ExternalGRPCTLSPort = 0 + case "bad-both-grpc-ports": + m.Tags["grpc_port"] = "" + m.Tags["grpc_tls_port"] = "" + case "bad-grpc-port": + m.Tags["grpc_port"] = "three" + m.Tags["grpc_tls_port"] = "" + case "bad-grpc-tls-port": + m.Tags["grpc_port"] = "" + m.Tags["grpc_tls_port"] = "three" + case "negative-grpc-port": + m.Tags["grpc_port"] = "-1" + m.Tags["grpc_tls_port"] = "" + case "negative-grpc-tls-port": + m.Tags["grpc_port"] = "" + m.Tags["grpc_tls_port"] = "-1" + case "zero-grpc-port": + m.Tags["grpc_port"] = "0" + m.Tags["grpc_tls_port"] = "" + case "zero-grpc-tls-port": + m.Tags["grpc_port"] = "" + m.Tags["grpc_tls_port"] = "0" default: t.Fatalf("unhandled variant: %s", variant) } @@ -174,11 +202,18 @@ func TestIsConsulServer(t *testing.T) { "bootstrapped": true, "optionals": true, "feature-namespaces": true, - // "no-role": false, - "bad-grpc-port": false, - "negative-grpc-port": false, - "zero-grpc-port": false, + // + "missing-grpc-port": true, + "missing-grpc-tls-port": true, + "missing-both-grpc-ports": true, + "bad-both-grpc-ports": false, + "bad-grpc-port": false, + "negative-grpc-port": false, + "zero-grpc-port": false, + "bad-grpc-tls-port": false, + "negative-grpc-tls-port": false, + "zero-grpc-tls-port": false, } for variant, expectOK := range cases { diff --git a/agent/testagent.go b/agent/testagent.go index 5701834b7c..ea5afff81d 100644 --- a/agent/testagent.go +++ b/agent/testagent.go @@ -66,9 +66,13 @@ type TestAgent struct { // and the directory will be removed once the test ends. DataDir string - // UseTLS, if true, will disable the HTTP port and enable the HTTPS + // UseHTTPS, if true, will disable the HTTP port and enable the HTTPS // one. - UseTLS bool + UseHTTPS bool + + // UseGRPCTLS, if true, will disable the GRPC port and enable the GRPC+TLS + // one. + UseGRPCTLS bool // dns is a reference to the first started DNS endpoint. // It is valid after Start(). @@ -183,7 +187,7 @@ func (a *TestAgent) Start(t *testing.T) error { Name: name, }) - portsConfig := randomPortsSource(t, a.UseTLS) + portsConfig := randomPortsSource(t, a.UseHTTPS, a.UseGRPCTLS) // Create NodeID outside the closure, so that it does not change testHCLConfig := TestConfigHCL(NodeID()) @@ -401,11 +405,11 @@ func (a *TestAgent) consulConfig() *consul.Config { // chance of port conflicts for concurrently executed test binaries. // Instead of relying on one set of ports to be sufficient we retry // starting the agent with different ports on port conflict. -func randomPortsSource(t *testing.T, tls bool) string { - ports := freeport.GetN(t, 7) +func randomPortsSource(t *testing.T, useHTTPS bool, useGRPCTLS bool) string { + ports := freeport.GetN(t, 8) var http, https int - if tls { + if useHTTPS { http = -1 https = ports[2] } else { @@ -413,6 +417,15 @@ func randomPortsSource(t *testing.T, tls bool) string { https = -1 } + var grpc, grpcTLS int + if useGRPCTLS { + grpc = -1 + grpcTLS = ports[7] + } else { + grpc = ports[6] + grpcTLS = -1 + } + return ` ports = { dns = ` + strconv.Itoa(ports[0]) + ` @@ -421,7 +434,8 @@ func randomPortsSource(t *testing.T, tls bool) string { serf_lan = ` + strconv.Itoa(ports[3]) + ` serf_wan = ` + strconv.Itoa(ports[4]) + ` server = ` + strconv.Itoa(ports[5]) + ` - grpc = ` + strconv.Itoa(ports[6]) + ` + grpc = ` + strconv.Itoa(grpc) + ` + grpc_tls = ` + strconv.Itoa(grpcTLS) + ` } ` } diff --git a/command/agent/agent.go b/command/agent/agent.go index cc08213e1c..ae455297a6 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -213,8 +213,8 @@ func (c *cmd) run(args []string) int { } ui.Info(fmt.Sprintf(" Datacenter: '%s' (Segment: '%s')", config.Datacenter, segment)) ui.Info(fmt.Sprintf(" Server: %v (Bootstrap: %v)", config.ServerMode, config.Bootstrap)) - ui.Info(fmt.Sprintf(" Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, DNS: %d)", config.ClientAddrs, - config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.DNSPort)) + ui.Info(fmt.Sprintf(" Client Addr: %v (HTTP: %d, HTTPS: %d, gRPC: %d, gRPC-TLS: %d, DNS: %d)", config.ClientAddrs, + config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.GRPCTLSPort, config.DNSPort)) ui.Info(fmt.Sprintf(" Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN, config.SerfPortLAN, config.SerfPortWAN)) ui.Info(fmt.Sprintf("Gossip Encryption: %t", config.EncryptKey != "")) diff --git a/command/connect/envoy/envoy.go b/command/connect/envoy/envoy.go index 342357dfab..5567809fd8 100644 --- a/command/connect/envoy/envoy.go +++ b/command/connect/envoy/envoy.go @@ -639,7 +639,7 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) { addr := c.grpcAddr if addr == "" { - port, err := c.lookupXDSPort() + port, protocol, err := c.lookupXDSPort() if err != nil { c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err)) } @@ -648,7 +648,7 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) { // enabled. port = 8502 } - addr = fmt.Sprintf("localhost:%v", port) + addr = fmt.Sprintf("%vlocalhost:%v", protocol, port) } // TODO: parse addr as a url instead of strings.HasPrefix/TrimPrefix @@ -697,39 +697,48 @@ func (c *cmd) xdsAddress(httpCfg *api.Config) (GRPC, error) { return g, nil } -func (c *cmd) lookupXDSPort() (int, error) { +func (c *cmd) lookupXDSPort() (int, string, error) { self, err := c.client.Agent().Self() if err != nil { - return 0, err + return 0, "", err } type response struct { XDS struct { - Port int + Ports struct { + Plaintext int + TLS int + } } } var resp response - if err := mapstructure.Decode(self, &resp); err == nil && resp.XDS.Port != 0 { - return resp.XDS.Port, nil + if err := mapstructure.Decode(self, &resp); err == nil { + if resp.XDS.Ports.TLS > 0 { + return resp.XDS.Ports.TLS, "https://", nil + } + if resp.XDS.Ports.Plaintext > 0 { + return resp.XDS.Ports.Plaintext, "http://", nil + } } // Fallback to old API for the case where a new consul CLI is being used with // an older API version. cfg, ok := self["DebugConfig"] if !ok { - return 0, fmt.Errorf("unexpected agent response: no debug config") + return 0, "", fmt.Errorf("unexpected agent response: no debug config") } + // TODO what does this mean? What did the old API look like? How does this affect compatibility? port, ok := cfg["GRPCPort"] if !ok { - return 0, fmt.Errorf("agent does not have grpc port enabled") + return 0, "", fmt.Errorf("agent does not have grpc port enabled") } portN, ok := port.(float64) if !ok { - return 0, fmt.Errorf("invalid grpc port in agent response") + return 0, "", fmt.Errorf("invalid grpc port in agent response") } - return int(portN), nil + return int(portN), "", nil } func (c *cmd) Synopsis() string { diff --git a/command/connect/envoy/envoy_test.go b/command/connect/envoy/envoy_test.go index 4eedd16d42..5236ee6739 100644 --- a/command/connect/envoy/envoy_test.go +++ b/command/connect/envoy/envoy_test.go @@ -117,8 +117,8 @@ type generateConfigTestCase struct { Files map[string]string ProxyConfig map[string]interface{} NamespacesEnabled bool - XDSPort int // only used for testing custom-configured grpc port - AgentSelf110 bool // fake the agent API from versions v1.10 and earlier + XDSPorts agent.GRPCPorts // only used for testing custom-configured grpc port + AgentSelf110 bool // fake the agent API from versions v1.10 and earlier WantArgs BootstrapTplArgs WantErr string } @@ -447,9 +447,9 @@ func TestGenerateConfig(t *testing.T) { }, }, { - Name: "xds-addr-config", - Flags: []string{"-proxy-id", "test-proxy"}, - XDSPort: 9999, + Name: "xds-addr-config", + Flags: []string{"-proxy-id", "test-proxy"}, + XDSPorts: agent.GRPCPorts{Plaintext: 9999, TLS: 0}, WantArgs: BootstrapTplArgs{ ProxyCluster: "test-proxy", ProxyID: "test-proxy", @@ -470,10 +470,36 @@ func TestGenerateConfig(t *testing.T) { PrometheusScrapePath: "/metrics", }, }, + { + Name: "grpc-tls-addr-config", + Flags: []string{"-proxy-id", "test-proxy"}, + XDSPorts: agent.GRPCPorts{Plaintext: 9997, TLS: 9998}, + AgentSelf110: false, + WantArgs: BootstrapTplArgs{ + ProxyCluster: "test-proxy", + ProxyID: "test-proxy", + // We don't know this til after the lookup so it will be empty in the + // initial args call we are testing here. + ProxySourceService: "", + // Should resolve IP, note this might not resolve the same way + // everywhere which might make this test brittle but not sure what else + // to do. + GRPC: GRPC{ + AgentAddress: "127.0.0.1", + AgentPort: "9998", + AgentTLS: true, + }, + AdminAccessLogPath: "/dev/null", + AdminBindAddress: "127.0.0.1", + AdminBindPort: "19000", + LocalAgentClusterName: xds.LocalAgentClusterName, + PrometheusScrapePath: "/metrics", + }, + }, { Name: "deprecated-grpc-addr-config", Flags: []string{"-proxy-id", "test-proxy"}, - XDSPort: 9999, + XDSPorts: agent.GRPCPorts{Plaintext: 9999, TLS: 0}, AgentSelf110: true, WantArgs: BootstrapTplArgs{ ProxyCluster: "test-proxy", @@ -1138,7 +1164,7 @@ func testMockAgent(tc generateConfigTestCase) http.HandlerFunc { case strings.Contains(r.URL.Path, "/agent/service"): testMockAgentProxyConfig(tc.ProxyConfig, tc.NamespacesEnabled)(w, r) case strings.Contains(r.URL.Path, "/agent/self"): - testMockAgentSelf(tc.XDSPort, tc.AgentSelf110)(w, r) + testMockAgentSelf(tc.XDSPorts, tc.AgentSelf110)(w, r) case strings.Contains(r.URL.Path, "/catalog/node-services"): testMockCatalogNodeServiceList()(w, r) default: @@ -1378,7 +1404,7 @@ func TestEnvoyCommand_canBindInternal(t *testing.T) { // testMockAgentSelf returns an empty /v1/agent/self response except GRPC // port is filled in to match the given wantXDSPort argument. -func testMockAgentSelf(wantXDSPort int, agentSelf110 bool) http.HandlerFunc { +func testMockAgentSelf(wantXDSPorts agent.GRPCPorts, agentSelf110 bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { resp := agent.Self{ Config: map[string]interface{}{ @@ -1388,10 +1414,17 @@ func testMockAgentSelf(wantXDSPort int, agentSelf110 bool) http.HandlerFunc { if agentSelf110 { resp.DebugConfig = map[string]interface{}{ - "GRPCPort": wantXDSPort, + "GRPCPort": wantXDSPorts.Plaintext, } } else { - resp.XDS = &agent.XDSSelf{Port: wantXDSPort} + resp.XDS = &agent.XDSSelf{ + // The deprecated Port field should default to TLS if it's available. + Port: wantXDSPorts.TLS, + Ports: wantXDSPorts, + } + if wantXDSPorts.TLS <= 0 { + resp.XDS.Port = wantXDSPorts.Plaintext + } } selfJSON, err := json.Marshal(resp) diff --git a/command/connect/envoy/testdata/grpc-tls-addr-config.golden b/command/connect/envoy/testdata/grpc-tls-addr-config.golden new file mode 100644 index 0000000000..e79cf0455d --- /dev/null +++ b/command/connect/envoy/testdata/grpc-tls-addr-config.golden @@ -0,0 +1,223 @@ +{ + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 19000 + } + } + }, + "node": { + "cluster": "test", + "id": "test-proxy", + "metadata": { + "namespace": "default", + "partition": "default" + } + }, + "layered_runtime": { + "layers": [ + { + "name": "base", + "static_layer": { + "re2.max_program_size.error_level": 1048576 + } + } + ] + }, + "static_resources": { + "clusters": [ + { + "name": "local_agent", + "ignore_health_on_host_removal": false, + "connect_timeout": "1s", + "type": "STATIC", + "transport_socket": { + "name": "tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "common_tls_context": { + "validation_context": { + "trusted_ca": { + "inline_string": "" + } + } + } + } + }, + "http2_protocol_options": {}, + "loadAssignment": { + "clusterName": "local_agent", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9998 + } + } + } + } + ] + } + ] + } + } + ] + }, + "stats_config": { + "stats_tags": [ + { + "regex": "^cluster\\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.custom_hash" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service_subset" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.service" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.namespace" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:([^.]+)\\.)?[^.]+\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.partition" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.datacenter" + }, + { + "regex": "^cluster\\.([^.]+\\.(?:[^.]+\\.)?([^.]+)\\.external\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.peer" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.routing_type" + }, + { + "regex": "^cluster\\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.destination.trust_domain" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.destination.target" + }, + { + "regex": "^cluster\\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.destination.full_target" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.(([^.]+)(?:\\.[^.]+)?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.service" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.datacenter" + }, + { + "regex": "^(?:tcp|http)\\.upstream_peered\\.([^.]+(?:\\.[^.]+)?\\.([^.]+)\\.)", + "tag_name": "consul.upstream.peer" + }, + { + "regex": "^(?:tcp|http)\\.upstream(?:_peered)?\\.([^.]+(?:\\.([^.]+))?(?:\\.[^.]+)?\\.[^.]+\\.)", + "tag_name": "consul.upstream.namespace" + }, + { + "regex": "^(?:tcp|http)\\.upstream\\.([^.]+(?:\\.[^.]+)?(?:\\.([^.]+))?\\.[^.]+\\.)", + "tag_name": "consul.upstream.partition" + }, + { + "regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.custom_hash" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service_subset" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.service" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.namespace" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?([^.]+)\\.internal[^.]*\\.[^.]+\\.consul\\.)", + "tag_name": "consul.datacenter" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)", + "tag_name": "consul.routing_type" + }, + { + "regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)", + "tag_name": "consul.trust_domain" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)", + "tag_name": "consul.target" + }, + { + "regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)", + "tag_name": "consul.full_target" + }, + { + "tag_name": "local_cluster", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.service", + "fixed_value": "test" + }, + { + "tag_name": "consul.source.namespace", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.partition", + "fixed_value": "default" + }, + { + "tag_name": "consul.source.datacenter", + "fixed_value": "dc1" + } + ], + "use_all_default_tags": true + }, + "dynamic_resources": { + "lds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "cds_config": { + "ads": {}, + "resource_api_version": "V3" + }, + "ads_config": { + "api_type": "DELTA_GRPC", + "transport_api_version": "V3", + "grpc_services": { + "initial_metadata": [ + { + "key": "x-consul-token", + "value": "" + } + ], + "envoy_grpc": { + "cluster_name": "local_agent" + } + } + } + } +} + diff --git a/website/content/docs/agent/config/config-files.mdx b/website/content/docs/agent/config/config-files.mdx index 2631378731..bbeca24a2d 100644 --- a/website/content/docs/agent/config/config-files.mdx +++ b/website/content/docs/agent/config/config-files.mdx @@ -608,6 +608,10 @@ Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'." automatically with this. This is set to `8502` by default when the agent runs in `-dev` mode. Currently gRPC is only used to expose Envoy xDS API to Envoy proxies. + - `grpc_tls` ((#grpc_tls_port)) - The gRPC API with TLS connections, -1 to disable. Default -1 (disabled). + **We recommend using `8502`** for `grpc_tls` by convention as some tooling will work + automatically with this. Usually only one of the `grpc_tls` or `grpc` ports should be defined. + Currently gRPC is only used to expose Envoy xDS API to Envoy proxies. - `serf_lan` ((#serf_lan_port)) - The Serf LAN port. Default 8301. TCP and UDP. Equivalent to the [`-serf-lan-port` command line flag](/docs/agent/config/cli-flags#_serf_lan_port). - `serf_wan` ((#serf_wan_port)) - The Serf WAN port. Default 8302. @@ -2003,7 +2007,7 @@ specially crafted certificate signed by the CA can be used to gain full access t interface. - `grpc` ((#tls_grpc)) Provides settings for the gRPC/xDS interface. To enable - the gRPC interface you must define a port via [`ports.grpc`](#grpc_port). + the gRPC interface you must define a port via [`ports.grpc_tls`](#grpc_tls_port). - `ca_file` ((#tls_grpc_ca_file)) Overrides [`tls.defaults.ca_file`](#tls_defaults_ca_file). diff --git a/website/content/docs/upgrading/upgrade-specific.mdx b/website/content/docs/upgrading/upgrade-specific.mdx index 2732ffe4f5..ab288d09d0 100644 --- a/website/content/docs/upgrading/upgrade-specific.mdx +++ b/website/content/docs/upgrading/upgrade-specific.mdx @@ -14,6 +14,32 @@ provided for their upgrades as a result of new features or changed behavior. This page is used to document those details separately from the standard upgrade flow. +## Consul 1.14.x + +### Service Mesh Compatibility + +##### Changes to gRPC TLS configuration + +**Configuration changes should be made** if using sidecar proxies or gateways +in conjunction with any of the following: +1. [`ports.https`](/docs/agent/config/config-files#https_port) - Encrypts gRPC in Consul 1.12 and prior +2. [`auto_encrypt`](/docs/agent/config/config-files#auto_encrypt) - Encrypts gRPC in Consul 1.13 and prior +3. [`auto_config`](/docs/agent/config/config-files#auto_config) - Encrypts gRPC in Consul 1.13 and prior + +Prior to Consul 1.14, it was possible for communication between Consul and Envoy over `ports.grpc` +to be encrypted by one of these features. + +In Consul 1.14, a new [`ports.grpc_tls`](/docs/agent/config/config-files#grpc_tls_port) configuration +is introduced. The existing [`ports.grpc`](/docs/agent/config/config-files#grpc_port) configuration +**will stop supporting encryption in a future release**. Now, the recommended way to encrypt gRPC +traffic is only via `ports.grpc_tls`. + +For most environments, the Envoy communication to Consul is loop-back only and does not benefit from encryption. + +If you would like to continue utilizing encryption for gRPC, change the existing `ports.grpc` to `ports.grpc_tls` in +your configuration during the upgrade to ensure compatibility with future releases. + + ## Consul 1.13.x ### Service Mesh Compatibility