diff --git a/agent/agent.go b/agent/agent.go index 0199f5cba9..a8885b9e60 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3954,6 +3954,15 @@ func (a *Agent) registerCache() { RefreshTimer: 0 * time.Second, RefreshTimeout: 10 * time.Minute, }) + + a.cache.RegisterType(cachetype.ConfigEntriesName, &cachetype.ConfigEntries{ + RPC: a, + }, &cache.RegisterOptions{ + // Maintain a blocking query, retry dropped connections quickly + Refresh: true, + RefreshTimer: 0 * time.Second, + RefreshTimeout: 10 * time.Minute, + }) } // defaultProxyCommand returns the default Connect managed proxy command. diff --git a/agent/cache-types/config_entry.go b/agent/cache-types/config_entry.go new file mode 100644 index 0000000000..2a93971666 --- /dev/null +++ b/agent/cache-types/config_entry.go @@ -0,0 +1,51 @@ +package cachetype + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/structs" +) + +// Recommended name for registration. +const ConfigEntriesName = "config-entries" + +// ConfigEntries supports fetching discovering configuration entries +type ConfigEntries struct { + RPC RPC +} + +func (c *ConfigEntries) Fetch(opts cache.FetchOptions, req cache.Request) (cache.FetchResult, error) { + var result cache.FetchResult + + // The request should be a ConfigEntryQuery. + reqReal, ok := req.(*structs.ConfigEntryQuery) + if !ok { + return result, fmt.Errorf( + "Internal cache failure: request wrong type: %T", req) + } + + // Set the minimum query index to our current index so we block + reqReal.QueryOptions.MinQueryIndex = opts.MinIndex + reqReal.QueryOptions.MaxQueryTime = opts.Timeout + + // Always allow stale - there's no point in hitting leader if the request is + // going to be served from cache and endup arbitrarily stale anyway. This + // allows cached service-discover to automatically read scale across all + // servers too. + reqReal.AllowStale = true + + // Fetch + var reply structs.IndexedConfigEntries + if err := c.RPC.RPC("ConfigEntry.List", reqReal, &reply); err != nil { + return result, err + } + + result.Value = &reply + result.Index = reply.QueryMeta.Index + return result, nil +} + +func (c *ConfigEntries) SupportsBlocking() bool { + return true +} diff --git a/agent/cache-types/config_entry_test.go b/agent/cache-types/config_entry_test.go new file mode 100644 index 0000000000..056c23ab82 --- /dev/null +++ b/agent/cache-types/config_entry_test.go @@ -0,0 +1,66 @@ +package cachetype + +import ( + "testing" + "time" + + "github.com/hashicorp/consul/agent/cache" + "github.com/hashicorp/consul/agent/structs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestConfigEntries(t *testing.T) { + rpc := TestRPC(t) + typ := &ConfigEntries{RPC: rpc} + + // Expect the proper RPC call. This also sets the expected value + // since that is return-by-pointer in the arguments. + var resp *structs.IndexedConfigEntries + rpc.On("RPC", "ConfigEntry.List", mock.Anything, mock.Anything).Return(nil). + Run(func(args mock.Arguments) { + req := args.Get(1).(*structs.ConfigEntryQuery) + require.Equal(t, uint64(24), req.QueryOptions.MinQueryIndex) + require.Equal(t, 1*time.Second, req.QueryOptions.MaxQueryTime) + require.True(t, req.AllowStale) + require.Equal(t, structs.ServiceResolver, req.Kind) + require.Equal(t, "", req.Name) + + reply := args.Get(2).(*structs.IndexedConfigEntries) + reply.Kind = structs.ServiceResolver + reply.Entries = []structs.ConfigEntry{ + &structs.ServiceResolverConfigEntry{Kind: structs.ServiceResolver, Name: "foo"}, + &structs.ServiceResolverConfigEntry{Kind: structs.ServiceResolver, Name: "bar"}, + } + reply.QueryMeta.Index = 48 + resp = reply + }) + + // Fetch + resultA, err := typ.Fetch(cache.FetchOptions{ + MinIndex: 24, + Timeout: 1 * time.Second, + }, &structs.ConfigEntryQuery{ + Datacenter: "dc1", + Kind: structs.ServiceResolver, + }) + require.NoError(t, err) + require.Equal(t, cache.FetchResult{ + Value: resp, + Index: 48, + }, resultA) + + rpc.AssertExpectations(t) +} + +func TestConfigEntries_badReqType(t *testing.T) { + rpc := TestRPC(t) + typ := &ConfigEntries{RPC: rpc} + + // Fetch + _, err := typ.Fetch(cache.FetchOptions{}, cache.TestRequest( + t, cache.RequestInfo{Key: "foo", MinIndex: 64})) + require.Error(t, err) + require.Contains(t, err.Error(), "wrong type") + rpc.AssertExpectations(t) +} diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 0e2b0dff50..22e94bcaa1 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -19,6 +19,7 @@ type configSnapshotMeshGateway struct { WatchedServices map[string]context.CancelFunc WatchedDatacenters map[string]context.CancelFunc ServiceGroups map[string]structs.CheckServiceNodes + ServiceResolvers map[string]*structs.ServiceResolverConfigEntry GatewayGroups map[string]structs.CheckServiceNodes } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 0ae2fd7335..accfe9e4c6 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -22,6 +22,7 @@ const ( intentionsWatchID = "intentions" serviceListWatchID = "service-list" datacentersWatchID = "datacenters" + serviceResolversWatchID = "service-resolvers" serviceIDPrefix = string(structs.UpstreamDestTypeService) + ":" preparedQueryIDPrefix = string(structs.UpstreamDestTypePreparedQuery) + ":" defaultPreparedQueryPollInterval = 30 * time.Second @@ -347,9 +348,10 @@ func (s *state) run() { case structs.ServiceKindMeshGateway: snap.MeshGateway.WatchedServices = make(map[string]context.CancelFunc) snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc) - // TODO (mesh-gateway) - maybe reuse UpstreamEndpoints? snap.MeshGateway.ServiceGroups = make(map[string]structs.CheckServiceNodes) snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes) + // there is no need to initialize the map of service resolvers as we + // fully rebuild it every time we get updates } // This turns out to be really fiddly/painful by just using time.Timer.C @@ -675,7 +677,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho return err } - // TODO (mesh-gateway) also should watch the resolver config for the service here + err = s.cache.Notify(ctx, cachetype.ConfigEntriesName, &structs.ConfigEntryQuery{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + Kind: structs.ServiceResolver, + }, serviceResolversWatchID, s.ch) + + if err != nil { + s.logger.Printf("[ERR] mesh-gateway: failed to register watch for service-resolver config entries") + cancel() + return err + } snap.MeshGateway.WatchedServices[svcName] = cancel } } @@ -735,6 +747,19 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho cancelFn() } } + case serviceResolversWatchID: + configEntries, ok := u.Result.(*structs.IndexedConfigEntries) + if !ok { + return fmt.Errorf("invalid type for services response: %T", u.Result) + } + + resolvers := make(map[string]*structs.ServiceResolverConfigEntry) + for _, entry := range configEntries.Entries { + if resolver, ok := entry.(*structs.ServiceResolverConfigEntry); ok { + resolvers[resolver.Name] = resolver + } + } + snap.MeshGateway.ServiceResolvers = resolvers default: switch { case strings.HasPrefix(u.CorrelationID, "connect-service:"): diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 2fe6e6b006..3fbc91b13e 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -2,6 +2,7 @@ package proxycfg import ( "context" + "fmt" "sync" "sync/atomic" "time" @@ -85,7 +86,7 @@ func TestCerts(t testing.T) (*structs.IndexedCARoots, *structs.IssuedCert) { ca := connect.TestCA(t, nil) roots := &structs.IndexedCARoots{ ActiveRootID: ca.ID, - TrustDomain: connect.TestClusterID, + TrustDomain: fmt.Sprintf("%s.consul", connect.TestClusterID), Roots: []*structs.CARoot{ca}, } return roots, TestLeafForCA(t, ca) @@ -183,7 +184,75 @@ func TestGatewayNodesDC2(t testing.T) structs.CheckServiceNodes { } } -func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes { +func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes { + return structs.CheckServiceNodes{ + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "bar-node-1", + Node: "bar-node-1", + Address: "10.1.1.4", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "bar-sidecar-proxy", + Address: "172.16.1.6", + Port: 2222, + Meta: map[string]string{ + "version": "1", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "bar", + Upstreams: structs.TestUpstreams(t), + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "bar-node-2", + Node: "bar-node-2", + Address: "10.1.1.5", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "bar-sidecar-proxy", + Address: "172.16.1.7", + Port: 2222, + Meta: map[string]string{ + "version": "1", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "bar", + Upstreams: structs.TestUpstreams(t), + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "bar-node-3", + Node: "bar-node-3", + Address: "10.1.1.6", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "bar-sidecar-proxy", + Address: "172.16.1.8", + Port: 2222, + Meta: map[string]string{ + "version": "2", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "bar", + Upstreams: structs.TestUpstreams(t), + }, + }, + }, + } +} + +func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes { return structs.CheckServiceNodes{ structs.CheckServiceNode{ Node: &structs.Node{ @@ -192,7 +261,19 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes { Address: "10.1.1.1", Datacenter: "dc1", }, - Service: structs.TestNodeServiceProxy(t), + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "foo-sidecar-proxy", + Address: "172.16.1.3", + Port: 2222, + Meta: map[string]string{ + "version": "1", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "foo", + Upstreams: structs.TestUpstreams(t), + }, + }, }, structs.CheckServiceNode{ Node: &structs.Node{ @@ -201,7 +282,69 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes { Address: "10.1.1.2", Datacenter: "dc1", }, - Service: structs.TestNodeServiceProxy(t), + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "foo-sidecar-proxy", + Address: "172.16.1.4", + Port: 2222, + Meta: map[string]string{ + "version": "1", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "foo", + Upstreams: structs.TestUpstreams(t), + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "foo-node-3", + Node: "foo-node-3", + Address: "10.1.1.3", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "foo-sidecar-proxy", + Address: "172.16.1.5", + Port: 2222, + Meta: map[string]string{ + "version": "2", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "foo", + Upstreams: structs.TestUpstreams(t), + }, + }, + }, + structs.CheckServiceNode{ + Node: &structs.Node{ + ID: "foo-node-4", + Node: "foo-node-4", + Address: "10.1.1.7", + Datacenter: "dc1", + }, + Service: &structs.NodeService{ + Kind: structs.ServiceKindConnectProxy, + Service: "foo-sidecar-proxy", + Address: "172.16.1.9", + Port: 2222, + Meta: map[string]string{ + "version": "2", + }, + Proxy: structs.ConnectProxyConfig{ + DestinationServiceName: "foo", + Upstreams: structs.TestUpstreams(t), + }, + }, + Checks: structs.HealthChecks{ + &structs.HealthCheck{ + Node: "foo-node-4", + ServiceName: "foo-sidecar-proxy", + Name: "proxy-alive", + Status: "warning", + }, + }, }, } } @@ -268,7 +411,8 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot { "dc2": nil, }, ServiceGroups: map[string]structs.CheckServiceNodes{ - "foo": TestGatewayServicesDC1(t), + "foo": TestGatewayServiceGroupFooDC1(t), + "bar": TestGatewayServiceGroupBarDC1(t), }, GatewayGroups: map[string]structs.CheckServiceNodes{ "dc2": TestGatewayNodesDC2(t), diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 6ef8aec822..9cda7ec3de 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -387,6 +387,31 @@ func (c *ConfigEntryQuery) RequestDatacenter() string { return c.Datacenter } +func (r *ConfigEntryQuery) CacheInfo() cache.RequestInfo { + info := cache.RequestInfo{ + Token: r.Token, + Datacenter: r.Datacenter, + MinIndex: r.MinQueryIndex, + Timeout: r.MaxQueryTime, + MaxAge: r.MaxAge, + MustRevalidate: r.MustRevalidate, + } + + v, err := hashstructure.Hash([]interface{}{ + r.Kind, + r.Name, + r.Filter, + }, nil) + if err == nil { + // If there is an error, we don't set the key. A blank key forces + // no cache for this request so the request is forwarded directly + // to the server. + info.Key = strconv.FormatUint(v, 10) + } + + return info +} + // ServiceConfigRequest is used when requesting the resolved configuration // for a service. type ServiceConfigRequest struct { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index df203e2004..8012f81a3b 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -84,33 +84,42 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) { - // TODO (mesh-gateway) will need to generate 1 cluster for each service subset as well. + // 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) + clusters := make([]proto.Message, 0, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups)) - // 1 cluster per remote dc + 1 cluster per local service - clusters := make([]proto.Message, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups)) - - var err error - idx := 0 // generate the remote dc clusters for dc, _ := range cfgSnap.MeshGateway.GatewayGroups { clusterName := DatacenterSNI(dc, cfgSnap) - clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap) + cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap) if err != nil { return nil, err } - idx += 1 + clusters = append(clusters, cluster) } // generate the per-service clusters for svc, _ := range cfgSnap.MeshGateway.ServiceGroups { - clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap) + clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap) - clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap) + cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap) if err != nil { return nil, err } - idx += 1 + clusters = append(clusters, cluster) + } + + // generate the service subset clusters + for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers { + for subsetName, _ := range resolver.Subsets { + clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap) + + cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) + } } return clusters, nil @@ -174,7 +183,7 @@ func (s *Server) makeUpstreamCluster(upstream structs.Upstream, cfgSnap *proxycf if upstream.Datacenter != "" { dc = upstream.Datacenter } - sni := ServiceSNI(upstream.DestinationName, ns, dc, cfgSnap) + sni := ServiceSNI(upstream.DestinationName, "", ns, dc, cfgSnap) cfg, err := ParseUpstreamConfig(upstream.Config) if err != nil { diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 1e7051b9d7..8ca9ec9dc3 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -9,6 +9,7 @@ import ( "text/template" "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" testinf "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" ) @@ -102,6 +103,27 @@ func TestClustersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotMeshGateway, setup: nil, }, + { + name: "mesh-gateway-service-subsets", + create: proxycfg.TestConfigSnapshotMeshGateway, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{ + "bar": &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "bar", + Subsets: map[string]structs.ServiceResolverSubset{ + "v1": structs.ServiceResolverSubset{ + Filter: "Service.Meta.Version == 1", + }, + "v2": structs.ServiceResolverSubset{ + Filter: "Service.Meta.Version == 2", + OnlyPassing: true, + }, + }, + }, + } + }, + }, } for _, tt := range tests { @@ -190,7 +212,7 @@ func expectClustersJSONResources(t *testing.T, snap *proxycfg.ConfigSnapshot, to }, "connectTimeout": "1s", - "tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "db.default.dc1.internal.11111111-2222-3333-4444-555555555555") + ` + "tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul") + ` }`, "prepared_query:geo-cache": ` { @@ -208,7 +230,7 @@ func expectClustersJSONResources(t *testing.T, snap *proxycfg.ConfigSnapshot, to }, "connectTimeout": "5s", - "tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555") + ` + "tlsContext": ` + expectedUpstreamTLSContextJSON(t, snap, "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul") + ` }`, } } diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 9250a9061f..671df61c6e 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -12,6 +12,8 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/api" + + bexpr "github.com/hashicorp/go-bexpr" ) // endpointsFromSnapshot returns the xDS API representation of the "endpoints" @@ -143,7 +145,7 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh // generate the endpoints for the local service groups for svc, endpoints := range cfgSnap.MeshGateway.ServiceGroups { - clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap) + clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap) la := makeLoadAssignment( clusterName, 0, @@ -155,6 +157,51 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh resources = append(resources, la) } + // generate the endpoints for the service subsets + for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers { + for subsetName, subset := range resolver.Subsets { + clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap) + + endpoints := cfgSnap.MeshGateway.ServiceGroups[svc] + + // locally execute the subsets filter + filterExp := subset.Filter + if subset.OnlyPassing { + // we could do another filter pass without bexpr but this simplifies things a bit + if filterExp != "" { + // TODO (filtering) - Update to "and all Checks as chk { chk.Status == passing }" + // once the syntax is supported + filterExp = fmt.Sprintf("(%s) and not Checks.Status != passing", filterExp) + } else { + filterExp = "not Checks.Status != passing" + } + } + + if filterExp != "" { + filter, err := bexpr.CreateFilter(filterExp, nil, endpoints) + if err != nil { + return nil, err + } + + raw, err := filter.Execute(endpoints) + if err != nil { + return nil, err + } + endpoints = raw.(structs.CheckServiceNodes) + } + + la := makeLoadAssignment( + clusterName, + 0, + []structs.CheckServiceNodes{ + endpoints, + }, + cfgSnap.Datacenter, + ) + resources = append(resources, la) + } + } + return resources, nil } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index b55c39c756..c1de4c925f 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -233,6 +233,40 @@ func Test_endpointsFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotMeshGateway, setup: nil, }, + { + name: "mesh-gateway-service-subsets", + create: proxycfg.TestConfigSnapshotMeshGateway, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{ + "bar": &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "bar", + Subsets: map[string]structs.ServiceResolverSubset{ + "v1": structs.ServiceResolverSubset{ + Filter: "Service.Meta.version == 1", + }, + "v2": structs.ServiceResolverSubset{ + Filter: "Service.Meta.version == 2", + OnlyPassing: true, + }, + }, + }, + "foo": &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "foo", + Subsets: map[string]structs.ServiceResolverSubset{ + "v1": structs.ServiceResolverSubset{ + Filter: "Service.Meta.version == 1", + }, + "v2": structs.ServiceResolverSubset{ + Filter: "Service.Meta.version == 2", + OnlyPassing: true, + }, + }, + }, + } + }, + }, } for _, tt := range tests { diff --git a/agent/xds/sni.go b/agent/xds/sni.go index a01021beb3..b41d49076f 100644 --- a/agent/xds/sni.go +++ b/agent/xds/sni.go @@ -10,7 +10,10 @@ func DatacenterSNI(dc string, cfgSnap *proxycfg.ConfigSnapshot) string { return fmt.Sprintf("%s.internal.%s", dc, cfgSnap.Roots.TrustDomain) } -func ServiceSNI(service string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string { - // TODO (mesh-gateway) - support service subsets here too - return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain) +func ServiceSNI(service string, subset string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string { + if subset == "" { + return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain) + } else { + return fmt.Sprintf("%s.%s.%s.%s.internal.%s", subset, service, namespace, datacenter, cfgSnap.Roots.TrustDomain) + } } diff --git a/agent/xds/testdata/clusters/custom-local-app.golden b/agent/xds/testdata/clusters/custom-local-app.golden index 79a2210be4..5266b1fc13 100644 --- a/agent/xds/testdata/clusters/custom-local-app.golden +++ b/agent/xds/testdata/clusters/custom-local-app.golden @@ -47,7 +47,7 @@ } } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { @@ -86,7 +86,7 @@ } } }, - "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { diff --git a/agent/xds/testdata/clusters/custom-timeouts.golden b/agent/xds/testdata/clusters/custom-timeouts.golden index a5688aa4e4..f1846f8eb9 100644 --- a/agent/xds/testdata/clusters/custom-timeouts.golden +++ b/agent/xds/testdata/clusters/custom-timeouts.golden @@ -59,7 +59,7 @@ } } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { @@ -98,7 +98,7 @@ } } }, - "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { diff --git a/agent/xds/testdata/clusters/custom-upstream.golden b/agent/xds/testdata/clusters/custom-upstream.golden index 3b9527ddf8..ffc14d89b5 100644 --- a/agent/xds/testdata/clusters/custom-upstream.golden +++ b/agent/xds/testdata/clusters/custom-upstream.golden @@ -59,7 +59,7 @@ } } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" } }, { @@ -95,7 +95,7 @@ } } }, - "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { diff --git a/agent/xds/testdata/clusters/defaults.golden b/agent/xds/testdata/clusters/defaults.golden index 25a9af804a..f1f404456f 100644 --- a/agent/xds/testdata/clusters/defaults.golden +++ b/agent/xds/testdata/clusters/defaults.golden @@ -59,7 +59,7 @@ } } }, - "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { @@ -98,7 +98,7 @@ } } }, - "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555" + "sni": "geo-cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" }, "outlierDetection": { diff --git a/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden new file mode 100644 index 0000000000..2a01dc2509 --- /dev/null +++ b/agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden @@ -0,0 +1,87 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/clusters/mesh-gateway.golden b/agent/xds/testdata/clusters/mesh-gateway.golden index 2f7899a338..7c8ef1f42b 100644 --- a/agent/xds/testdata/clusters/mesh-gateway.golden +++ b/agent/xds/testdata/clusters/mesh-gateway.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.Cluster", - "name": "dc2.internal.11111111-2222-3333-4444-555555555555", + "name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "type": "EDS", "edsClusterConfig": { "edsConfig": { @@ -19,7 +19,23 @@ }, { "@type": "type.googleapis.com/envoy.api.v2.Cluster", - "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555", + "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + } + } + }, + "connectTimeout": "5s", + "outlierDetection": { + + } + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Cluster", + "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "type": "EDS", "edsClusterConfig": { "edsConfig": { diff --git a/agent/xds/testdata/endpoints/mesh-gateway-service-subsets.golden b/agent/xds/testdata/endpoints/mesh-gateway-service-subsets.golden new file mode 100644 index 0000000000..87ac0b806a --- /dev/null +++ b/agent/xds/testdata/endpoints/mesh-gateway-service-subsets.golden @@ -0,0 +1,257 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.1", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "198.18.1.2", + "portValue": 443 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.3", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.4", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.5", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.9", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.6", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.7", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.8", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.6", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.7", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.8", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "v1.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.3", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.4", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "v2.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.5", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/endpoints/mesh-gateway.golden b/agent/xds/testdata/endpoints/mesh-gateway.golden index 236cf4aeac..ad013f1df0 100644 --- a/agent/xds/testdata/endpoints/mesh-gateway.golden +++ b/agent/xds/testdata/endpoints/mesh-gateway.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555", + "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "endpoints": [ { "lbEndpoints": [ @@ -37,7 +37,7 @@ }, { "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", - "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555", + "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", "endpoints": [ { "lbEndpoints": [ @@ -45,7 +45,7 @@ "endpoint": { "address": { "socketAddress": { - "address": "127.0.0.2", + "address": "172.16.1.3", "portValue": 2222 } } @@ -57,7 +57,77 @@ "endpoint": { "address": { "socketAddress": { - "address": "127.0.0.2", + "address": "172.16.1.4", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.5", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.9", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment", + "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.6", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.7", + "portValue": 2222 + } + } + }, + "healthStatus": "HEALTHY", + "loadBalancingWeight": 1 + }, + { + "endpoint": { + "address": { + "socketAddress": { + "address": "172.16.1.8", "portValue": 2222 } } diff --git a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden index 33bafcd9a9..0ef43158bc 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-custom-addresses.golden @@ -14,14 +14,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_bar_dc2_tcp" } } @@ -61,14 +61,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_baz_dc2_tcp" } } @@ -108,14 +108,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_default_dc2_tcp" } } @@ -155,14 +155,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_foo_dc2_tcp" } } diff --git a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden index ed0e11aee8..7ee0199667 100644 --- a/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden +++ b/agent/xds/testdata/listeners/mesh-gateway-tagged-addresses.golden @@ -14,14 +14,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_lan_dc2_tcp" } } @@ -61,14 +61,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_wan_dc2_tcp" } } diff --git a/agent/xds/testdata/listeners/mesh-gateway.golden b/agent/xds/testdata/listeners/mesh-gateway.golden index 6f95010b33..85b6ad1f4c 100644 --- a/agent/xds/testdata/listeners/mesh-gateway.golden +++ b/agent/xds/testdata/listeners/mesh-gateway.golden @@ -14,14 +14,14 @@ { "filterChainMatch": { "serverNames": [ - "*.dc2.internal.11111111-2222-3333-4444-555555555555" + "*.dc2.internal.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ { "name": "envoy.tcp_proxy", "config": { - "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555", + "cluster": "dc2.internal.11111111-2222-3333-4444-555555555555.consul", "stat_prefix": "mesh_gateway_remote_default_dc2_tcp" } }