From 7e65678c528d12f235642865c84078be4698155a Mon Sep 17 00:00:00 2001 From: freddygv Date: Sat, 23 Oct 2021 13:19:51 -0600 Subject: [PATCH 01/14] Update mesh gateway proxy watches for partitions This commit updates mesh gateway watches for cross-partitions communication. * Mesh gateways are keyed by partition and datacenter. * Mesh gateways will now watch gateways in partitions that export services to their partition. * Mesh gateways in non-default partitions will not have cross-datacenter watches. They are not involved in traditional WAN federation. --- agent/proxycfg/mesh_gateway.go | 142 +++++++++++++++++------------ agent/proxycfg/mesh_gateway_oss.go | 19 ++++ agent/proxycfg/snapshot.go | 10 +- agent/proxycfg/state.go | 1 + agent/proxycfg/state_test.go | 4 +- agent/proxycfg/testing.go | 4 +- 6 files changed, 114 insertions(+), 66 deletions(-) create mode 100644 agent/proxycfg/mesh_gateway_oss.go diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index 6a0856cf94..d5463e9187 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/structs" @@ -31,7 +32,10 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er wildcardEntMeta := s.proxyID.WithWildcardNamespace() - // Watch for all services + // Watch for all services. + // Eventually we will have to watch connect enable instances for each service as well as the + // destination services themselves but those notifications will be setup later. + // We cannot setup those watches until we know what the services are. err = s.cache.Notify(ctx, cachetype.CatalogServiceListName, &structs.DCSpecificRequest{ Datacenter: s.source.Datacenter, QueryOptions: structs.QueryOptions{Token: s.token}, @@ -43,45 +47,6 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er return snap, err } - if s.meta[structs.MetaWANFederationKey] == "1" { - // Conveniently we can just use this service meta attribute in one - // place here to set the machinery in motion and leave the conditional - // behavior out of the rest of the package. - err = s.cache.Notify(ctx, cachetype.FederationStateListMeshGatewaysName, &structs.DCSpecificRequest{ - Datacenter: s.source.Datacenter, - QueryOptions: structs.QueryOptions{Token: s.token}, - Source: *s.source, - }, federationStateListGatewaysWatchID, s.ch) - if err != nil { - return snap, err - } - - err = s.health.Notify(ctx, structs.ServiceSpecificRequest{ - Datacenter: s.source.Datacenter, - QueryOptions: structs.QueryOptions{Token: s.token}, - ServiceName: structs.ConsulServiceName, - }, consulServerListWatchID, s.ch) - if err != nil { - return snap, err - } - } - - // Eventually we will have to watch connect enable instances for each service as well as the - // destination services themselves but those notifications will be setup later. However we - // cannot setup those watches until we know what the services are. from the service list - // watch above - - err = s.cache.Notify(ctx, cachetype.CatalogDatacentersName, &structs.DatacentersRequest{ - QueryOptions: structs.QueryOptions{Token: s.token, MaxAge: 30 * time.Second}, - }, datacentersWatchID, s.ch) - if err != nil { - return snap, err - } - - // Once we start getting notified about the datacenters we will setup watches on the - // gateways within those other datacenters. We cannot do that here because we don't - // know what they are yet. - // Watch service-resolvers so we can setup service subset clusters err = s.cache.Notify(ctx, cachetype.ConfigEntriesName, &structs.ConfigEntryQuery{ Datacenter: s.source.Datacenter, @@ -95,17 +60,66 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er return snap, err } + if s.proxyID.PartitionOrEmpty() == acl.DefaultPartitionName { + if err := s.initializeCrossDCWatches(ctx); err != nil { + return snap, err + } + } + + if err := s.initializeEntWatches(ctx); err != nil { + return snap, err + } + snap.MeshGateway.WatchedServices = make(map[structs.ServiceName]context.CancelFunc) - snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc) + snap.MeshGateway.WatchedGateways = make(map[string]context.CancelFunc) snap.MeshGateway.ServiceGroups = make(map[structs.ServiceName]structs.CheckServiceNodes) snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes) snap.MeshGateway.ServiceResolvers = make(map[structs.ServiceName]*structs.ServiceResolverConfigEntry) snap.MeshGateway.HostnameDatacenters = make(map[string]structs.CheckServiceNodes) + // there is no need to initialize the map of service resolvers as we // fully rebuild it every time we get updates return snap, err } +func (s *handlerMeshGateway) initializeCrossDCWatches(ctx context.Context) error { + if s.meta[structs.MetaWANFederationKey] == "1" { + // Conveniently we can just use this service meta attribute in one + // place here to set the machinery in motion and leave the conditional + // behavior out of the rest of the package. + err := s.cache.Notify(ctx, cachetype.FederationStateListMeshGatewaysName, &structs.DCSpecificRequest{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + Source: *s.source, + }, federationStateListGatewaysWatchID, s.ch) + if err != nil { + return err + } + + err = s.health.Notify(ctx, structs.ServiceSpecificRequest{ + Datacenter: s.source.Datacenter, + QueryOptions: structs.QueryOptions{Token: s.token}, + ServiceName: structs.ConsulServiceName, + }, consulServerListWatchID, s.ch) + if err != nil { + return err + } + } + + err := s.cache.Notify(ctx, cachetype.CatalogDatacentersName, &structs.DatacentersRequest{ + QueryOptions: structs.QueryOptions{Token: s.token, MaxAge: 30 * time.Second}, + }, datacentersWatchID, s.ch) + if err != nil { + return err + } + + // Once we start getting notified about the datacenters we will setup watches on the + // gateways within those other datacenters. We cannot do that here because we don't + // know what they are yet. + + return nil +} + func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEvent, snap *ConfigSnapshot) error { if u.Err != nil { return fmt.Errorf("error filling agent cache: %v", u.Err) @@ -120,6 +134,7 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve return fmt.Errorf("invalid type for response: %T", u.Result) } snap.Roots = roots + case federationStateListGatewaysWatchID: dcIndexedNodes, ok := u.Result.(*structs.DatacenterIndexedCheckServiceNodes) if !ok { @@ -181,8 +196,8 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve cancelFn() } } - snap.MeshGateway.WatchedServicesSet = true + case datacentersWatchID: datacentersRaw, ok := u.Result.(*[]string) if !ok { @@ -199,7 +214,10 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve continue } - if _, ok := snap.MeshGateway.WatchedDatacenters[dc]; !ok { + entMeta := structs.DefaultEnterpriseMetaInDefaultPartition() + gk := GatewayKey{Datacenter: dc, Partition: entMeta.PartitionOrDefault()} + + if _, ok := snap.MeshGateway.WatchedGateways[gk.String()]; !ok { ctx, cancel := context.WithCancel(ctx) err := s.cache.Notify(ctx, cachetype.InternalServiceDumpName, &structs.ServiceDumpRequest{ Datacenter: dc, @@ -207,36 +225,42 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve ServiceKind: structs.ServiceKindMeshGateway, UseServiceKind: true, Source: *s.source, - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, fmt.Sprintf("mesh-gateway:%s", dc), s.ch) + EnterpriseMeta: *entMeta, + }, fmt.Sprintf("mesh-gateway:%s", gk.String()), s.ch) if err != nil { meshLogger.Error("failed to register watch for mesh-gateway", "datacenter", dc, + "partition", entMeta.PartitionOrDefault(), "error", err, ) cancel() return err } - - snap.MeshGateway.WatchedDatacenters[dc] = cancel + snap.MeshGateway.WatchedGateways[gk.String()] = cancel } } - for dc, cancelFn := range snap.MeshGateway.WatchedDatacenters { + for key, cancelFn := range snap.MeshGateway.WatchedGateways { + gk := gatewayKeyFromString(key) + if gk.Datacenter == s.source.Datacenter { + // Only cross-DC watches are managed by the datacenters watch. + continue + } + found := false for _, dcCurrent := range datacenters { - if dcCurrent == dc { + if dcCurrent == gk.Datacenter { found = true break } } - if !found { - delete(snap.MeshGateway.WatchedDatacenters, dc) + delete(snap.MeshGateway.WatchedGateways, key) cancelFn() } } + case serviceResolversWatchID: configEntries, ok := u.Result.(*structs.IndexedConfigEntries) if !ok { @@ -286,23 +310,27 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve } else if _, ok := snap.MeshGateway.ServiceGroups[sn]; ok { delete(snap.MeshGateway.ServiceGroups, sn) } + case strings.HasPrefix(u.CorrelationID, "mesh-gateway:"): resp, ok := u.Result.(*structs.IndexedNodesWithGateways) if !ok { return fmt.Errorf("invalid type for response: %T", u.Result) } - dc := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:") - delete(snap.MeshGateway.GatewayGroups, dc) - delete(snap.MeshGateway.HostnameDatacenters, dc) + key := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:") + delete(snap.MeshGateway.GatewayGroups, key) + delete(snap.MeshGateway.HostnameDatacenters, key) if len(resp.Nodes) > 0 { - snap.MeshGateway.GatewayGroups[dc] = resp.Nodes - snap.MeshGateway.HostnameDatacenters[dc] = hostnameEndpoints( + snap.MeshGateway.GatewayGroups[key] = resp.Nodes + snap.MeshGateway.HostnameDatacenters[key] = hostnameEndpoints( s.logger.Named(logging.MeshGateway), snap.Datacenter, resp.Nodes) } + default: - // do nothing for now + if err := s.handleEntUpdate(meshLogger, ctx, u, snap); err != nil { + return err + } } } diff --git a/agent/proxycfg/mesh_gateway_oss.go b/agent/proxycfg/mesh_gateway_oss.go new file mode 100644 index 0000000000..e1b2113c9f --- /dev/null +++ b/agent/proxycfg/mesh_gateway_oss.go @@ -0,0 +1,19 @@ +// +build !consulent + +package proxycfg + +import ( + "context" + + "github.com/hashicorp/go-hclog" + + "github.com/hashicorp/consul/agent/cache" +) + +func (s *handlerMeshGateway) initializeEntWatches(_ context.Context) error { + return nil +} + +func (s *handlerMeshGateway) handleEntUpdate(_ hclog.Logger, _ context.Context, _ cache.UpdateEvent, _ *ConfigSnapshot) error { + return nil +} diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 2a1965a8a1..3c915a5698 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -256,10 +256,10 @@ type configSnapshotMeshGateway struct { // health check to pass. WatchedServicesSet bool - // WatchedDatacenters is a map of datacenter name to a cancel function. + // WatchedGateways is a map of GatewayKeys to a cancel function. // This cancel function is tied to the watch of mesh-gateway services in - // that datacenter. - WatchedDatacenters map[string]context.CancelFunc + // that datacenter/partition. + WatchedGateways map[string]context.CancelFunc // ServiceGroups is a map of service name to the service instances of that // service in the local datacenter. @@ -315,7 +315,7 @@ func (c *configSnapshotMeshGateway) IsEmpty() bool { } return len(c.WatchedServices) == 0 && !c.WatchedServicesSet && - len(c.WatchedDatacenters) == 0 && + len(c.WatchedGateways) == 0 && len(c.ServiceGroups) == 0 && len(c.ServiceResolvers) == 0 && len(c.GatewayGroups) == 0 && @@ -466,7 +466,7 @@ func (s *ConfigSnapshot) Clone() (*ConfigSnapshot, error) { snap.TerminatingGateway.WatchedConfigs = nil snap.TerminatingGateway.WatchedResolvers = nil case structs.ServiceKindMeshGateway: - snap.MeshGateway.WatchedDatacenters = nil + snap.MeshGateway.WatchedGateways = nil snap.MeshGateway.WatchedServices = nil case structs.ServiceKindIngressGateway: snap.IngressGateway.WatchedUpstreams = nil diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 2c443f6be6..cb46764f30 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -37,6 +37,7 @@ const ( datacentersWatchID = "datacenters" serviceResolversWatchID = "service-resolvers" gatewayServicesWatchID = "gateway-services" + exportingPartitionsWatchID = "exporting-partitions" gatewayConfigWatchID = "gateway-config" externalServiceIDPrefix = "external-service:" serviceLeafIDPrefix = "service-leaf:" diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 896f3ac225..38c2da65b7 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -750,7 +750,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Equal(t, indexedRoots, snap.Roots) require.Empty(t, snap.MeshGateway.WatchedServices) require.False(t, snap.MeshGateway.WatchedServicesSet) - require.Empty(t, snap.MeshGateway.WatchedDatacenters) + require.Empty(t, snap.MeshGateway.WatchedGateways) require.Empty(t, snap.MeshGateway.ServiceGroups) require.Empty(t, snap.MeshGateway.ServiceResolvers) require.Empty(t, snap.MeshGateway.GatewayGroups) @@ -772,7 +772,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { require.Equal(t, indexedRoots, snap.Roots) require.Empty(t, snap.MeshGateway.WatchedServices) require.True(t, snap.MeshGateway.WatchedServicesSet) - require.Empty(t, snap.MeshGateway.WatchedDatacenters) + require.Empty(t, snap.MeshGateway.WatchedGateways) require.Empty(t, snap.MeshGateway.ServiceGroups) require.Empty(t, snap.MeshGateway.ServiceResolvers) require.Empty(t, snap.MeshGateway.GatewayGroups) diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 94275419ab..e8f40c3901 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -1542,8 +1542,8 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera structs.NewServiceName("bar", nil): nil, }, WatchedServicesSet: true, - WatchedDatacenters: map[string]context.CancelFunc{ - "dc2": nil, + WatchedGateways: map[string]context.CancelFunc{ + "default.dc2": nil, }, ServiceGroups: map[structs.ServiceName]structs.CheckServiceNodes{ structs.NewServiceName("foo", nil): TestGatewayServiceGroupFooDC1(t), From 110fae820a27871d2b8214246547d93fe06f8f80 Mon Sep 17 00:00:00 2001 From: freddygv Date: Sat, 23 Oct 2021 14:17:29 -0600 Subject: [PATCH 02/14] Update xds pkg to account for GatewayKey --- agent/proxycfg/snapshot.go | 36 +++++++++++++++++++++++------------- agent/xds/clusters.go | 24 ++++++++++++------------ agent/xds/endpoints.go | 18 +++++++++--------- agent/xds/listeners.go | 18 +++++++++--------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 3c915a5698..ac36d15a7d 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -60,7 +60,11 @@ type GatewayKey struct { } func (k GatewayKey) String() string { - return k.Partition + "." + k.Datacenter + resp := k.Datacenter + if k.Partition != "" { + resp = k.Partition + "." + resp + } + return resp } func (k GatewayKey) IsEmpty() bool { @@ -69,10 +73,11 @@ func (k GatewayKey) IsEmpty() bool { func gatewayKeyFromString(s string) GatewayKey { split := strings.SplitN(s, ".", 2) - return GatewayKey{ - Partition: split[0], - Datacenter: split[1], + + if len(split) == 1 { + return GatewayKey{Datacenter: split[0]} } + return GatewayKey{Partition: split[0], Datacenter: split[1]} } // ServicePassthroughAddrs contains the LAN addrs @@ -285,7 +290,7 @@ type configSnapshotMeshGateway struct { HostnameDatacenters map[string]structs.CheckServiceNodes } -func (c *configSnapshotMeshGateway) Datacenters() []string { +func (c *configSnapshotMeshGateway) Keys() []GatewayKey { sz1, sz2 := len(c.GatewayGroups), len(c.FedStateGateways) sz := sz1 @@ -293,20 +298,25 @@ func (c *configSnapshotMeshGateway) Datacenters() []string { sz = sz2 } - dcs := make([]string, 0, sz) - for dc := range c.GatewayGroups { - dcs = append(dcs, dc) + keys := make([]GatewayKey, 0, sz) + for key := range c.GatewayGroups { + keys = append(keys, gatewayKeyFromString(key)) } - for dc := range c.FedStateGateways { - if _, ok := c.GatewayGroups[dc]; !ok { - dcs = append(dcs, dc) + for key := range c.FedStateGateways { + if _, ok := c.GatewayGroups[key]; !ok { + keys = append(keys, gatewayKeyFromString(key)) } } // Always sort the results to ensure we generate deterministic things over // xDS, such as mesh-gateway listener filter chains. - sort.Strings(dcs) - return dcs + sort.Slice(keys, func(i, j int) bool { + if keys[i].Datacenter != keys[j].Datacenter { + return keys[i].Datacenter < keys[j].Datacenter + } + return keys[i].Partition < keys[j].Partition + }) + return keys } func (c *configSnapshotMeshGateway) IsEmpty() bool { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index eac2ac561f..3937da1705 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -202,21 +202,21 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - datacenters := cfgSnap.MeshGateway.Datacenters() + keys := cfgSnap.MeshGateway.Keys() // 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) - clusters := make([]proto.Message, 0, len(datacenters)+len(cfgSnap.MeshGateway.ServiceGroups)) + clusters := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) // generate the remote dc clusters - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } opts := gatewayClusterOpts{ - name: connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain), - hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[dc], - isRemote: dc != cfgSnap.Datacenter, + name: connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()], + isRemote: key.Datacenter != cfgSnap.Datacenter, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) @@ -224,18 +224,18 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { // Add all of the remote wildcard datacenter mappings for servers. - for _, dc := range datacenters { - hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[dc] + for _, key := range keys { + hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[key.String()] // If the DC is our current DC then this cluster is for traffic from a remote DC to a local server. // HostnameDatacenters is populated with gateway addresses, so it does not apply here. - if dc == cfgSnap.Datacenter { + if key.Datacenter == cfgSnap.Datacenter { hostnameEndpoints = nil } opts := gatewayClusterOpts{ - name: cfgSnap.ServerSNIFn(dc, ""), + name: cfgSnap.ServerSNIFn(key.Datacenter, ""), hostnameEndpoints: hostnameEndpoints, - isRemote: dc != cfgSnap.Datacenter, + isRemote: key.Datacenter != cfgSnap.Datacenter, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 5ac1fd80c8..7d349c8759 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -109,28 +109,28 @@ func (s *ResourceGenerator) endpointsFromSnapshotTerminatingGateway(cfgSnap *pro } func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - datacenters := cfgSnap.MeshGateway.Datacenters() - resources := make([]proto.Message, 0, len(datacenters)+len(cfgSnap.MeshGateway.ServiceGroups)) + keys := cfgSnap.MeshGateway.Keys() + resources := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) // generate the endpoints for the gateways in the remote datacenters - for _, dc := range datacenters { + for _, key := range keys { // Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address // EDS cannot resolve hostnames so we provide them through CDS instead - if dc == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[dc]) > 0 { + if key.Datacenter == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { continue } - endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[dc] + endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[key.String()] if !ok { - endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[dc] + endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[key.String()] if !ok { // not possible - s.Logger.Error("skipping mesh gateway endpoints because no definition found", "datacenter", dc) + s.Logger.Error("skipping mesh gateway endpoints because no definition found", "datacenter", key) continue } } { // standard connect - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) + clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) la := makeLoadAssignment( clusterName, @@ -143,7 +143,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - clusterName := cfgSnap.ServerSNIFn(dc, "") + clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") la := makeLoadAssignment( clusterName, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 5abf7f125e..c7a7e00a0e 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1137,13 +1137,13 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, // TODO (mesh-gateway) - Do we need to create clusters for all the old trust domains as well? // We need 1 Filter Chain per datacenter - datacenters := cfgSnap.MeshGateway.Datacenters() - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + keys := cfgSnap.MeshGateway.Keys() + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) - filterName := fmt.Sprintf("%s.%s", name, dc) + clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) + filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") if err != nil { return nil, err @@ -1160,12 +1160,12 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } - clusterName := cfgSnap.ServerSNIFn(dc, "") - filterName := fmt.Sprintf("%s.%s", name, dc) + clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") + filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") if err != nil { return nil, err From 935112a47abe66077cac157f49359e96e1b5b26d Mon Sep 17 00:00:00 2001 From: freddygv Date: Sun, 24 Oct 2021 09:16:28 -0600 Subject: [PATCH 03/14] Account for partition in SNI for gateways --- agent/connect/sni.go | 13 +++++++++++-- agent/connect/sni_test.go | 36 +++++++++++++++++++++++++++++++++--- agent/xds/clusters.go | 2 +- agent/xds/endpoints.go | 2 +- agent/xds/listeners.go | 2 +- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/agent/connect/sni.go b/agent/connect/sni.go index 91f5008892..867c73b728 100644 --- a/agent/connect/sni.go +++ b/agent/connect/sni.go @@ -24,8 +24,17 @@ func UpstreamSNI(u *structs.Upstream, subset string, dc string, trustDomain stri return ServiceSNI(u.DestinationName, subset, u.DestinationNamespace, u.DestinationPartition, dc, trustDomain) } -func DatacenterSNI(dc string, trustDomain string) string { - return fmt.Sprintf("%s.internal.%s", dc, trustDomain) +func GatewaySNI(dc string, partition, trustDomain string) string { + if partition == "" { + partition = "default" + } + + switch partition { + case "default": + return dotJoin(dc, internal, trustDomain) + default: + return dotJoin(partition, dc, internalVersion, trustDomain) + } } func ServiceSNI(service string, subset string, namespace string, partition string, datacenter string, trustDomain string) string { diff --git a/agent/connect/sni_test.go b/agent/connect/sni_test.go index 64fc97d8a3..0ead8b485a 100644 --- a/agent/connect/sni_test.go +++ b/agent/connect/sni_test.go @@ -95,9 +95,39 @@ func TestUpstreamSNI(t *testing.T) { }) } -func TestDatacenterSNI(t *testing.T) { - require.Equal(t, "foo."+testTrustDomainSuffix1, DatacenterSNI("foo", testTrustDomain1)) - require.Equal(t, "bar."+testTrustDomainSuffix2, DatacenterSNI("bar", testTrustDomain2)) +func TestGatewaySNI(t *testing.T) { + type testCase struct { + name string + dc string + trustDomain string + expect string + } + + run := func(t *testing.T, tc testCase) { + got := GatewaySNI(tc.dc, "", tc.trustDomain) + require.Equal(t, tc.expect, got) + } + + cases := []testCase{ + { + name: "foo in domain1", + dc: "foo", + trustDomain: "domain1", + expect: "foo.internal.domain1", + }, + { + name: "bar in domain2", + dc: "bar", + trustDomain: "domain2", + expect: "bar.internal.domain2", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + run(t, c) + }) + } } func TestServiceSNI(t *testing.T) { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 3937da1705..bc11cc0f76 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -214,7 +214,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co } opts := gatewayClusterOpts{ - name: connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain), + name: connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain), hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()], isRemote: key.Datacenter != cfgSnap.Datacenter, } diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 7d349c8759..793d148b10 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -130,7 +130,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } { // standard connect - clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) + clusterName := connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain) la := makeLoadAssignment( clusterName, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index c7a7e00a0e..6b727ff6f3 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1142,7 +1142,7 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, if key.Datacenter == cfgSnap.Datacenter { continue // skip local } - clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) + clusterName := connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain) filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") if err != nil { From a33b6923e0c57c8fe34e324063cf00f36b926602 Mon Sep 17 00:00:00 2001 From: freddygv Date: Sun, 24 Oct 2021 09:51:55 -0600 Subject: [PATCH 04/14] Account for partitions in xds gen for mesh gw This commit avoids skipping gateways in remote partitions of the local DC when generating listeners/clusters/endpoints. --- agent/proxycfg/snapshot.go | 4 ++++ agent/xds/clusters.go | 2 +- agent/xds/endpoints.go | 8 +++++--- agent/xds/listeners.go | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index ac36d15a7d..3b0acaaefd 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -71,6 +71,10 @@ func (k GatewayKey) IsEmpty() bool { return k.Partition == "" && k.Datacenter == "" } +func (k GatewayKey) Matches(dc, partition string) bool { + return k.Partition == partition && k.Datacenter == dc +} + func gatewayKeyFromString(s string) GatewayKey { split := strings.SplitN(s, ".", 2) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index bc11cc0f76..b0788d324e 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -209,7 +209,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co // generate the remote dc clusters for _, key := range keys { - if key.Datacenter == cfgSnap.Datacenter { + if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) { continue // skip local } diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 793d148b10..6d36956c8d 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -114,9 +114,11 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C // generate the endpoints for the gateways in the remote datacenters for _, key := range keys { - // Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address - // EDS cannot resolve hostnames so we provide them through CDS instead - if key.Datacenter == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { + // Skip creating endpoints for mesh gateways in local DC/partition and gateways. + // Also skip gateways with a hostname as their address. + // EDS cannot resolve hostnames, so we provide them through CDS instead. + if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) || + len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { continue } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 6b727ff6f3..ab4a45b257 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1139,9 +1139,10 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, // We need 1 Filter Chain per datacenter keys := cfgSnap.MeshGateway.Keys() for _, key := range keys { - if key.Datacenter == cfgSnap.Datacenter { + if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) { continue // skip local } + clusterName := connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain) filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") From 954d21c6ba00c7d4a22df3301571df695a6c23cb Mon Sep 17 00:00:00 2001 From: freddygv Date: Sun, 24 Oct 2021 18:38:02 -0600 Subject: [PATCH 05/14] Register the ExportingPartitions cache type --- agent/agent.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index 68103decf0..bb0b3e832d 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -3786,6 +3786,8 @@ func (a *Agent) registerCache() { a.cache.RegisterType(cachetype.FederationStateListMeshGatewaysName, &cachetype.FederationStateListMeshGateways{RPC: a}) + + a.registerEntCache() } // LocalState returns the agent's local state From 5bf2497f71c688c37a5707b2e81d1d97ea3e28f6 Mon Sep 17 00:00:00 2001 From: freddygv Date: Mon, 25 Oct 2021 18:30:02 -0600 Subject: [PATCH 06/14] Add oss impl of registerEntCache --- agent/agent_oss.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/agent_oss.go b/agent/agent_oss.go index d266b2f524..fff8ef26e2 100644 --- a/agent/agent_oss.go +++ b/agent/agent_oss.go @@ -54,3 +54,5 @@ func (a *Agent) enterpriseStats() map[string]map[string]string { func (a *Agent) AgentEnterpriseMeta() *structs.EnterpriseMeta { return structs.NodeEnterpriseMetaInDefaultPartition() } + +func (a *Agent) registerEntCache() {} From 327e6bff25edf7d7468987de0ff3c0499abbfb54 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 15:42:30 -0600 Subject: [PATCH 07/14] Leave todo about default name --- agent/connect/sni.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agent/connect/sni.go b/agent/connect/sni.go index 867c73b728..2df3e0d11f 100644 --- a/agent/connect/sni.go +++ b/agent/connect/sni.go @@ -26,6 +26,7 @@ func UpstreamSNI(u *structs.Upstream, subset string, dc string, trustDomain stri func GatewaySNI(dc string, partition, trustDomain string) string { if partition == "" { + // TODO(partitions) Make default available in OSS as a constant for uses like this one partition = "default" } @@ -39,9 +40,10 @@ func GatewaySNI(dc string, partition, trustDomain string) string { func ServiceSNI(service string, subset string, namespace string, partition string, datacenter string, trustDomain string) string { if namespace == "" { - namespace = "default" + namespace = structs.IntentionDefaultNamespace } if partition == "" { + // TODO(partitions) Make default available in OSS as a constant for uses like this one partition = "default" } From 12923f5ebc4fb544a9c35a6455e118f256889bf9 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 15:58:23 -0600 Subject: [PATCH 08/14] PR comments --- agent/proxycfg/mesh_gateway.go | 4 +++- agent/proxycfg/snapshot.go | 2 +- agent/proxycfg/state.go | 1 - agent/xds/clusters.go | 8 ++++++-- agent/xds/endpoints.go | 20 ++++++++++++-------- agent/xds/listeners.go | 8 ++++++-- 6 files changed, 28 insertions(+), 15 deletions(-) diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index d5463e9187..22f510620b 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -13,6 +13,8 @@ import ( "github.com/hashicorp/consul/logging" ) +const exportingPartitionsWatchID = "exporting-partitions" + type handlerMeshGateway struct { handlerState } @@ -33,7 +35,7 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er wildcardEntMeta := s.proxyID.WithWildcardNamespace() // Watch for all services. - // Eventually we will have to watch connect enable instances for each service as well as the + // Eventually we will have to watch connect enabled instances for each service as well as the // destination services themselves but those notifications will be setup later. // We cannot setup those watches until we know what the services are. err = s.cache.Notify(ctx, cachetype.CatalogServiceListName, &structs.DCSpecificRequest{ diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 3b0acaaefd..dce76f8c31 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -294,7 +294,7 @@ type configSnapshotMeshGateway struct { HostnameDatacenters map[string]structs.CheckServiceNodes } -func (c *configSnapshotMeshGateway) Keys() []GatewayKey { +func (c *configSnapshotMeshGateway) GatewayKeys() []GatewayKey { sz1, sz2 := len(c.GatewayGroups), len(c.FedStateGateways) sz := sz1 diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index cb46764f30..2c443f6be6 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -37,7 +37,6 @@ const ( datacentersWatchID = "datacenters" serviceResolversWatchID = "service-resolvers" gatewayServicesWatchID = "gateway-services" - exportingPartitionsWatchID = "exporting-partitions" gatewayConfigWatchID = "gateway-config" externalServiceIDPrefix = "external-service:" serviceLeafIDPrefix = "service-leaf:" diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index b0788d324e..9e7477c8d7 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -19,6 +19,7 @@ import ( "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/wrappers" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" @@ -202,7 +203,7 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - keys := cfgSnap.MeshGateway.Keys() + keys := cfgSnap.MeshGateway.GatewayKeys() // 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) clusters := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) @@ -222,7 +223,10 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co clusters = append(clusters, cluster) } - if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { + if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && + cfgSnap.ServerSNIFn != nil { + // Add all of the remote wildcard datacenter mappings for servers. for _, key := range keys { hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[key.String()] diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 6d36956c8d..e40f4a71c4 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -10,6 +10,7 @@ import ( "github.com/golang/protobuf/proto" bexpr "github.com/hashicorp/go-bexpr" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" @@ -109,14 +110,13 @@ func (s *ResourceGenerator) endpointsFromSnapshotTerminatingGateway(cfgSnap *pro } func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - keys := cfgSnap.MeshGateway.Keys() + keys := cfgSnap.MeshGateway.GatewayKeys() resources := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) - // generate the endpoints for the gateways in the remote datacenters for _, key := range keys { - // Skip creating endpoints for mesh gateways in local DC/partition and gateways. - // Also skip gateways with a hostname as their address. - // EDS cannot resolve hostnames, so we provide them through CDS instead. + // Skip creating endpoints for mesh gateways in local DC/partition. + // Also skip gateways with a hostname as their address. EDS cannot resolve hostnames, + // so we provide them through CDS instead. if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) || len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { continue @@ -144,9 +144,11 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C resources = append(resources, la) } - if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") + if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && + cfgSnap.ServerSNIFn != nil { + clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") la := makeLoadAssignment( clusterName, []loadAssignmentEndpointGroup{ @@ -159,7 +161,9 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } // generate endpoints for our servers if WAN federation is enabled - if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { + if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && + cfgSnap.ServerSNIFn != nil { var allServersLbEndpoints []*envoy_endpoint_v3.LbEndpoint for _, srv := range cfgSnap.MeshGateway.ConsulServers { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index ab4a45b257..1f9e7f1f57 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect/ca" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -1137,7 +1138,7 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, // TODO (mesh-gateway) - Do we need to create clusters for all the old trust domains as well? // We need 1 Filter Chain per datacenter - keys := cfgSnap.MeshGateway.Keys() + keys := cfgSnap.MeshGateway.GatewayKeys() for _, key := range keys { if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) { continue // skip local @@ -1160,7 +1161,10 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, }) } - if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { + if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && + cfgSnap.ServerSNIFn != nil { + for _, key := range keys { if key.Datacenter == cfgSnap.Datacenter { continue // skip local From 448701dbd831d970de804cfbe2ff5f4c8ed44be5 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 16:10:30 -0600 Subject: [PATCH 09/14] Replace default partition check --- agent/proxycfg/mesh_gateway.go | 3 +-- agent/xds/clusters.go | 2 +- agent/xds/endpoints.go | 6 +++--- agent/xds/listeners.go | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index 22f510620b..6c3457a29f 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/structs" @@ -62,7 +61,7 @@ func (s *handlerMeshGateway) initialize(ctx context.Context) (ConfigSnapshot, er return snap, err } - if s.proxyID.PartitionOrEmpty() == acl.DefaultPartitionName { + if s.proxyID.InDefaultPartition() { if err := s.initializeCrossDCWatches(ctx); err != nil { return snap, err } diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 9e7477c8d7..35a2be8230 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -223,7 +223,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co clusters = append(clusters, cluster) } - if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + if cfgSnap.ProxyID.InDefaultPartition() && cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index e40f4a71c4..3cc2c4dd16 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -3,6 +3,7 @@ package xds import ( "errors" "fmt" + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" @@ -10,7 +11,6 @@ import ( "github.com/golang/protobuf/proto" bexpr "github.com/hashicorp/go-bexpr" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" @@ -144,7 +144,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C resources = append(resources, la) } - if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + if cfgSnap.ProxyID.InDefaultPartition() && cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { @@ -161,7 +161,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } // generate endpoints for our servers if WAN federation is enabled - if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + if cfgSnap.ProxyID.InDefaultPartition() && cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { var allServersLbEndpoints []*envoy_endpoint_v3.LbEndpoint diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 1f9e7f1f57..47f7cbeb10 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect/ca" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -1161,7 +1160,7 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, }) } - if cfgSnap.ProxyID.PartitionOrEmpty() == acl.DefaultPartitionName && + if cfgSnap.ProxyID.InDefaultPartition() && cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { From d28b9052b26a1381951e7e528ace40e6dd418891 Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 16:22:04 -0600 Subject: [PATCH 10/14] Move the exportingpartitions constant to enterprise --- agent/proxycfg/mesh_gateway.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index 6c3457a29f..39f8d9d109 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -12,8 +12,6 @@ import ( "github.com/hashicorp/consul/logging" ) -const exportingPartitionsWatchID = "exporting-partitions" - type handlerMeshGateway struct { handlerState } From c72bbb6e8d81294b1533870306fba88dc3222f7e Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 16:25:35 -0600 Subject: [PATCH 11/14] Split up locality check from hostname check --- agent/xds/endpoints.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 3cc2c4dd16..073a7ac320 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -114,11 +114,12 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C resources := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) for _, key := range keys { - // Skip creating endpoints for mesh gateways in local DC/partition. + if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) { + continue // skip local + } // Also skip gateways with a hostname as their address. EDS cannot resolve hostnames, // so we provide them through CDS instead. - if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) || - len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { + if len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { continue } From 9480670b728dc9b35dc077f1727829504d9ddb3e Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 26 Oct 2021 16:31:51 -0600 Subject: [PATCH 12/14] Fixup imports --- agent/xds/clusters.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 35a2be8230..3069aed88c 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -19,7 +19,6 @@ import ( "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/wrappers" - "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" From 3a2061544dda7f0315e8cedd3d766089167161e3 Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 27 Oct 2021 09:49:47 -0600 Subject: [PATCH 13/14] Fixup partitions assertion --- agent/proxycfg/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index dce76f8c31..6d69ab523d 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -72,7 +72,7 @@ func (k GatewayKey) IsEmpty() bool { } func (k GatewayKey) Matches(dc, partition string) bool { - return k.Partition == partition && k.Datacenter == dc + return structs.EqualPartitions(k.Partition, partition) && k.Datacenter == dc } func gatewayKeyFromString(s string) GatewayKey { From e93c144d2f3d5884ca0d89a86cc2d0cbb05b2adb Mon Sep 17 00:00:00 2001 From: freddygv Date: Wed, 27 Oct 2021 12:36:44 -0600 Subject: [PATCH 14/14] Update comments --- agent/xds/clusters.go | 4 ++-- agent/xds/listeners.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 3069aed88c..0bcd66fb93 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -204,10 +204,10 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { keys := cfgSnap.MeshGateway.GatewayKeys() - // 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) + // 1 cluster per remote dc/partition + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) clusters := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) - // generate the remote dc clusters + // Generate the remote clusters for _, key := range keys { if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) { continue // skip local diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 47f7cbeb10..276ea58f75 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1135,8 +1135,7 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, l := makePortListener(name, addr, port, envoy_core_v3.TrafficDirection_UNSPECIFIED) l.ListenerFilters = []*envoy_listener_v3.ListenerFilter{tlsInspector} - // TODO (mesh-gateway) - Do we need to create clusters for all the old trust domains as well? - // We need 1 Filter Chain per datacenter + // We need 1 Filter Chain per remote cluster keys := cfgSnap.MeshGateway.GatewayKeys() for _, key := range keys { if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrEmpty()) {