From 110fae820a27871d2b8214246547d93fe06f8f80 Mon Sep 17 00:00:00 2001 From: freddygv Date: Sat, 23 Oct 2021 14:17:29 -0600 Subject: [PATCH] Update xds pkg to account for GatewayKey --- agent/proxycfg/snapshot.go | 36 +++++++++++++++++++++++------------- agent/xds/clusters.go | 24 ++++++++++++------------ agent/xds/endpoints.go | 18 +++++++++--------- agent/xds/listeners.go | 18 +++++++++--------- 4 files changed, 53 insertions(+), 43 deletions(-) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 3c915a5698..ac36d15a7d 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -60,7 +60,11 @@ type GatewayKey struct { } func (k GatewayKey) String() string { - return k.Partition + "." + k.Datacenter + resp := k.Datacenter + if k.Partition != "" { + resp = k.Partition + "." + resp + } + return resp } func (k GatewayKey) IsEmpty() bool { @@ -69,10 +73,11 @@ func (k GatewayKey) IsEmpty() bool { func gatewayKeyFromString(s string) GatewayKey { split := strings.SplitN(s, ".", 2) - return GatewayKey{ - Partition: split[0], - Datacenter: split[1], + + if len(split) == 1 { + return GatewayKey{Datacenter: split[0]} } + return GatewayKey{Partition: split[0], Datacenter: split[1]} } // ServicePassthroughAddrs contains the LAN addrs @@ -285,7 +290,7 @@ type configSnapshotMeshGateway struct { HostnameDatacenters map[string]structs.CheckServiceNodes } -func (c *configSnapshotMeshGateway) Datacenters() []string { +func (c *configSnapshotMeshGateway) Keys() []GatewayKey { sz1, sz2 := len(c.GatewayGroups), len(c.FedStateGateways) sz := sz1 @@ -293,20 +298,25 @@ func (c *configSnapshotMeshGateway) Datacenters() []string { sz = sz2 } - dcs := make([]string, 0, sz) - for dc := range c.GatewayGroups { - dcs = append(dcs, dc) + keys := make([]GatewayKey, 0, sz) + for key := range c.GatewayGroups { + keys = append(keys, gatewayKeyFromString(key)) } - for dc := range c.FedStateGateways { - if _, ok := c.GatewayGroups[dc]; !ok { - dcs = append(dcs, dc) + for key := range c.FedStateGateways { + if _, ok := c.GatewayGroups[key]; !ok { + keys = append(keys, gatewayKeyFromString(key)) } } // Always sort the results to ensure we generate deterministic things over // xDS, such as mesh-gateway listener filter chains. - sort.Strings(dcs) - return dcs + sort.Slice(keys, func(i, j int) bool { + if keys[i].Datacenter != keys[j].Datacenter { + return keys[i].Datacenter < keys[j].Datacenter + } + return keys[i].Partition < keys[j].Partition + }) + return keys } func (c *configSnapshotMeshGateway) IsEmpty() bool { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index eac2ac561f..3937da1705 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -202,21 +202,21 @@ func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - datacenters := cfgSnap.MeshGateway.Datacenters() + keys := cfgSnap.MeshGateway.Keys() // 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended) - clusters := make([]proto.Message, 0, len(datacenters)+len(cfgSnap.MeshGateway.ServiceGroups)) + clusters := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) // generate the remote dc clusters - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } opts := gatewayClusterOpts{ - name: connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain), - hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[dc], - isRemote: dc != cfgSnap.Datacenter, + name: connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain), + hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()], + isRemote: key.Datacenter != cfgSnap.Datacenter, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) @@ -224,18 +224,18 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { // Add all of the remote wildcard datacenter mappings for servers. - for _, dc := range datacenters { - hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[dc] + for _, key := range keys { + hostnameEndpoints := cfgSnap.MeshGateway.HostnameDatacenters[key.String()] // If the DC is our current DC then this cluster is for traffic from a remote DC to a local server. // HostnameDatacenters is populated with gateway addresses, so it does not apply here. - if dc == cfgSnap.Datacenter { + if key.Datacenter == cfgSnap.Datacenter { hostnameEndpoints = nil } opts := gatewayClusterOpts{ - name: cfgSnap.ServerSNIFn(dc, ""), + name: cfgSnap.ServerSNIFn(key.Datacenter, ""), hostnameEndpoints: hostnameEndpoints, - isRemote: dc != cfgSnap.Datacenter, + isRemote: key.Datacenter != cfgSnap.Datacenter, } cluster := s.makeGatewayCluster(cfgSnap, opts) clusters = append(clusters, cluster) diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 5ac1fd80c8..7d349c8759 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -109,28 +109,28 @@ func (s *ResourceGenerator) endpointsFromSnapshotTerminatingGateway(cfgSnap *pro } func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { - datacenters := cfgSnap.MeshGateway.Datacenters() - resources := make([]proto.Message, 0, len(datacenters)+len(cfgSnap.MeshGateway.ServiceGroups)) + keys := cfgSnap.MeshGateway.Keys() + resources := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups)) // generate the endpoints for the gateways in the remote datacenters - for _, dc := range datacenters { + for _, key := range keys { // Skip creating endpoints for mesh gateways in local DC and gateways in remote DCs with a hostname as their address // EDS cannot resolve hostnames so we provide them through CDS instead - if dc == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[dc]) > 0 { + if key.Datacenter == cfgSnap.Datacenter || len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 { continue } - endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[dc] + endpoints, ok := cfgSnap.MeshGateway.GatewayGroups[key.String()] if !ok { - endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[dc] + endpoints, ok = cfgSnap.MeshGateway.FedStateGateways[key.String()] if !ok { // not possible - s.Logger.Error("skipping mesh gateway endpoints because no definition found", "datacenter", dc) + s.Logger.Error("skipping mesh gateway endpoints because no definition found", "datacenter", key) continue } } { // standard connect - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) + clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) la := makeLoadAssignment( clusterName, @@ -143,7 +143,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - clusterName := cfgSnap.ServerSNIFn(dc, "") + clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") la := makeLoadAssignment( clusterName, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 5abf7f125e..c7a7e00a0e 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -1137,13 +1137,13 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, // TODO (mesh-gateway) - Do we need to create clusters for all the old trust domains as well? // We need 1 Filter Chain per datacenter - datacenters := cfgSnap.MeshGateway.Datacenters() - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + keys := cfgSnap.MeshGateway.Keys() + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } - clusterName := connect.DatacenterSNI(dc, cfgSnap.Roots.TrustDomain) - filterName := fmt.Sprintf("%s.%s", name, dc) + clusterName := connect.DatacenterSNI(key.Datacenter, cfgSnap.Roots.TrustDomain) + filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") if err != nil { return nil, err @@ -1160,12 +1160,12 @@ func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, } if cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" && cfgSnap.ServerSNIFn != nil { - for _, dc := range datacenters { - if dc == cfgSnap.Datacenter { + for _, key := range keys { + if key.Datacenter == cfgSnap.Datacenter { continue // skip local } - clusterName := cfgSnap.ServerSNIFn(dc, "") - filterName := fmt.Sprintf("%s.%s", name, dc) + clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "") + filterName := fmt.Sprintf("%s.%s", name, key.String()) dcTCPProxy, err := makeTCPProxyFilter(filterName, clusterName, "mesh_gateway_remote.") if err != nil { return nil, err