From 90ce8974565a68d57faea65ca49cba9f3891c6d7 Mon Sep 17 00:00:00 2001 From: freddygv Date: Thu, 28 Oct 2021 18:47:42 -0600 Subject: [PATCH] Store GatewayKey in proxycfg snapshot for re-use --- agent/proxycfg/manager_test.go | 2 + agent/proxycfg/mesh_gateway.go | 4 +- agent/proxycfg/snapshot.go | 1 + agent/proxycfg/state.go | 1 + agent/proxycfg/terminating_gateway.go | 2 +- agent/proxycfg/testing.go | 66 +++++++++++++++------------ agent/xds/clusters.go | 3 +- agent/xds/endpoints.go | 10 ++-- 8 files changed, 50 insertions(+), 39 deletions(-) diff --git a/agent/proxycfg/manager_test.go b/agent/proxycfg/manager_test.go index 0198289364..6410dc852f 100644 --- a/agent/proxycfg/manager_test.go +++ b/agent/proxycfg/manager_test.go @@ -239,6 +239,7 @@ func TestManager_BasicLifecycle(t *testing.T) { IntentionsSet: true, }, Datacenter: "dc1", + Locality: GatewayKey{Datacenter: "dc1", Partition: structs.PartitionOrDefault("")}, }, }, { @@ -296,6 +297,7 @@ func TestManager_BasicLifecycle(t *testing.T) { IntentionsSet: true, }, Datacenter: "dc1", + Locality: GatewayKey{Datacenter: "dc1", Partition: structs.PartitionOrDefault("")}, }, }, } diff --git a/agent/proxycfg/mesh_gateway.go b/agent/proxycfg/mesh_gateway.go index dffca3e9a5..3bcce18e7a 100644 --- a/agent/proxycfg/mesh_gateway.go +++ b/agent/proxycfg/mesh_gateway.go @@ -144,7 +144,7 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve for dc, nodes := range dcIndexedNodes.DatacenterNodes { snap.MeshGateway.HostnameDatacenters[dc] = hostnameEndpoints( s.logger.Named(logging.MeshGateway), - GatewayKey{Partition: snap.ProxyID.PartitionOrDefault(), Datacenter: snap.Datacenter}, + snap.Locality, nodes, ) } @@ -327,7 +327,7 @@ func (s *handlerMeshGateway) handleUpdate(ctx context.Context, u cache.UpdateEve snap.MeshGateway.GatewayGroups[key] = resp.Nodes snap.MeshGateway.HostnameDatacenters[key] = hostnameEndpoints( s.logger.Named(logging.MeshGateway), - GatewayKey{Partition: snap.ProxyID.PartitionOrDefault(), Datacenter: snap.Datacenter}, + snap.Locality, resp.Nodes, ) } diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index 6d69ab523d..e318120e7a 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -408,6 +408,7 @@ type ConfigSnapshot struct { Proxy structs.ConnectProxyConfig Datacenter string IntentionDefaultAllow bool + Locality GatewayKey ServerSNIFn ServerSNIFunc Roots *structs.IndexedCARoots diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 31a380b1c3..cd30003531 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -254,6 +254,7 @@ func newConfigSnapshotFromServiceInstance(s serviceInstance, config stateConfig) TaggedAddresses: s.taggedAddresses, Proxy: s.proxyCfg, Datacenter: config.source.Datacenter, + Locality: GatewayKey{Datacenter: config.source.Datacenter, Partition: s.proxyID.PartitionOrDefault()}, ServerSNIFn: config.serverSNIFn, IntentionDefaultAllow: config.intentionDefaultAllow, } diff --git a/agent/proxycfg/terminating_gateway.go b/agent/proxycfg/terminating_gateway.go index c185ed013b..b08985b293 100644 --- a/agent/proxycfg/terminating_gateway.go +++ b/agent/proxycfg/terminating_gateway.go @@ -286,7 +286,7 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u cache.Up snap.TerminatingGateway.ServiceGroups[sn] = resp.Nodes snap.TerminatingGateway.HostnameServices[sn] = hostnameEndpoints( s.logger, - GatewayKey{Partition: snap.ProxyID.PartitionOrDefault(), Datacenter: snap.Datacenter}, + snap.Locality, resp.Nodes, ) } diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index e8f40c3901..870d8e7a1d 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" cachetype "github.com/hashicorp/consul/agent/cache-types" "github.com/hashicorp/consul/agent/connect" @@ -674,11 +675,12 @@ func TestConfigSnapshot(t testing.T) *ConfigSnapshot { upstreams := structs.TestUpstreams(t) return &ConfigSnapshot{ - Kind: structs.ServiceKindConnectProxy, - Service: "web-sidecar-proxy", - ProxyID: structs.NewServiceID("web-sidecar-proxy", nil), - Address: "0.0.0.0", - Port: 9999, + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindConnectProxy, + Service: "web-sidecar-proxy", + ProxyID: structs.NewServiceID("web-sidecar-proxy", nil), + Address: "0.0.0.0", + Port: 9999, Proxy: structs.ConnectProxyConfig{ DestinationServiceID: "web", DestinationServiceName: "web", @@ -798,11 +800,12 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE roots, leaf := TestCerts(t) snap := &ConfigSnapshot{ - Kind: structs.ServiceKindConnectProxy, - Service: "web-sidecar-proxy", - ProxyID: structs.NewServiceID("web-sidecar-proxy", nil), - Address: "0.0.0.0", - Port: 9999, + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindConnectProxy, + Service: "web-sidecar-proxy", + ProxyID: structs.NewServiceID("web-sidecar-proxy", nil), + Address: "0.0.0.0", + Port: 9999, Proxy: structs.ConnectProxyConfig{ DestinationServiceID: "web", DestinationServiceName: "web", @@ -1510,11 +1513,12 @@ func TestConfigSnapshotMeshGatewayNoServices(t testing.T) *ConfigSnapshot { func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFederationStates bool) *ConfigSnapshot { roots, _ := TestCerts(t) snap := &ConfigSnapshot{ - Kind: structs.ServiceKindMeshGateway, - Service: "mesh-gateway", - ProxyID: structs.NewServiceID("mesh-gateway", nil), - Address: "1.2.3.4", - Port: 8443, + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindMeshGateway, + Service: "mesh-gateway", + ProxyID: structs.NewServiceID("mesh-gateway", nil), + Address: "1.2.3.4", + Port: 8443, Proxy: structs.ConnectProxyConfig{ Config: map[string]interface{}{}, }, @@ -1721,6 +1725,7 @@ func testConfigSnapshotIngressGateway( roots, leaf := TestCerts(t) snap := &ConfigSnapshot{ + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, Kind: structs.ServiceKindIngressGateway, Service: "ingress-gateway", ProxyID: structs.NewServiceID("ingress-gateway", nil), @@ -1760,11 +1765,12 @@ func testConfigSnapshotIngressGateway( func TestConfigSnapshotExposeConfig(t testing.T) *ConfigSnapshot { return &ConfigSnapshot{ - Kind: structs.ServiceKindConnectProxy, - Service: "web-proxy", - ProxyID: structs.NewServiceID("web-proxy", nil), - Address: "1.2.3.4", - Port: 8080, + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindConnectProxy, + Service: "web-proxy", + ProxyID: structs.NewServiceID("web-proxy", nil), + Address: "1.2.3.4", + Port: 8080, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "web", DestinationServiceID: "web", @@ -1801,10 +1807,11 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C roots, _ := TestCerts(t) snap := &ConfigSnapshot{ - Kind: structs.ServiceKindTerminatingGateway, - Service: "terminating-gateway", - ProxyID: structs.NewServiceID("terminating-gateway", nil), - Address: "1.2.3.4", + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindTerminatingGateway, + Service: "terminating-gateway", + ProxyID: structs.NewServiceID("terminating-gateway", nil), + Address: "1.2.3.4", TaggedAddresses: map[string]structs.ServiceAddress{ structs.TaggedAddressWAN: { Address: "198.18.0.1", @@ -2035,11 +2042,12 @@ func testConfigSnapshotTerminatingGateway(t testing.T, populateServices bool) *C func TestConfigSnapshotGRPCExposeHTTP1(t testing.T) *ConfigSnapshot { return &ConfigSnapshot{ - Kind: structs.ServiceKindConnectProxy, - Service: "grpc-proxy", - ProxyID: structs.NewServiceID("grpc-proxy", nil), - Address: "1.2.3.4", - Port: 8080, + Locality: GatewayKey{Datacenter: "dc1", Partition: acl.DefaultPartitionName}, + Kind: structs.ServiceKindConnectProxy, + Service: "grpc-proxy", + ProxyID: structs.NewServiceID("grpc-proxy", nil), + Address: "1.2.3.4", + Port: 8080, Proxy: structs.ConnectProxyConfig{ DestinationServiceName: "grpc", DestinationServiceID: "grpc", diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 0910eed769..441b587355 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -299,10 +299,9 @@ func (s *ResourceGenerator) makeGatewayServiceClusters( hostnameEndpoints = cfgSnap.TerminatingGateway.HostnameServices[svc] } - localKey := proxycfg.GatewayKey{Partition: cfgSnap.ProxyID.PartitionOrDefault(), Datacenter: cfgSnap.Datacenter} var isRemote bool if len(services[svc]) > 0 { - isRemote = !localKey.Matches(services[svc][0].Node.Datacenter, services[svc][0].Node.PartitionOrDefault()) + isRemote = !cfgSnap.Locality.Matches(services[svc][0].Node.Datacenter, services[svc][0].Node.PartitionOrDefault()) } opts := gatewayClusterOpts{ diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 9c9a02b2f4..57c014995c 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -51,7 +51,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg. es := s.endpointsFromDiscoveryChain( id, chain, - proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: cfgSnap.ProxyID.PartitionOrDefault()}, + cfgSnap.Locality, cfgSnap.ConnectProxy.UpstreamConfig[id], cfgSnap.ConnectProxy.WatchedUpstreamEndpoints[id], cfgSnap.ConnectProxy.WatchedGatewayEndpoints[id], @@ -79,7 +79,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg. []loadAssignmentEndpointGroup{ {Endpoints: endpoints}, }, - proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: cfgSnap.ProxyID.PartitionOrDefault()}, + cfgSnap.Locality, ) resources = append(resources, la) } @@ -140,7 +140,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C []loadAssignmentEndpointGroup{ {Endpoints: endpoints}, }, - proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: cfgSnap.ProxyID.PartitionOrDefault()}, + cfgSnap.Locality, ) resources = append(resources, la) } @@ -155,7 +155,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C []loadAssignmentEndpointGroup{ {Endpoints: endpoints}, }, - proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: cfgSnap.ProxyID.PartitionOrDefault()}, + cfgSnap.Locality, ) resources = append(resources, la) } @@ -256,7 +256,7 @@ func (s *ResourceGenerator) endpointsFromServicesAndResolvers( la := makeLoadAssignment( clusterName, groups, - proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: cfgSnap.ProxyID.PartitionOrDefault()}, + cfgSnap.Locality, ) resources = append(resources, la) }