From 74e4200d07bc1b7ace9d027a4721c18922d6dd7d Mon Sep 17 00:00:00 2001 From: Nathan Coleman Date: Wed, 31 Jan 2024 10:46:04 -0500 Subject: [PATCH] =?UTF-8?q?[NET-6429]=20Program=20ProxyStateTemplate=20to?= =?UTF-8?q?=20route=20cross-partition=20traffi=E2=80=A6=20(#20410)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [NET-6429] Program ProxyStateTemplate to route cross-partition traffic to the correct destination mesh gateway * Program mesh port to route wildcarded gateway SNI to the appropriate remote partition's mesh gateway * Update target + route ports in service endpoint refs when building PST * Use proper name of local datacenter when constructing SNI for gateway target * Use destination identities for TLS when routing L4 traffic through the mesh gateway * Use new constants, move comment to correct location * Use new constants for port names * Update test assertions * Undo debug logging change --- .../gatewayproxy/builder/builder.go | 203 +++++++++++++++--- .../gatewayproxy/builder/builder_test.go | 52 +---- .../controllers/gatewayproxy/controller.go | 17 +- .../gatewayproxy/fetcher/data_fetcher.go | 17 ++ .../sidecarproxy/fetcher/data_fetcher.go | 66 +++--- 5 files changed, 254 insertions(+), 101 deletions(-) diff --git a/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go b/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go index 2772888f43..7f770a7de4 100644 --- a/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go +++ b/internal/mesh/internal/controllers/gatewayproxy/builder/builder.go @@ -12,8 +12,8 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/hashicorp/consul/agent/connect" - "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/internal/mesh/internal/controllers/gatewayproxy/fetcher" + "github.com/hashicorp/consul/internal/mesh/internal/controllers/meshgateways" "github.com/hashicorp/consul/internal/mesh/internal/types" "github.com/hashicorp/consul/internal/resource" pbauth "github.com/hashicorp/consul/proto-public/pbauth/v2beta1" @@ -24,7 +24,9 @@ import ( "github.com/hashicorp/consul/proto-public/pbresource" ) -const nullRouteClusterName = "null_route_cluster" +const ( + nullRouteClusterName = "null_route_cluster" +) type proxyStateTemplateBuilder struct { workload *types.DecodedWorkload @@ -33,9 +35,10 @@ type proxyStateTemplateBuilder struct { exportedServices []*pbmulticluster.ComputedExportedService logger hclog.Logger trustDomain string + remoteGatewayIDs []*pbresource.ID } -func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload, exportedServices []*pbmulticluster.ComputedExportedService, logger hclog.Logger, dataFetcher *fetcher.Fetcher, dc, trustDomain string) *proxyStateTemplateBuilder { +func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload, exportedServices []*pbmulticluster.ComputedExportedService, logger hclog.Logger, dataFetcher *fetcher.Fetcher, dc, trustDomain string, remoteGatewayIDs []*pbresource.ID) *proxyStateTemplateBuilder { return &proxyStateTemplateBuilder{ workload: workload, dataFetcher: dataFetcher, @@ -43,6 +46,7 @@ func NewProxyStateTemplateBuilder(workload *types.DecodedWorkload, exportedServi exportedServices: exportedServices, logger: logger, trustDomain: trustDomain, + remoteGatewayIDs: remoteGatewayIDs, } } @@ -67,8 +71,15 @@ func (b *proxyStateTemplateBuilder) listeners() []*pbproxystate.Listener { // if the address defines no ports we assume the intention is to bind to all // ports on the workload if len(address.Ports) == 0 { - for _, workloadPort := range b.workload.Data.Ports { - listeners = append(listeners, b.buildListener(address, workloadPort.Port)) + for portName, workloadPort := range b.workload.Data.Ports { + switch portName { + case meshgateways.LANPortName: + listeners = append(listeners, b.meshListener(address, workloadPort.Port)) + case meshgateways.WANPortName: + listeners = append(listeners, b.wanListener(address, workloadPort.Port)) + default: + b.logger.Warn("encountered unexpected port on mesh gateway workload", "port", portName) + } } return listeners } @@ -80,16 +91,38 @@ func (b *proxyStateTemplateBuilder) listeners() []*pbproxystate.Listener { continue } - listeners = append(listeners, b.buildListener(address, workloadPort.Port)) + switch portName { + case meshgateways.LANPortName: + listeners = append(listeners, b.meshListener(address, workloadPort.Port)) + case meshgateways.WANPortName: + listeners = append(listeners, b.wanListener(address, workloadPort.Port)) + default: + b.logger.Warn("encountered unexpected port on mesh gateway workload", "port", portName) + } } return listeners } -func (b *proxyStateTemplateBuilder) buildListener(address *pbcatalog.WorkloadAddress, port uint32) *pbproxystate.Listener { +// meshListener constructs a pbproxystate.Listener that receives outgoing +// traffic from the local partition where the mesh gateway mode is "local". This +// traffic will be sent to a mesh gateway in a remote partition. +func (b *proxyStateTemplateBuilder) meshListener(address *pbcatalog.WorkloadAddress, port uint32) *pbproxystate.Listener { + return b.listener("mesh_listener", address, port, pbproxystate.Direction_DIRECTION_OUTBOUND, b.meshRouters()) +} + +// wanListener constructs a pbproxystate.Listener that receives incoming +// traffic from the public internet, either from a mesh gateway in a remote partition +// where the mesh gateway mode is "local" or from a service in a remote partition +// where the mesh gateway mode is "remote". +func (b *proxyStateTemplateBuilder) wanListener(address *pbcatalog.WorkloadAddress, port uint32) *pbproxystate.Listener { + return b.listener("wan_listener", address, port, pbproxystate.Direction_DIRECTION_INBOUND, b.wanRouters()) +} + +func (b *proxyStateTemplateBuilder) listener(name string, address *pbcatalog.WorkloadAddress, port uint32, direction pbproxystate.Direction, routers []*pbproxystate.Router) *pbproxystate.Listener { return &pbproxystate.Listener{ - Name: xdscommon.PublicListenerName, - Direction: pbproxystate.Direction_DIRECTION_INBOUND, + Name: name, + Direction: direction, BindAddress: &pbproxystate.Listener_HostPort{ HostPort: &pbproxystate.HostPortAddress{ Host: address.Host, @@ -111,14 +144,55 @@ func (b *proxyStateTemplateBuilder) buildListener(address *pbcatalog.WorkloadAdd }, }, }, - Routers: b.routers(), + Routers: routers, } } -// routers loops through the ports and consumers for each exported service and generates -// a pbproxystate.Router matching the SNI to the target cluster. The target port name -// will be included in the ALPN. The targeted cluster will marry this port name with the SNI. -func (b *proxyStateTemplateBuilder) routers() []*pbproxystate.Router { +// meshRouters loops through the list of mesh gateways in other partitions and generates +// a pbproxystate.Router matching the partition + datacenter of the SNI to the target +// cluster. Traffic flowing through this router originates in the local partition where +// the mesh gateway mode is "local". +func (b *proxyStateTemplateBuilder) meshRouters() []*pbproxystate.Router { + var routers []*pbproxystate.Router + + for _, remoteGatewayID := range b.remoteGatewayIDs { + serviceID := resource.ReplaceType(pbcatalog.ServiceType, remoteGatewayID) + service, err := b.dataFetcher.FetchService(context.Background(), serviceID) + if err != nil { + b.logger.Trace("error reading exported service", "error", err) + continue + } else if service == nil { + b.logger.Trace("service does not exist, skipping router", "service", serviceID) + continue + } + + routers = append(routers, &pbproxystate.Router{ + Match: &pbproxystate.Match{ + ServerNames: []string{ + fmt.Sprintf("*.%s", b.clusterNameForRemoteGateway(remoteGatewayID)), + }, + }, + Destination: &pbproxystate.Router_L4{ + L4: &pbproxystate.L4Destination{ + Destination: &pbproxystate.L4Destination_Cluster{ + Cluster: &pbproxystate.DestinationCluster{ + Name: b.clusterNameForRemoteGateway(remoteGatewayID), + }, + }, + StatPrefix: "prefix", + }, + }, + }) + } + + return routers +} + +// wanRouters loops through the ports and consumers for each exported service and generates +// a pbproxystate.Router matching the SNI to the target cluster. Traffic flowing through this +// router originates from a mesh gateway in a remote partition where the mesh gateway mode is +// "local" or from a service in a remote partition where the mesh gateway mode is "remote". +func (b *proxyStateTemplateBuilder) wanRouters() []*pbproxystate.Router { var routers []*pbproxystate.Router for _, exportedService := range b.exportedServices { @@ -133,17 +207,21 @@ func (b *proxyStateTemplateBuilder) routers() []*pbproxystate.Router { } for _, port := range service.Data.Ports { + if port.Protocol == pbcatalog.Protocol_PROTOCOL_MESH { + continue + } + for _, consumer := range exportedService.Consumers { routers = append(routers, &pbproxystate.Router{ Match: &pbproxystate.Match{ AlpnProtocols: []string{alpnProtocol(port.TargetPort)}, - ServerNames: []string{b.sni(exportedService.TargetRef, consumer)}, + ServerNames: []string{b.sniForExportedService(exportedService.TargetRef, consumer)}, }, Destination: &pbproxystate.Router_L4{ L4: &pbproxystate.L4Destination{ Destination: &pbproxystate.L4Destination_Cluster{ Cluster: &pbproxystate.DestinationCluster{ - Name: b.clusterName(exportedService.TargetRef, consumer, port.TargetPort), + Name: b.clusterNameForExportedService(exportedService.TargetRef, consumer, port.TargetPort), }, }, StatPrefix: "prefix", @@ -160,6 +238,7 @@ func (b *proxyStateTemplateBuilder) routers() []*pbproxystate.Router { func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster { clusters := map[string]*pbproxystate.Cluster{} + // Clusters handling incoming traffic from a remote partition for _, exportedService := range b.exportedServices { serviceID := resource.IDFromReference(exportedService.TargetRef) service, err := b.dataFetcher.FetchService(context.Background(), serviceID) @@ -172,8 +251,12 @@ func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster } for _, port := range service.Data.Ports { + if port.Protocol == pbcatalog.Protocol_PROTOCOL_MESH { + continue + } + for _, consumer := range exportedService.Consumers { - clusterName := b.clusterName(exportedService.TargetRef, consumer, port.TargetPort) + clusterName := b.clusterNameForExportedService(exportedService.TargetRef, consumer, port.TargetPort) clusters[clusterName] = &pbproxystate.Cluster{ Name: clusterName, Protocol: pbproxystate.Protocol_PROTOCOL_TCP, // TODO @@ -188,6 +271,31 @@ func (b *proxyStateTemplateBuilder) clusters() map[string]*pbproxystate.Cluster } } + // Clusters handling outgoing traffic from the local partition + for _, remoteGatewayID := range b.remoteGatewayIDs { + serviceID := resource.ReplaceType(pbcatalog.ServiceType, remoteGatewayID) + service, err := b.dataFetcher.FetchService(context.Background(), serviceID) + if err != nil { + b.logger.Trace("error reading exported service", "error", err) + continue + } else if service == nil { + b.logger.Trace("service does not exist, skipping router", "service", serviceID) + continue + } + + clusterName := b.clusterNameForRemoteGateway(remoteGatewayID) + clusters[clusterName] = &pbproxystate.Cluster{ + Name: clusterName, + Protocol: pbproxystate.Protocol_PROTOCOL_TCP, // TODO + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{}, + }, + }, + AltStatName: "prefix", + } + } + // Add null route cluster for any unmatched traffic clusters[nullRouteClusterName] = &pbproxystate.Cluster{ Name: nullRouteClusterName, @@ -232,6 +340,7 @@ func (b *proxyStateTemplateBuilder) Build() *meshv2beta1.ProxyStateTemplate { func (b *proxyStateTemplateBuilder) requiredEndpoints() map[string]*pbproxystate.EndpointRef { requiredEndpoints := make(map[string]*pbproxystate.EndpointRef) + // Endpoints for clusters handling incoming traffic from another partition for _, exportedService := range b.exportedServices { serviceID := resource.IDFromReference(exportedService.TargetRef) service, err := b.dataFetcher.FetchService(context.Background(), serviceID) @@ -244,38 +353,74 @@ func (b *proxyStateTemplateBuilder) requiredEndpoints() map[string]*pbproxystate } for _, port := range service.Data.Ports { + if port.Protocol == pbcatalog.Protocol_PROTOCOL_MESH { + continue + } + for _, consumer := range exportedService.Consumers { - clusterName := b.clusterName(exportedService.TargetRef, consumer, port.TargetPort) + clusterName := b.clusterNameForExportedService(exportedService.TargetRef, consumer, port.TargetPort) + requiredEndpoints[clusterName] = &pbproxystate.EndpointRef{ - Id: resource.ReplaceType(pbcatalog.ServiceEndpointsType, serviceID), - // In the case of a mesh gateway, the route port and mesh port are the same, since you are always - // routing to same port that you add in the endpoint. This is different from a sidecar proxy, where - // the receiving proxy listens on the mesh port and forwards to a different workload port. + Id: resource.ReplaceType(pbcatalog.ServiceEndpointsType, serviceID), RoutePort: port.TargetPort, - MeshPort: port.TargetPort, + MeshPort: "mesh", } } } } + // Endpoints for clusters handling outgoing traffic from the local partition + for _, remoteGatewayID := range b.remoteGatewayIDs { + serviceID := resource.ReplaceType(pbcatalog.ServiceType, remoteGatewayID) + service, err := b.dataFetcher.FetchService(context.Background(), serviceID) + if err != nil { + b.logger.Trace("error reading exported service", "error", err) + continue + } else if service == nil { + b.logger.Trace("service does not exist, skipping router", "service", serviceID) + continue + } + + clusterName := b.clusterNameForRemoteGateway(remoteGatewayID) + + // In the case of a mesh gateway, the route port and mesh port are the same, since you are always + // routing to same port that you add in the endpoint. This is different from a sidecar proxy, where + // the receiving proxy listens on the mesh port and forwards to a different workload port. + requiredEndpoints[clusterName] = &pbproxystate.EndpointRef{ + Id: resource.ReplaceType(pbcatalog.ServiceEndpointsType, serviceID), + MeshPort: meshgateways.WANPortName, + RoutePort: meshgateways.WANPortName, + } + } + return requiredEndpoints } -func (b *proxyStateTemplateBuilder) clusterName(serviceRef *pbresource.Reference, consumer *pbmulticluster.ComputedExportedServiceConsumer, port string) string { - return fmt.Sprintf("%s.%s", port, b.sni(serviceRef, consumer)) +// clusterNameForExportedService generates a cluster name for a given service +// that is being exported from the local partition to a remote partition. This +// partition may reside in the same datacenter or in a remote datacenter. +func (b *proxyStateTemplateBuilder) clusterNameForExportedService(serviceRef *pbresource.Reference, consumer *pbmulticluster.ComputedExportedServiceConsumer, port string) string { + return fmt.Sprintf("%s.%s", port, b.sniForExportedService(serviceRef, consumer)) } -func (b *proxyStateTemplateBuilder) sni(serviceRef *pbresource.Reference, consumer *pbmulticluster.ComputedExportedServiceConsumer) string { - switch tConsumer := consumer.Tenancy.(type) { +func (b *proxyStateTemplateBuilder) sniForExportedService(serviceRef *pbresource.Reference, consumer *pbmulticluster.ComputedExportedServiceConsumer) string { + switch consumer.Tenancy.(type) { case *pbmulticluster.ComputedExportedServiceConsumer_Partition: - return connect.ServiceSNI(serviceRef.Name, "", serviceRef.Tenancy.Namespace, tConsumer.Partition, b.dc, b.trustDomain) + return connect.ServiceSNI(serviceRef.Name, "", serviceRef.Tenancy.Namespace, serviceRef.Tenancy.Partition, b.dc, b.trustDomain) case *pbmulticluster.ComputedExportedServiceConsumer_Peer: - return connect.PeeredServiceSNI(serviceRef.Name, serviceRef.Tenancy.Namespace, serviceRef.Tenancy.Partition, tConsumer.Peer, b.trustDomain) + return connect.PeeredServiceSNI(serviceRef.Name, serviceRef.Tenancy.Namespace, serviceRef.Tenancy.Partition, b.dc, b.trustDomain) default: return "" } } +// clusterNameForRemoteGateway generates a cluster name for a given remote mesh +// gateway. This will be used to route traffic from the local partition to the mesh +// gateway for a remote partition. +func (b *proxyStateTemplateBuilder) clusterNameForRemoteGateway(remoteGatewayID *pbresource.ID) string { + return connect.GatewaySNI(b.dc, remoteGatewayID.Tenancy.Partition, b.trustDomain) +} + func alpnProtocol(portName string) string { return fmt.Sprintf("consul~%s", portName) } diff --git a/internal/mesh/internal/controllers/gatewayproxy/builder/builder_test.go b/internal/mesh/internal/controllers/gatewayproxy/builder/builder_test.go index 3f4968c6c9..16a5868035 100644 --- a/internal/mesh/internal/controllers/gatewayproxy/builder/builder_test.go +++ b/internal/mesh/internal/controllers/gatewayproxy/builder/builder_test.go @@ -15,7 +15,6 @@ import ( "github.com/hashicorp/consul/agent/connect" svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing" - "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/internal/catalog" "github.com/hashicorp/consul/internal/controller" "github.com/hashicorp/consul/internal/mesh/internal/controllers/gatewayproxy/fetcher" @@ -200,7 +199,7 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build "without address ports": suite.workloadWithOutAddressPorts, } { testutil.RunStep(suite.T(), name, func(t *testing.T) { - builder := NewProxyStateTemplateBuilder(workload, suite.exportedServicesPeerData.Data.Services, logger, f, dc, trustDomain) + builder := NewProxyStateTemplateBuilder(workload, suite.exportedServicesPeerData.Data.Services, logger, f, dc, trustDomain, nil) expectedProxyStateTemplate := &pbmesh.ProxyStateTemplate{ ProxyState: &pbmesh.ProxyState{ Identity: &pbresource.Reference{ @@ -210,7 +209,7 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build }, Listeners: []*pbproxystate.Listener{ { - Name: xdscommon.PublicListenerName, + Name: "wan_listener", Direction: pbproxystate.Direction_DIRECTION_INBOUND, BindAddress: &pbproxystate.Listener_HostPort{ HostPort: &pbproxystate.HostPortAddress{ @@ -237,29 +236,13 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build { Match: &pbproxystate.Match{ AlpnProtocols: []string{"consul~tcp"}, - ServerNames: []string{connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")}, + ServerNames: []string{connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, dc, "trustDomain")}, }, Destination: &pbproxystate.Router_L4{ L4: &pbproxystate.L4Destination{ Destination: &pbproxystate.L4Destination_Cluster{ Cluster: &pbproxystate.DestinationCluster{ - Name: fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")), - }, - }, - StatPrefix: "prefix", - }, - }, - }, - { - Match: &pbproxystate.Match{ - AlpnProtocols: []string{"consul~mesh"}, - ServerNames: []string{connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")}, - }, - Destination: &pbproxystate.Router_L4{ - L4: &pbproxystate.L4Destination{ - Destination: &pbproxystate.L4Destination_Cluster{ - Cluster: &pbproxystate.DestinationCluster{ - Name: fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")), + Name: fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, dc, "trustDomain")), }, }, StatPrefix: "prefix", @@ -285,18 +268,8 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build }, Protocol: pbproxystate.Protocol_PROTOCOL_TCP, }, - fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")): { - Name: fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")), - Group: &pbproxystate.Cluster_EndpointGroup{ - EndpointGroup: &pbproxystate.EndpointGroup{ - Group: &pbproxystate.EndpointGroup_Dynamic{}, - }, - }, - AltStatName: "prefix", - Protocol: pbproxystate.Protocol_PROTOCOL_TCP, // TODO - }, - fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")): { - Name: fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")), + fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, dc, "trustDomain")): { + Name: fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, dc, "trustDomain")), Group: &pbproxystate.Cluster_EndpointGroup{ EndpointGroup: &pbproxystate.EndpointGroup{ Group: &pbproxystate.EndpointGroup_Dynamic{}, @@ -308,23 +281,14 @@ func (suite *proxyStateTemplateBuilderSuite) TestProxyStateTemplateBuilder_Build }, }, RequiredEndpoints: map[string]*pbproxystate.EndpointRef{ - fmt.Sprintf("mesh.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")): { - Id: &pbresource.ID{ - Name: "api-1", - Type: pbcatalog.ServiceEndpointsType, - Tenancy: tenancy, - }, - RoutePort: "mesh", - MeshPort: "mesh", - }, - fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, "api-1", "trustDomain")): { + fmt.Sprintf("tcp.%s", connect.PeeredServiceSNI("api-1", tenancy.Namespace, tenancy.Partition, dc, "trustDomain")): { Id: &pbresource.ID{ Name: "api-1", Type: pbcatalog.ServiceEndpointsType, Tenancy: tenancy, }, RoutePort: "tcp", - MeshPort: "tcp", + MeshPort: "mesh", }, }, RequiredLeafCertificates: make(map[string]*pbproxystate.LeafCertificateRef), diff --git a/internal/mesh/internal/controllers/gatewayproxy/controller.go b/internal/mesh/internal/controllers/gatewayproxy/controller.go index 5868947e35..4dac4611c4 100644 --- a/internal/mesh/internal/controllers/gatewayproxy/controller.go +++ b/internal/mesh/internal/controllers/gatewayproxy/controller.go @@ -119,6 +119,7 @@ func (r *reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req c Type: pbmulticluster.ComputedExportedServicesType, } + // This covers any incoming requests from outside my partition to services inside my partition var exportedServices []*pbmulticluster.ComputedExportedService dec, err := dataFetcher.FetchComputedExportedServices(ctx, exportedServicesID) if err != nil { @@ -129,13 +130,27 @@ func (r *reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req c exportedServices = dec.Data.Services } + // This covers any incoming requests from inside my partition to services outside my partition + meshGateways, err := dataFetcher.FetchMeshGateways(ctx) + if err != nil { + rt.Logger.Warn("error reading the associated mesh gateways", "error", err) + } + + var remoteGatewayIDs []*pbresource.ID + for _, meshGateway := range meshGateways { + // If this is the mesh gateway in my local partition + datacenter, skip + if meshGateway.Id.Tenancy.Partition != req.ID.Tenancy.Partition { + remoteGatewayIDs = append(remoteGatewayIDs, meshGateway.Id) + } + } + trustDomain, err := r.getTrustDomain() if err != nil { rt.Logger.Error("error fetching trust domain to compute proxy state template", "error", err) return err } - newPST := builder.NewProxyStateTemplateBuilder(workload, exportedServices, rt.Logger, dataFetcher, r.dc, trustDomain).Build() + newPST := builder.NewProxyStateTemplateBuilder(workload, exportedServices, rt.Logger, dataFetcher, r.dc, trustDomain, remoteGatewayIDs).Build() proxyTemplateData, err := anypb.New(newPST) if err != nil { diff --git a/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go b/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go index 6d9f274c2d..3aa80df6d2 100644 --- a/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go +++ b/internal/mesh/internal/controllers/gatewayproxy/fetcher/data_fetcher.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/internal/mesh/internal/controllers/sidecarproxy/cache" "github.com/hashicorp/consul/internal/mesh/internal/types" "github.com/hashicorp/consul/internal/resource" @@ -44,6 +45,22 @@ func (f *Fetcher) FetchMeshGateway(ctx context.Context, id *pbresource.ID) (*typ return dec, nil } +// FetchMeshGateways fetches all MeshGateway resources known to the local server. +func (f *Fetcher) FetchMeshGateways(ctx context.Context) ([]*types.DecodedMeshGateway, error) { + tenancy := resource.DefaultClusteredTenancy() + tenancy.Partition = acl.WildcardPartitionName + + dec, err := resource.ListDecodedResource[*pbmesh.MeshGateway](ctx, f.client, &pbresource.ListRequest{ + Type: pbmesh.MeshGatewayType, + Tenancy: tenancy, + }) + if err != nil { + return nil, err + } + + return dec, nil +} + // FetchProxyStateTemplate fetches a service resource from the resource service. // This will panic if the type field in the ID argument is not a ProxyStateTemplate type. func (f *Fetcher) FetchProxyStateTemplate(ctx context.Context, id *pbresource.ID) (*types.DecodedProxyStateTemplate, error) { diff --git a/internal/mesh/internal/controllers/sidecarproxy/fetcher/data_fetcher.go b/internal/mesh/internal/controllers/sidecarproxy/fetcher/data_fetcher.go index 3707377cdc..71422e33fb 100644 --- a/internal/mesh/internal/controllers/sidecarproxy/fetcher/data_fetcher.go +++ b/internal/mesh/internal/controllers/sidecarproxy/fetcher/data_fetcher.go @@ -190,15 +190,34 @@ func (f *Fetcher) FetchComputedExplicitDestinationsData( targetServiceID := resource.IDFromReference(routeTarget.BackendRef.Ref) // Fetch ServiceEndpoints. - serviceEndpointsRef := &pbproxystate.EndpointRef{ - Id: resource.ReplaceType(pbcatalog.ServiceEndpointsType, targetServiceID), - MeshPort: routeTarget.MeshPort, - RoutePort: routeTarget.BackendRef.Port, + serviceEndpointID := resource.ReplaceType(pbcatalog.ServiceEndpointsType, targetServiceID) + se, err := f.FetchServiceEndpoints(ctx, serviceEndpointID) + if err != nil { + return nil, err + } + + if se != nil { + routeTarget.ServiceEndpointsRef = &pbproxystate.EndpointRef{ + Id: se.Id, + MeshPort: routeTarget.MeshPort, + RoutePort: routeTarget.BackendRef.Port, + } + routeTarget.ServiceEndpoints = se.Data + // Gather all identities. + var identities []*pbresource.Reference + for _, identity := range se.GetData().GetIdentities() { + identities = append(identities, &pbresource.Reference{ + Name: identity, + Tenancy: se.Resource.Id.Tenancy, + }) + } + routeTarget.IdentityRefs = identities } // If the target service is in a different partition and the mesh gateway mode is // "local" or "remote", use the ServiceEndpoints for the corresponding MeshGateway - // instead of the ServiceEndpoints for the target service. + // instead of the ServiceEndpoints for the target service. The IdentityRefs on the + // target will remain the same for TCP targets. // // TODO(nathancoleman) Consider cross-datacenter case as well if routeTarget.BackendRef.Ref.Tenancy.Partition != proxyID.Tenancy.Partition { @@ -210,7 +229,7 @@ func (f *Fetcher) FetchComputedExplicitDestinationsData( switch mode { case pbmesh.MeshGatewayMode_MESH_GATEWAY_MODE_LOCAL: // Use ServiceEndpoints for the MeshGateway in the source service's partition - serviceEndpointsRef = &pbproxystate.EndpointRef{ + routeTarget.ServiceEndpointsRef = &pbproxystate.EndpointRef{ Id: &pbresource.ID{ Type: pbcatalog.ServiceEndpointsType, Name: meshgateways.GatewayName, @@ -219,9 +238,16 @@ func (f *Fetcher) FetchComputedExplicitDestinationsData( MeshPort: meshgateways.LANPortName, RoutePort: meshgateways.LANPortName, } + + se, err := f.FetchServiceEndpoints(ctx, routeTarget.ServiceEndpointsRef.Id) + if err != nil { + return nil, err + } else if se != nil { + routeTarget.ServiceEndpoints = se.GetData() + } case pbmesh.MeshGatewayMode_MESH_GATEWAY_MODE_REMOTE: // Use ServiceEndpoints for the MeshGateway in the target service's partition - serviceEndpointsRef = &pbproxystate.EndpointRef{ + routeTarget.ServiceEndpointsRef = &pbproxystate.EndpointRef{ Id: &pbresource.ID{ Type: pbcatalog.ServiceEndpointsType, Name: meshgateways.GatewayName, @@ -230,28 +256,14 @@ func (f *Fetcher) FetchComputedExplicitDestinationsData( MeshPort: meshgateways.WANPortName, RoutePort: meshgateways.WANPortName, } - } - } - se, err := f.FetchServiceEndpoints(ctx, serviceEndpointsRef.Id) - if err != nil { - return nil, err - } - - if se != nil { - // We need to make sure the Uid is set - serviceEndpointsRef.Id = se.Id - routeTarget.ServiceEndpointsRef = serviceEndpointsRef - routeTarget.ServiceEndpoints = se.Data - // Gather all identities. - var identities []*pbresource.Reference - for _, identity := range se.GetData().GetIdentities() { - identities = append(identities, &pbresource.Reference{ - Name: identity, - Tenancy: se.Resource.Id.Tenancy, - }) + se, err := f.FetchServiceEndpoints(ctx, routeTarget.ServiceEndpointsRef.Id) + if err != nil { + return nil, err + } else if se != nil { + routeTarget.ServiceEndpoints = se.GetData() + } } - routeTarget.IdentityRefs = identities } }