Enable outbound peered requests to go through local mesh gateway (#14763)

agent/proxycfg/connect_proxy.go

@@ -26,6 +26,7 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e
 	snap.ConnectProxy.UpstreamPeerTrustBundles = watch.NewMap[string, *pbpeering.PeeringTrustBundle]()
 	snap.ConnectProxy.WatchedGateways = make(map[UpstreamID]map[string]context.CancelFunc)
 	snap.ConnectProxy.WatchedGatewayEndpoints = make(map[UpstreamID]map[string]structs.CheckServiceNodes)
+	snap.ConnectProxy.WatchedLocalGWEndpoints = watch.NewMap[string, structs.CheckServiceNodes]()
 	snap.ConnectProxy.WatchedServiceChecks = make(map[structs.ServiceID][]structs.CheckType)
 	snap.ConnectProxy.PreparedQueryEndpoints = make(map[UpstreamID]structs.CheckServiceNodes)
 	snap.ConnectProxy.DestinationsUpstream = watch.NewMap[UpstreamID, *structs.ServiceConfigEntry]()
@@ -184,7 +185,7 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e
 
 		switch u.DestinationType {
 		case structs.UpstreamDestTypePreparedQuery:
-			err = s.dataSources.PreparedQuery.Notify(ctx, &structs.PreparedQueryExecuteRequest{
+			err := s.dataSources.PreparedQuery.Notify(ctx, &structs.PreparedQueryExecuteRequest{
 				Datacenter:    dc,
 				QueryOptions:  structs.QueryOptions{Token: s.token, MaxAge: defaultPreparedQueryPollInterval},
 				QueryIDOrName: u.DestinationName,
@@ -200,51 +201,14 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e
 
 		case "":
 			if u.DestinationPeer != "" {
-				// NOTE: An upstream that points to a peer by definition will
+				err := s.setupWatchesForPeeredUpstream(ctx, snap.ConnectProxy, u, dc)
-				// only ever watch a single catalog query, so a map key of just
-				// "UID" is sufficient to cover the peer data watches here.
-
-				s.logger.Trace("initializing watch of peered upstream", "upstream", uid)
-
-				snap.ConnectProxy.PeerUpstreamEndpoints.InitWatch(uid, nil)
-				err := s.dataSources.Health.Notify(ctx, &structs.ServiceSpecificRequest{
-					PeerName:   uid.Peer,
-					Datacenter: dc,
-					QueryOptions: structs.QueryOptions{
-						Token: s.token,
-					},
-					ServiceName: u.DestinationName,
-					Connect:     true,
-					// Note that Identifier doesn't type-prefix for service any more as it's
-					// the default and makes metrics and other things much cleaner. It's
-					// simpler for us if we have the type to make things unambiguous.
-					Source:         *s.source,
-					EnterpriseMeta: uid.EnterpriseMeta,
-				}, upstreamPeerWatchIDPrefix+uid.String(), s.ch)
 				if err != nil {
 					return snap, err
 				}
-
-				// Check whether a watch for this peer exists to avoid duplicates.
-				if ok := snap.ConnectProxy.UpstreamPeerTrustBundles.IsWatched(uid.Peer); !ok {
-					peerCtx, cancel := context.WithCancel(ctx)
-					if err := s.dataSources.TrustBundle.Notify(peerCtx, &cachetype.TrustBundleReadRequest{
-						Request: &pbpeering.TrustBundleReadRequest{
-							Name:      uid.Peer,
-							Partition: uid.PartitionOrDefault(),
-						},
-						QueryOptions: structs.QueryOptions{Token: s.token},
-					}, peerTrustBundleIDPrefix+uid.Peer, s.ch); err != nil {
-						cancel()
-						return snap, fmt.Errorf("error while watching trust bundle for peer %q: %w", uid.Peer, err)
-					}
-
-					snap.ConnectProxy.UpstreamPeerTrustBundles.InitWatch(uid.Peer, cancel)
-				}
 				continue
 			}
 
-			err = s.dataSources.CompiledDiscoveryChain.Notify(ctx, &structs.DiscoveryChainRequest{
+			err := s.dataSources.CompiledDiscoveryChain.Notify(ctx, &structs.DiscoveryChainRequest{
 				Datacenter:             s.source.Datacenter,
 				QueryOptions:           structs.QueryOptions{Token: s.token},
 				Name:                   u.DestinationName,
@@ -267,6 +231,76 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e
 	return snap, nil
 }
 
+func (s *handlerConnectProxy) setupWatchesForPeeredUpstream(
+	ctx context.Context,
+	snapConnectProxy configSnapshotConnectProxy,
+	u structs.Upstream,
+	dc string,
+) error {
+	uid := NewUpstreamID(&u)
+
+	s.logger.Trace("initializing watch of peered upstream", "upstream", uid)
+
+	// NOTE: An upstream that points to a peer by definition will
+	// only ever watch a single catalog query, so a map key of just
+	// "UID" is sufficient to cover the peer data watches here.
+	snapConnectProxy.PeerUpstreamEndpoints.InitWatch(uid, nil)
+	err := s.dataSources.Health.Notify(ctx, &structs.ServiceSpecificRequest{
+		PeerName:   uid.Peer,
+		Datacenter: dc,
+		QueryOptions: structs.QueryOptions{
+			Token: s.token,
+		},
+		ServiceName:    u.DestinationName,
+		Connect:        true,
+		Source:         *s.source,
+		EnterpriseMeta: uid.EnterpriseMeta,
+	}, upstreamPeerWatchIDPrefix+uid.String(), s.ch)
+	if err != nil {
+		return err
+	}
+
+	// Check whether a watch for this peer exists to avoid duplicates.
+	if ok := snapConnectProxy.UpstreamPeerTrustBundles.IsWatched(uid.Peer); !ok {
+		peerCtx, cancel := context.WithCancel(ctx)
+		if err := s.dataSources.TrustBundle.Notify(peerCtx, &cachetype.TrustBundleReadRequest{
+			Request: &pbpeering.TrustBundleReadRequest{
+				Name:      uid.Peer,
+				Partition: uid.PartitionOrDefault(),
+			},
+			QueryOptions: structs.QueryOptions{Token: s.token},
+		}, peerTrustBundleIDPrefix+uid.Peer, s.ch); err != nil {
+			cancel()
+			return fmt.Errorf("error while watching trust bundle for peer %q: %w", uid.Peer, err)
+		}
+
+		snapConnectProxy.UpstreamPeerTrustBundles.InitWatch(uid.Peer, cancel)
+	}
+
+	// If a peered upstream is set to local mesh gw mode,
+	// set up a watch for them.
+	if u.MeshGateway.Mode == structs.MeshGatewayModeLocal {
+		gk := GatewayKey{
+			Partition:  s.source.NodePartitionOrDefault(),
+			Datacenter: s.source.Datacenter,
+		}
+		if !snapConnectProxy.WatchedLocalGWEndpoints.IsWatched(gk.String()) {
+			opts := gatewayWatchOpts{
+				internalServiceDump: s.dataSources.InternalServiceDump,
+				notifyCh:            s.ch,
+				source:              *s.source,
+				token:               s.token,
+				key:                 gk,
+			}
+			if err := watchMeshGateway(ctx, opts); err != nil {
+				return fmt.Errorf("error while watching for local mesh gateway: %w", err)
+			}
+			snapConnectProxy.WatchedLocalGWEndpoints.InitWatch(gk.String(), nil)
+		}
+	}
+	return nil
+}
+
 func (s *handlerConnectProxy) handleUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot) error {
 	if u.Err != nil {
 		return fmt.Errorf("error filling agent cache: %v", u.Err)

agent/proxycfg/manager_test.go

@@ -224,6 +224,7 @@ func TestManager_BasicLifecycle(t *testing.T) {
 						WatchedGatewayEndpoints: map[UpstreamID]map[string]structs.CheckServiceNodes{
 							dbUID: {},
 						},
+						WatchedLocalGWEndpoints: watch.NewMap[string, structs.CheckServiceNodes](),
 						UpstreamConfig: map[UpstreamID]*structs.Upstream{
 							NewUpstreamID(&upstreams[0]): &upstreams[0],
 							NewUpstreamID(&upstreams[1]): &upstreams[1],
@@ -287,6 +288,7 @@ func TestManager_BasicLifecycle(t *testing.T) {
 						WatchedGatewayEndpoints: map[UpstreamID]map[string]structs.CheckServiceNodes{
 							dbUID: {},
 						},
+						WatchedLocalGWEndpoints: watch.NewMap[string, structs.CheckServiceNodes](),
 						UpstreamConfig: map[UpstreamID]*structs.Upstream{
 							NewUpstreamID(&upstreams[0]): &upstreams[0],
 							NewUpstreamID(&upstreams[1]): &upstreams[1],

agent/proxycfg/snapshot.go

@@ -58,6 +58,16 @@ type ConfigSnapshotUpstreams struct {
 	// backing endpoints of a mesh gateway.
 	WatchedGatewayEndpoints map[UpstreamID]map[string]structs.CheckServiceNodes
 
+	// WatchedLocalGWEndpoints is used to store the backing endpoints of
+	// a local mesh gateway. Currently, this is used by peered upstreams
+	// configured with local mesh gateway mode so that they can watch for
+	// gateway endpoints.
+	//
+	// Note that the string form of GatewayKey is used as the key so empty
+	// fields can be normalized in OSS.
+	//   GatewayKey.String() -> structs.CheckServiceNodes
+	WatchedLocalGWEndpoints watch.Map[string, structs.CheckServiceNodes]
+
 	// UpstreamConfig is a map to an upstream's configuration.
 	UpstreamConfig map[UpstreamID]*structs.Upstream
 

agent/proxycfg/state.go

@@ -476,6 +476,13 @@ type gatewayWatchOpts struct {
 }
 
 func watchMeshGateway(ctx context.Context, opts gatewayWatchOpts) error {
+	var correlationId string
+	if opts.upstreamID.Name == "" {
+		correlationId = fmt.Sprintf("mesh-gateway:%s", opts.key.String())
+	} else {
+		correlationId = fmt.Sprintf("mesh-gateway:%s:%s", opts.key.String(), opts.upstreamID.String())
+	}
+
 	return opts.internalServiceDump.Notify(ctx, &structs.ServiceDumpRequest{
 		Datacenter:     opts.key.Datacenter,
 		QueryOptions:   structs.QueryOptions{Token: opts.token},
@@ -483,5 +490,5 @@ func watchMeshGateway(ctx context.Context, opts gatewayWatchOpts) error {
 		UseServiceKind: true,
 		Source:         opts.source,
 		EnterpriseMeta: *structs.DefaultEnterpriseMetaInPartition(opts.key.Partition),
-	}, fmt.Sprintf("mesh-gateway:%s:%s", opts.key.String(), opts.upstreamID.String()), opts.notifyCh)
+	}, correlationId, opts.notifyCh)
 }

agent/proxycfg/testing_peering.go

@@ -249,3 +249,131 @@ func TestConfigSnapshotPeeringTProxy(t testing.T) *ConfigSnapshot {
 		},
 	})
 }
+
+func TestConfigSnapshotPeeringLocalMeshGateway(t testing.T) *ConfigSnapshot {
+	var (
+		paymentsUpstream = structs.Upstream{
+			DestinationName: "payments",
+			DestinationPeer: "cloud",
+			LocalBindPort:   9090,
+			MeshGateway:     structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeLocal},
+		}
+		paymentsUID = NewUpstreamID(&paymentsUpstream)
+
+		refundsUpstream = structs.Upstream{
+			DestinationName: "refunds",
+			DestinationPeer: "cloud",
+			LocalBindPort:   9090,
+			MeshGateway:     structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeLocal},
+		}
+		refundsUID = NewUpstreamID(&refundsUpstream)
+	)
+
+	const peerTrustDomain = "1c053652-8512-4373-90cf-5a7f6263a994.consul"
+
+	return TestConfigSnapshot(t, func(ns *structs.NodeService) {
+		ns.Proxy.Upstreams = structs.Upstreams{
+			paymentsUpstream,
+			refundsUpstream,
+		}
+	}, []UpdateEvent{
+		{
+			CorrelationID: peerTrustBundleIDPrefix + "cloud",
+			Result: &pbpeering.TrustBundleReadResponse{
+				Bundle: TestPeerTrustBundles(t).Bundles[0],
+			},
+		},
+		{
+			CorrelationID: upstreamPeerWatchIDPrefix + paymentsUID.String(),
+			Result: &structs.IndexedCheckServiceNodes{
+				Nodes: []structs.CheckServiceNode{
+					{
+						Node: &structs.Node{
+							Address:    "85.252.102.31",
+							Datacenter: "cloud-dc",
+						},
+						Service: &structs.NodeService{
+							Service: "payments-sidecar-proxy",
+							Kind:    structs.ServiceKindConnectProxy,
+							Port:    443,
+							TaggedAddresses: map[string]structs.ServiceAddress{
+								structs.TaggedAddressLAN: {
+									Address: "85.252.102.31",
+									Port:    443,
+								},
+								structs.TaggedAddressWAN: {
+									Address: "123.us-east-1.elb.notaws.com",
+									Port:    8443,
+								},
+							},
+							Connect: structs.ServiceConnect{
+								PeerMeta: &structs.PeeringServiceMeta{
+									SNI: []string{
+										"payments.default.default.cloud.external." + peerTrustDomain,
+									},
+									SpiffeID: []string{
+										"spiffe://" + peerTrustDomain + "/ns/default/dc/cloud-dc/svc/payments",
+									},
+									Protocol: "tcp",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			CorrelationID: upstreamPeerWatchIDPrefix + refundsUID.String(),
+			Result: &structs.IndexedCheckServiceNodes{
+				Nodes: []structs.CheckServiceNode{
+					{
+						Node: &structs.Node{
+							Address:    "106.96.90.233",
+							Datacenter: "cloud-dc",
+						},
+						Service: &structs.NodeService{
+							Service: "refunds-sidecar-proxy",
+							Kind:    structs.ServiceKindConnectProxy,
+							Port:    443,
+							Connect: structs.ServiceConnect{
+								PeerMeta: &structs.PeeringServiceMeta{
+									SNI: []string{
+										"refunds.default.default.cloud.external." + peerTrustDomain,
+									},
+									SpiffeID: []string{
+										"spiffe://" + peerTrustDomain + "/ns/default/dc/cloud-dc/svc/refunds",
+									},
+									Protocol: "tcp",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			CorrelationID: "mesh-gateway:dc1",
+			Result: &structs.IndexedCheckServiceNodes{
+				Nodes: structs.CheckServiceNodes{
+					structs.CheckServiceNode{
+						Node: &structs.Node{
+							ID:         "mesh-gateway",
+							Node:       "mesh-gateway",
+							Address:    "10.0.0.1",
+							Datacenter: "dc1",
+						},
+						Service: &structs.NodeService{
+							Kind:    structs.ServiceKindMeshGateway,
+							Service: "mesh-gateway",
+							Port:    1234,
+							TaggedAddresses: map[string]structs.ServiceAddress{
+								structs.TaggedAddressWAN: {Address: "172.100.0.14", Port: 8080},
+							},
+							EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
+						},
+					},
+				},
+			},
+		},
+	})
+}

agent/proxycfg/upstreams.go

@@ -191,17 +191,19 @@ func (s *handlerUpstreams) handleUpdateUpstreams(ctx context.Context, u UpdateEv
 			return fmt.Errorf("invalid type for response: %T", u.Result)
 		}
 		correlationID := strings.TrimPrefix(u.CorrelationID, "mesh-gateway:")
-		key, uidString, ok := removeColonPrefix(correlationID)
+		key, uidString, ok := strings.Cut(correlationID, ":")
-		if !ok {
+		if ok {
-			return fmt.Errorf("invalid correlation id %q", u.CorrelationID)
+			// correlationID formatted with an upstreamID
-		}
 			uid := UpstreamIDFromString(uidString)
 
 			if _, ok = upstreamsSnapshot.WatchedGatewayEndpoints[uid]; !ok {
 				upstreamsSnapshot.WatchedGatewayEndpoints[uid] = make(map[string]structs.CheckServiceNodes)
 			}
 			upstreamsSnapshot.WatchedGatewayEndpoints[uid][key] = resp.Nodes
-
+		} else {
+			// event was for local gateways only
+			upstreamsSnapshot.WatchedLocalGWEndpoints.Set(key, resp.Nodes)
+		}
 	default:
 		return fmt.Errorf("unknown correlation ID: %s", u.CorrelationID)
 	}

agent/xds/clusters.go

@@ -794,8 +794,12 @@ func (s *ResourceGenerator) makeUpstreamClusterForPeerService(
 
 		useEDS := true
 		if _, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpointsUseHostnames[uid]; ok {
+			// If we're using local mesh gw, the fact that upstreams use hostnames don't matter.
+			// If we're not using local mesh gw, then resort to CDS.
+			if upstreamConfig.MeshGateway.Mode != structs.MeshGatewayModeLocal {
 				useEDS = false
 			}
+		}
 
 		// If none of the service instances are addressed by a hostname we
 		// provide the endpoint IP addresses via EDS

agent/xds/endpoints.go

@@ -9,7 +9,7 @@ import (
 	envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 
 	"github.com/golang/protobuf/proto"
-	bexpr "github.com/hashicorp/go-bexpr"
+	"github.com/hashicorp/go-bexpr"
 
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/proxycfg"
@@ -108,7 +108,6 @@ func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.
 		clusterName := generatePeeredClusterName(uid, tbs)
 
 		loadAssignment, err := s.makeUpstreamLoadAssignmentForPeerService(cfgSnap, clusterName, uid)
-
 		if err != nil {
 			return nil, err
 		}
@@ -416,6 +415,25 @@ func (s *ResourceGenerator) makeUpstreamLoadAssignmentForPeerService(cfgSnap *pr
 		return la, err
 	}
 
+	upstream := cfgSnap.ConnectProxy.UpstreamConfig[uid]
+	// If an upstream is configured with local mesh gw mode, we make a load assignment
+	// from the gateway endpoints instead of those of the upstreams.
+	if upstream != nil && upstream.MeshGateway.Mode == structs.MeshGatewayModeLocal {
+		localGw, ok := cfgSnap.ConnectProxy.WatchedLocalGWEndpoints.Get(cfgSnap.Locality.String())
+		if !ok {
+			// local GW is not ready; return early
+			return la, nil
+		}
+		la = makeLoadAssignment(
+			clusterName,
+			[]loadAssignmentEndpointGroup{
+				{Endpoints: localGw},
+			},
+			cfgSnap.Locality,
+		)
+		return la, nil
+	}
+
 	// Also skip peer instances with a hostname as their address. EDS
 	// cannot resolve hostnames, so we provide them through CDS instead.
 	if _, ok := upstreamsSnapshot.PeerUpstreamEndpointsUseHostnames[uid]; ok {

agent/xds/listeners.go

@@ -275,7 +275,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 				return nil
 			}
 			configuredPorts[svcConfig.Destination.Port] = struct{}{}
-			const name = "~http" //name used for the shared route name
+			const name = "~http" // name used for the shared route name
 			routeName := clusterNameForDestination(cfgSnap, name, fmt.Sprintf("%d", svcConfig.Destination.Port), svcConfig.NamespaceOrDefault(), svcConfig.PartitionOrDefault())
 			filterChain, err := s.makeUpstreamFilterChain(filterChainOpts{
 				routeName:  routeName,
@@ -2016,6 +2016,10 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForPeeredListener(
 		cfg.ConnectTimeoutMs = 5000
 	}
 
+	if cfg.MeshGateway.Mode == "" && u != nil {
+		cfg.MeshGateway = u.MeshGateway
+	}
+
 	return cfg
 }
 

agent/xds/resources_test.go

@@ -156,6 +156,10 @@ func TestAllResourcesFromSnapshot(t *testing.T) {
 			name:   "transparent-proxy-with-peered-upstreams",
 			create: proxycfg.TestConfigSnapshotPeeringTProxy,
 		},
+		{
+			name:   "local-mesh-gateway-with-peered-upstreams",
+			create: proxycfg.TestConfigSnapshotPeeringLocalMeshGateway,
+		},
 	}
 	tests = append(tests, getConnectProxyTransparentProxyGoldenTestCases()...)
 	tests = append(tests, getMeshGatewayPeeringGoldenTestCases()...)

agent/xds/testdata/clusters/local-mesh-gateway-with-peered-upstreams.latest.golden

@@ -0,0 +1,146 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "name": "local_app",
+      "type": "STATIC",
+      "connectTimeout": "5s",
+      "loadAssignment": {
+        "clusterName": "local_app",
+        "endpoints": [
+          {
+            "lbEndpoints": [
+              {
+                "endpoint": {
+                  "address": {
+                    "socketAddress": {
+                      "address": "127.0.0.1",
+                      "portValue": 8080
+                    }
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "name": "payments.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          },
+          "resourceApiVersion": "V3"
+        }
+      },
+      "connectTimeout": "5s",
+      "circuitBreakers": {
+
+      },
+      "outlierDetection": {
+        "maxEjectionPercent": 100
+      },
+      "commonLbConfig": {
+        "healthyPanicThreshold": {
+
+        }
+      },
+      "transportSocket": {
+        "name": "tls",
+        "typedConfig": {
+          "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+          "commonTlsContext": {
+            "tlsParams": {
+
+            },
+            "tlsCertificates": [
+              {
+                "certificateChain": {
+                  "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
+                },
+                "privateKey": {
+                  "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
+                }
+              }
+            ],
+            "validationContext": {
+              "trustedCa": {
+                "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n"
+              },
+              "matchSubjectAltNames": [
+                {
+                  "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cloud-dc/svc/payments"
+                }
+              ]
+            }
+          },
+          "sni": "payments.default.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul"
+        }
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+      "name": "refunds.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          },
+          "resourceApiVersion": "V3"
+        }
+      },
+      "connectTimeout": "5s",
+      "circuitBreakers": {
+
+      },
+      "outlierDetection": {
+        "maxEjectionPercent": 100
+      },
+      "commonLbConfig": {
+        "healthyPanicThreshold": {
+
+        }
+      },
+      "transportSocket": {
+        "name": "tls",
+        "typedConfig": {
+          "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
+          "commonTlsContext": {
+            "tlsParams": {
+
+            },
+            "tlsCertificates": [
+              {
+                "certificateChain": {
+                  "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
+                },
+                "privateKey": {
+                  "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
+                }
+              }
+            ],
+            "validationContext": {
+              "trustedCa": {
+                "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n"
+              },
+              "matchSubjectAltNames": [
+                {
+                  "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cloud-dc/svc/refunds"
+                }
+              ]
+            }
+          },
+          "sni": "refunds.default.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul"
+        }
+      }
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+  "nonce": "00000001"
+}

agent/xds/testdata/endpoints/local-mesh-gateway-with-peered-upstreams.latest.golden

@@ -0,0 +1,51 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName": "payments.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.0.0.1",
+                    "portValue": 1234
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+      "clusterName": "refunds.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "10.0.0.1",
+                    "portValue": 1234
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+  "nonce": "00000001"
+}

agent/xds/testdata/listeners/local-mesh-gateway-with-peered-upstreams.latest.golden

@@ -0,0 +1,119 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
+      "name": "payments?peer=cloud:127.0.0.1:9090",
+      "address": {
+        "socketAddress": {
+          "address": "127.0.0.1",
+          "portValue": 9090
+        }
+      },
+      "filterChains": [
+        {
+          "filters": [
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "statPrefix": "upstream_peered.payments.default.cloud",
+                "cluster": "payments.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul"
+              }
+            }
+          ]
+        }
+      ],
+      "trafficDirection": "OUTBOUND"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
+      "name": "public_listener:0.0.0.0:9999",
+      "address": {
+        "socketAddress": {
+          "address": "0.0.0.0",
+          "portValue": 9999
+        }
+      },
+      "filterChains": [
+        {
+          "filters": [
+            {
+              "name": "envoy.filters.network.rbac",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
+                "rules": {
+
+                },
+                "statPrefix": "connect_authz"
+              }
+            },
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "statPrefix": "public_listener",
+                "cluster": "local_app"
+              }
+            }
+          ],
+          "transportSocket": {
+            "name": "tls",
+            "typedConfig": {
+              "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
+              "commonTlsContext": {
+                "tlsParams": {
+
+                },
+                "tlsCertificates": [
+                  {
+                    "certificateChain": {
+                      "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n"
+                    },
+                    "privateKey": {
+                      "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n"
+                    }
+                  }
+                ],
+                "validationContext": {
+                  "trustedCa": {
+                    "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
+                  }
+                }
+              },
+              "requireClientCertificate": true
+            }
+          }
+        }
+      ],
+      "trafficDirection": "INBOUND"
+    },
+    {
+      "@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
+      "name": "refunds?peer=cloud:127.0.0.1:9090",
+      "address": {
+        "socketAddress": {
+          "address": "127.0.0.1",
+          "portValue": 9090
+        }
+      },
+      "filterChains": [
+        {
+          "filters": [
+            {
+              "name": "envoy.filters.network.tcp_proxy",
+              "typedConfig": {
+                "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
+                "statPrefix": "upstream_peered.refunds.default.cloud",
+                "cluster": "refunds.default.cloud.external.1c053652-8512-4373-90cf-5a7f6263a994.consul"
+              }
+            }
+          ]
+        }
+      ],
+      "trafficDirection": "OUTBOUND"
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
+  "nonce": "00000001"
+}

agent/xds/testdata/routes/local-mesh-gateway-with-peered-upstreams.latest.golden

@@ -0,0 +1,5 @@
+{
+  "versionInfo": "00000001",
+  "typeUrl": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+  "nonce": "00000001"
+}