[NET-5772] Make tcp external service registered on terminating gw reachable from peered cluster (#19881)

* Include SNI + root PEMs from peered cluster on terminating gw filter chain

This allows an external service registered on a terminating gateway to be exported to and reachable from a peered cluster

* Abstract existing logic into re-usable function

* Regenerate golden files w/ new listener logic

* Add changelog entry

* Use peering bundles that are stable across test runs

.changelog/19881.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+xds: Make TCP external service registered with terminating gateway reachable from peered cluster
+```

agent/xds/listeners.go

@@ -29,6 +29,7 @@ import (
 	envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
 	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+
 	"github.com/hashicorp/consul/agent/xds/config"
 	"github.com/hashicorp/consul/agent/xds/naming"
 	"github.com/hashicorp/consul/agent/xds/platform"
@@ -1178,29 +1179,9 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
 	}
 
 	// Inject peering trust bundles if this service is exported to peered clusters.
-	if len(peerBundles) > 0 {
+	err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext, peerBundles)
-		spiffeConfig, err := makeSpiffeValidatorConfig(
+	if err != nil {
-			cfgSnap.Roots.TrustDomain,
+		return nil, err
-			cfgSnap.RootPEMs(),
-			peerBundles,
-		)
-		if err != nil {
-			return nil, err
-		}
-
-		typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
-		if !ok {
-			return nil, fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
-		}
-
-		// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
-		// We nil it out here since the local roots are included in the SPIFFE validator config.
-		typ.ValidationContext.TrustedCa = nil
-		typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
-			// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
-			Name:        "envoy.tls.cert_validator.spiffe",
-			TypedConfig: spiffeConfig,
-		}
 	}
 
 	return makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{
@@ -1209,6 +1190,32 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh
 	})
 }
 
+func injectSpiffeValidatorConfigForPeers(cfgSnap *proxycfg.ConfigSnapshot, tlsContext *envoy_tls_v3.CommonTlsContext, peerBundles []*pbpeering.PeeringTrustBundle) error {
+	if len(peerBundles) == 0 {
+		return nil
+	}
+
+	spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), peerBundles)
+	if err != nil {
+		return err
+	}
+
+	typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext)
+	if !ok {
+		return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType)
+	}
+
+	// makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA.
+	// We nil it out here since the local roots are included in the SPIFFE validator config.
+	typ.ValidationContext.TrustedCa = nil
+	typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{
+		// The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib.
+		Name:        "envoy.tls.cert_validator.spiffe",
+		TypedConfig: spiffeConfig,
+	}
+	return nil
+}
+
 // SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own.
 // With cluster peering we expect peered clusters to have independent certificate authorities.
 // This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS,
@@ -1752,6 +1759,15 @@ type terminatingGatewayFilterChainOpts struct {
 }
 
 func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, tgtwyOpts terminatingGatewayFilterChainOpts) (*envoy_listener_v3.FilterChain, error) {
+	// We need to at least match the SNI and use the root PEMs from the local cluster; however, requests coming
+	// from peered clusters where the external service is exported to will have their own SNI and root PEMs.
+	sniMatches := []string{tgtwyOpts.cluster}
+	for _, bundle := range tgtwyOpts.peerTrustBundles {
+		svc := tgtwyOpts.service
+		sourceSNI := connect.PeeredServiceSNI(svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), bundle.PeerName, cfgSnap.Roots.TrustDomain)
+		sniMatches = append(sniMatches, sourceSNI)
+	}
+
 	tlsContext := &envoy_tls_v3.DownstreamTlsContext{
 		CommonTlsContext: makeCommonTLSContext(
 			cfgSnap.TerminatingGateway.ServiceLeaves[tgtwyOpts.service],
@@ -1760,13 +1776,19 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.
 		),
 		RequireClientCertificate: &wrapperspb.BoolValue{Value: true},
 	}
+
+	err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext.CommonTlsContext, tgtwyOpts.peerTrustBundles)
+	if err != nil {
+		return nil, err
+	}
+
 	transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
 	if err != nil {
 		return nil, err
 	}
 
 	filterChain := &envoy_listener_v3.FilterChain{
-		FilterChainMatch: makeSNIFilterChainMatch(tgtwyOpts.cluster),
+		FilterChainMatch: makeSNIFilterChainMatch(sniMatches...),
 		Filters:          make([]*envoy_listener_v3.Filter, 0, 3),
 		TransportSocket:  transportSocket,
 	}

agent/xds/resources_test.go

@@ -29,7 +29,6 @@ import (
 	"github.com/hashicorp/consul/agent/xds/testcommon"
 	"github.com/hashicorp/consul/agent/xdsv2"
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
-	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/types"
 )
@@ -2281,32 +2280,25 @@ func getTerminatingGatewayPeeringGoldenTestCases() []goldenTestCase {
 		{
 			name: "terminating-gateway-with-peer-trust-bundle",
 			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
-				roots, _ := proxycfg.TestCerts(t)
+				bundles := proxycfg.TestPeerTrustBundles(t)
 				return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, []proxycfg.UpdateEvent{
 					{
 						CorrelationID: "peer-trust-bundle:web",
-						Result: &pbpeering.TrustBundleListByServiceResponse{
+						Result:        bundles,
-							Bundles: []*pbpeering.PeeringTrustBundle{
-								{
-									TrustDomain: "foo.bar.gov",
-									PeerName:    "dc2",
-									Partition:   "default",
-									RootPEMs: []string{
-										roots.Roots[0].RootCert,
-									},
-									ExportedPartition: "default",
-									CreateIndex:       0,
-									ModifyIndex:       0,
-								},
-							},
-						},
 					},
 					{
 						CorrelationID: "service-intentions:web",
 						Result: structs.SimplifiedIntentions{
 							{
 								SourceName:           "source",
-								SourcePeer:           "dc2",
+								SourcePeer:           bundles.Bundles[0].PeerName,
+								DestinationName:      "web",
+								DestinationPartition: "default",
+								Action:               structs.IntentionActionAllow,
+							},
+							{
+								SourceName:           "source",
+								SourcePeer:           bundles.Bundles[1].PeerName,
 								DestinationName:      "web",
 								DestinationPartition: "default",
 								Action:               structs.IntentionActionAllow,

agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden

@@ -163,7 +163,9 @@
         {
           "filterChainMatch": {
             "serverNames": [
-              "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
+              "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
+              "web.default.default.peer-a.external.11111111-2222-3333-4444-555555555555.consul",
+              "web.default.default.peer-b.external.11111111-2222-3333-4444-555555555555.consul"
             ]
           },
           "filters": [
@@ -184,7 +186,16 @@
                           "authenticated": {
                             "principalName": {
                               "safeRegex": {
-                                "regex": "^spiffe://foo.bar.gov/ns/default/dc/[^/]+/svc/source$"
+                                "regex": "^spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/[^/]+/svc/source$"
+                              }
+                            }
+                          }
+                        },
+                        {
+                          "authenticated": {
+                            "principalName": {
+                              "safeRegex": {
+                                "regex": "^spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/[^/]+/svc/source$"
                               }
                             }
                           }
@@ -222,8 +233,31 @@
                 ],
                 "tlsParams": {},
                 "validationContext": {
-                  "trustedCa": {
+                  "customValidatorConfig": {
-                    "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
+                    "name": "envoy.tls.cert_validator.spiffe",
+                    "typedConfig": {
+                      "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig",
+                      "trustDomains": [
+                        {
+                          "name": "11111111-2222-3333-4444-555555555555.consul",
+                          "trustBundle": {
+                            "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
+                          }
+                        },
+                        {
+                          "name": "1c053652-8512-4373-90cf-5a7f6263a994.consul",
+                          "trustBundle": {
+                            "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n"
+                          }
+                        },
+                        {
+                          "name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul",
+                          "trustBundle": {
+                            "inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n"
+                          }
+                        }
+                      ]
+                    }
                   }
                 }
               },