diff --git a/agent/proxycfg/connect_proxy.go b/agent/proxycfg/connect_proxy.go index 1564e19e1c..f5d469d2e5 100644 --- a/agent/proxycfg/connect_proxy.go +++ b/agent/proxycfg/connect_proxy.go @@ -193,7 +193,7 @@ func (s *handlerConnectProxy) initialize(ctx context.Context) (ConfigSnapshot, e if err := (*handlerUpstreams)(s).resetWatchesFromChain(ctx, uid, chain, &snap.ConnectProxy.ConfigSnapshotUpstreams); err != nil { return snap, fmt.Errorf("error while resetting watches from chain: %w", err) } - return snap, nil + continue } err = s.cache.Notify(ctx, cachetype.CompiledDiscoveryChainName, &structs.DiscoveryChainRequest{ diff --git a/agent/proxycfg/testing_peering.go b/agent/proxycfg/testing_peering.go new file mode 100644 index 0000000000..3b469f79d4 --- /dev/null +++ b/agent/proxycfg/testing_peering.go @@ -0,0 +1,79 @@ +package proxycfg + +import ( + "github.com/mitchellh/go-testing-interface" + + "github.com/hashicorp/consul/agent/structs" +) + +func TestConfigSnapshotPeering(t testing.T) *ConfigSnapshot { + var ( + paymentsUpstream = structs.Upstream{ + DestinationName: "payments", + DestinationPeer: "cloud", + LocalBindPort: 9090, + } + paymentsUID = NewUpstreamID(&paymentsUpstream) + + refundsUpstream = structs.Upstream{ + DestinationName: "refunds", + DestinationPeer: "cloud", + LocalBindPort: 9090, + } + refundsUID = NewUpstreamID(&refundsUpstream) + ) + + return TestConfigSnapshot(t, func(ns *structs.NodeService) { + ns.Proxy.Upstreams = structs.Upstreams{ + paymentsUpstream, + refundsUpstream, + } + }, []UpdateEvent{ + { + CorrelationID: "upstream-target:payments.default.default.dc1:" + paymentsUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: []structs.CheckServiceNode{ + { + Node: &structs.Node{ + Address: "85.252.102.31", + Datacenter: "cloud-dc", + }, + Service: &structs.NodeService{ + Service: "payments-sidecar-proxy", + Kind: structs.ServiceKindConnectProxy, + Port: 443, + Connect: structs.ServiceConnect{ + PeerMeta: &structs.PeeringServiceMeta{ + SpiffeID: []string{"spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cloud-dc/svc/payments"}, + }, + }, + }, + }, + }, + }, + }, + { + CorrelationID: "upstream-target:refunds.default.default.dc1:" + refundsUID.String(), + Result: &structs.IndexedCheckServiceNodes{ + Nodes: []structs.CheckServiceNode{ + { + Node: &structs.Node{ + Address: "106.96.90.233", + Datacenter: "cloud-dc", + }, + Service: &structs.NodeService{ + Service: "refunds-sidecar-proxy", + Kind: structs.ServiceKindConnectProxy, + Port: 443, + Connect: structs.ServiceConnect{ + PeerMeta: &structs.PeeringServiceMeta{ + SpiffeID: []string{"spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/cloud-dc/svc/refunds"}, + }, + }, + }, + }, + }, + }, + }, + }) +} diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 85edd4f24b..e4847bec47 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -677,13 +677,27 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( sni := target.SNI clusterName := CustomizeClusterName(target.Name, chain) - targetSpiffeID := connect.SpiffeIDService{ - Host: cfgSnap.Roots.TrustDomain, - Namespace: target.Namespace, - Partition: target.Partition, - Datacenter: target.Datacenter, - Service: target.Service, - }.URI().String() + // Get the SpiffeID for upstream SAN validation. + // + // For imported services the SpiffeID is embedded in the proxy instances. + // Whereas for local services we can construct the SpiffeID from the chain target. + var targetSpiffeID string + if uid.Peer != "" { + for _, e := range chainEndpoints[targetID] { + targetSpiffeID = e.Service.Connect.PeerMeta.SpiffeID[0] + + // Only grab the first because it is the same for all instances. + break + } + } else { + targetSpiffeID = connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: target.Namespace, + Partition: target.Partition, + Datacenter: target.Datacenter, + Service: target.Service, + }.URI().String() + } if failoverThroughMeshGateway { actualTargetID := firstHealthyTarget( @@ -730,9 +744,7 @@ func (s *ResourceGenerator) makeUpstreamClustersForDiscoveryChain( spiffeIDs = append(spiffeIDs, id) } } - sort.Slice(spiffeIDs, func(i, j int) bool { - return spiffeIDs[i] < spiffeIDs[j] - }) + sort.Strings(spiffeIDs) s.Logger.Debug("generating cluster for", "cluster", clusterName) c := &envoy_cluster_v3.Cluster{ diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 4c04f4e0e5..7d89dfc1e5 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -37,6 +37,10 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshot(t, nil, nil) }, }, + { + name: "connect-proxy-with-peered-upstreams", + create: proxycfg.TestConfigSnapshotPeering, + }, { name: "connect-proxy-with-tls-outgoing-min-version-auto", create: func(t testinf.T) *proxycfg.ConfigSnapshot { diff --git a/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden b/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden new file mode 100644 index 0000000000..befaa062da --- /dev/null +++ b/agent/xds/testdata/clusters/connect-proxy-with-peered-upstreams.latest.golden @@ -0,0 +1,148 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "local_app", + "type": "STATIC", + "connectTimeout": "5s", + "loadAssignment": { + "clusterName": "local_app", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "127.0.0.1", + "portValue": 8080 + } + } + } + } + ] + } + ] + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "payments.default.dc1.internal.trustdomain.consul", + "altStatName": "payments.default.dc1.internal.trustdomain.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/cloud-dc/svc/payments" + } + ] + } + }, + "sni": "payments.default.dc1.internal.trustdomain.consul" + } + } + }, + { + "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "name": "refunds.default.dc1.internal.trustdomain.consul", + "altStatName": "refunds.default.dc1.internal.trustdomain.consul", + "type": "EDS", + "edsClusterConfig": { + "edsConfig": { + "ads": { + + }, + "resourceApiVersion": "V3" + } + }, + "connectTimeout": "5s", + "circuitBreakers": { + + }, + "outlierDetection": { + + }, + "commonLbConfig": { + "healthyPanicThreshold": { + + } + }, + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + }, + "matchSubjectAltNames": [ + { + "exact": "spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/cloud-dc/svc/refunds" + } + ] + } + }, + "sni": "refunds.default.dc1.internal.trustdomain.consul" + } + } + } + ], + "typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster", + "nonce": "00000001" +} \ No newline at end of file