2018-10-03 18:18:55 +00:00
|
|
|
package xds
|
|
|
|
|
|
|
|
import (
|
|
|
|
"errors"
|
2019-06-24 19:05:36 +00:00
|
|
|
"fmt"
|
2021-10-26 22:10:30 +00:00
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
|
|
|
|
envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
|
|
|
|
envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
|
2021-02-22 21:00:15 +00:00
|
|
|
|
2020-06-23 20:19:56 +00:00
|
|
|
"github.com/golang/protobuf/proto"
|
2021-02-22 21:00:15 +00:00
|
|
|
bexpr "github.com/hashicorp/go-bexpr"
|
|
|
|
|
2022-04-05 21:10:06 +00:00
|
|
|
"github.com/hashicorp/consul/acl"
|
2019-08-19 18:03:03 +00:00
|
|
|
"github.com/hashicorp/consul/agent/connect"
|
2018-10-03 18:18:55 +00:00
|
|
|
"github.com/hashicorp/consul/agent/proxycfg"
|
|
|
|
"github.com/hashicorp/consul/agent/structs"
|
2019-03-22 19:37:14 +00:00
|
|
|
"github.com/hashicorp/consul/api"
|
2018-10-03 18:18:55 +00:00
|
|
|
)
|
|
|
|
|
2020-02-19 16:57:55 +00:00
|
|
|
const (
|
|
|
|
UnnamedSubset = ""
|
|
|
|
)
|
|
|
|
|
2018-10-03 18:18:55 +00:00
|
|
|
// endpointsFromSnapshot returns the xDS API representation of the "endpoints"
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
|
2018-10-03 18:18:55 +00:00
|
|
|
if cfgSnap == nil {
|
|
|
|
return nil, errors.New("nil config given")
|
|
|
|
}
|
2019-06-24 19:05:36 +00:00
|
|
|
|
|
|
|
switch cfgSnap.Kind {
|
|
|
|
case structs.ServiceKindConnectProxy:
|
2020-03-27 21:57:16 +00:00
|
|
|
return s.endpointsFromSnapshotConnectProxy(cfgSnap)
|
2020-04-13 16:33:01 +00:00
|
|
|
case structs.ServiceKindTerminatingGateway:
|
|
|
|
return s.endpointsFromSnapshotTerminatingGateway(cfgSnap)
|
2019-06-18 00:52:01 +00:00
|
|
|
case structs.ServiceKindMeshGateway:
|
2020-03-27 21:57:16 +00:00
|
|
|
return s.endpointsFromSnapshotMeshGateway(cfgSnap)
|
2020-04-16 21:00:48 +00:00
|
|
|
case structs.ServiceKindIngressGateway:
|
|
|
|
return s.endpointsFromSnapshotIngressGateway(cfgSnap)
|
2019-06-24 19:05:36 +00:00
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// endpointsFromSnapshotConnectProxy returns the xDS API representation of the "endpoints"
|
|
|
|
// (upstream instances) in the snapshot.
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
|
2022-06-03 21:42:50 +00:00
|
|
|
// TODO: this estimate is wrong
|
2019-08-05 18:30:35 +00:00
|
|
|
resources := make([]proto.Message, 0,
|
2022-06-03 21:42:50 +00:00
|
|
|
len(cfgSnap.ConnectProxy.PreparedQueryEndpoints)+
|
|
|
|
len(cfgSnap.ConnectProxy.PeerUpstreamEndpoints)+
|
|
|
|
len(cfgSnap.ConnectProxy.WatchedUpstreamEndpoints))
|
2019-07-02 03:10:51 +00:00
|
|
|
|
2022-06-01 21:53:52 +00:00
|
|
|
// NOTE: Any time we skip a chain below we MUST also skip that discovery chain in clusters.go
|
|
|
|
// so that the sets of endpoints generated matches the sets of clusters.
|
2022-01-20 16:12:04 +00:00
|
|
|
for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain {
|
|
|
|
upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid]
|
2021-12-13 22:30:49 +00:00
|
|
|
|
|
|
|
explicit := upstreamCfg.HasLocalPortOrSocket()
|
2022-01-20 16:12:04 +00:00
|
|
|
if _, implicit := cfgSnap.ConnectProxy.IntentionUpstreams[uid]; !implicit && !explicit {
|
2021-12-13 22:30:49 +00:00
|
|
|
// Discovery chain is not associated with a known explicit or implicit upstream so it is skipped.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2021-03-17 19:40:49 +00:00
|
|
|
es := s.endpointsFromDiscoveryChain(
|
2022-01-20 16:12:04 +00:00
|
|
|
uid,
|
2021-03-17 19:40:49 +00:00
|
|
|
chain,
|
2021-10-29 00:47:42 +00:00
|
|
|
cfgSnap.Locality,
|
2021-12-13 22:30:49 +00:00
|
|
|
upstreamCfg,
|
2022-01-20 16:12:04 +00:00
|
|
|
cfgSnap.ConnectProxy.WatchedUpstreamEndpoints[uid],
|
|
|
|
cfgSnap.ConnectProxy.WatchedGatewayEndpoints[uid],
|
2021-03-17 19:40:49 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, es...)
|
|
|
|
}
|
|
|
|
|
2022-06-03 21:42:50 +00:00
|
|
|
// NOTE: Any time we skip an upstream below we MUST also skip that same
|
|
|
|
// upstream in clusters.go so that the sets of endpoints generated matches
|
|
|
|
// the sets of clusters.
|
|
|
|
//
|
|
|
|
// TODO(peering): make this work for tproxy
|
|
|
|
for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() {
|
|
|
|
upstreamCfg := cfgSnap.ConnectProxy.UpstreamConfig[uid]
|
|
|
|
|
|
|
|
explicit := upstreamCfg.HasLocalPortOrSocket()
|
|
|
|
if _, implicit := cfgSnap.ConnectProxy.IntentionUpstreams[uid]; !implicit && !explicit {
|
|
|
|
// Not associated with a known explicit or implicit upstream so it is skipped.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
peerMeta := cfgSnap.ConnectProxy.UpstreamPeerMeta(uid)
|
|
|
|
|
|
|
|
// TODO(peering): if we replicated service metadata separately from the
|
|
|
|
// instances we wouldn't have to flip/flop this cluster name like this.
|
|
|
|
clusterName := peerMeta.PrimarySNI()
|
|
|
|
if clusterName == "" {
|
|
|
|
clusterName = uid.EnvoyID()
|
|
|
|
}
|
|
|
|
|
|
|
|
endpoints, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpoints[uid]
|
|
|
|
if ok {
|
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
|
|
|
[]loadAssignmentEndpointGroup{
|
|
|
|
{Endpoints: endpoints},
|
|
|
|
},
|
|
|
|
cfgSnap.Locality,
|
|
|
|
)
|
|
|
|
resources = append(resources, la)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-17 21:17:43 +00:00
|
|
|
// Looping over explicit upstreams is only needed for prepared queries because they do not have discovery chains
|
2019-07-02 03:10:51 +00:00
|
|
|
for _, u := range cfgSnap.Proxy.Upstreams {
|
2021-03-17 21:17:43 +00:00
|
|
|
if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
|
2021-03-17 19:40:49 +00:00
|
|
|
continue
|
|
|
|
}
|
2022-01-20 16:12:04 +00:00
|
|
|
uid := proxycfg.NewUpstreamID(&u)
|
2019-07-02 03:10:51 +00:00
|
|
|
|
2021-03-17 19:40:49 +00:00
|
|
|
dc := u.Datacenter
|
|
|
|
if dc == "" {
|
|
|
|
dc = cfgSnap.Datacenter
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
2021-03-17 19:40:49 +00:00
|
|
|
clusterName := connect.UpstreamSNI(&u, "", dc, cfgSnap.Roots.TrustDomain)
|
2019-07-02 03:10:51 +00:00
|
|
|
|
2022-01-20 16:12:04 +00:00
|
|
|
endpoints, ok := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid]
|
2021-03-17 19:40:49 +00:00
|
|
|
if ok {
|
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
|
|
|
[]loadAssignmentEndpointGroup{
|
|
|
|
{Endpoints: endpoints},
|
|
|
|
},
|
2021-10-29 00:47:42 +00:00
|
|
|
cfgSnap.Locality,
|
2020-04-16 21:00:48 +00:00
|
|
|
)
|
2021-03-17 19:40:49 +00:00
|
|
|
resources = append(resources, la)
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
2019-06-18 00:52:01 +00:00
|
|
|
}
|
2019-07-02 03:10:51 +00:00
|
|
|
|
2019-06-18 00:52:01 +00:00
|
|
|
return resources, nil
|
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) filterSubsetEndpoints(subset *structs.ServiceResolverSubset, endpoints structs.CheckServiceNodes) (structs.CheckServiceNodes, error) {
|
2020-02-19 16:57:55 +00:00
|
|
|
// locally execute the subsets filter
|
|
|
|
if subset.Filter != "" {
|
|
|
|
filter, err := bexpr.CreateFilter(subset.Filter, nil, endpoints)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
raw, err := filter.Execute(endpoints)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return raw.(structs.CheckServiceNodes), nil
|
|
|
|
}
|
|
|
|
return endpoints, nil
|
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromSnapshotTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
|
2020-04-14 14:59:23 +00:00
|
|
|
return s.endpointsFromServicesAndResolvers(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers)
|
2020-04-13 16:33:01 +00:00
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
|
2021-10-26 21:58:23 +00:00
|
|
|
keys := cfgSnap.MeshGateway.GatewayKeys()
|
2021-10-23 20:17:29 +00:00
|
|
|
resources := make([]proto.Message, 0, len(keys)+len(cfgSnap.MeshGateway.ServiceGroups))
|
2019-06-18 00:52:01 +00:00
|
|
|
|
2021-10-23 20:17:29 +00:00
|
|
|
for _, key := range keys {
|
2021-10-29 00:41:58 +00:00
|
|
|
if key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrDefault()) {
|
2021-10-26 22:25:35 +00:00
|
|
|
continue // skip local
|
|
|
|
}
|
2021-10-26 21:58:23 +00:00
|
|
|
// Also skip gateways with a hostname as their address. EDS cannot resolve hostnames,
|
|
|
|
// so we provide them through CDS instead.
|
2021-10-26 22:25:35 +00:00
|
|
|
if len(cfgSnap.MeshGateway.HostnameDatacenters[key.String()]) > 0 {
|
2020-06-03 21:28:45 +00:00
|
|
|
continue
|
2020-03-09 20:59:02 +00:00
|
|
|
}
|
2020-06-03 21:28:45 +00:00
|
|
|
|
2021-11-09 16:45:36 +00:00
|
|
|
// Mesh gateways in remote DCs are discovered in two ways:
|
|
|
|
//
|
|
|
|
// 1. Via an Internal.ServiceDump RPC in the remote DC (GatewayGroups).
|
|
|
|
// 2. In the federation state that is replicated from the primary DC (FedStateGateways).
|
|
|
|
//
|
|
|
|
// We determine which set to use based on whichever contains the highest
|
|
|
|
// raft ModifyIndex (and is therefore most up-to-date).
|
|
|
|
//
|
|
|
|
// Previously, GatewayGroups was always given presedence over FedStateGateways
|
|
|
|
// but this was problematic when using mesh gateways for WAN federation.
|
|
|
|
//
|
|
|
|
// Consider the following example:
|
|
|
|
//
|
|
|
|
// - Primary and Secondary DCs are WAN Federated via local mesh gateways.
|
|
|
|
//
|
|
|
|
// - Secondary DC's mesh gateway is running on an ephemeral compute instance
|
|
|
|
// and is abruptly terminated and rescheduled with a *new IP address*.
|
|
|
|
//
|
|
|
|
// - Primary DC's mesh gateway is no longer able to connect to the Secondary
|
|
|
|
// DC as its proxy is configured with the old IP address. Therefore any RPC
|
|
|
|
// from the Primary to the Secondary DC will fail (including the one to
|
|
|
|
// discover the gateway's new IP address).
|
|
|
|
//
|
|
|
|
// - Secondary DC performs its regular anti-entropy of federation state data
|
|
|
|
// to the Primary DC (this succeeds as there is still connectivity in this
|
|
|
|
// direction).
|
|
|
|
//
|
|
|
|
// - At this point the Primary DC's mesh gateway should observe the new IP
|
|
|
|
// address and reconfigure its proxy, however as we always prioritised
|
|
|
|
// GatewayGroups this didn't happen and the connection remained severed.
|
|
|
|
maxModifyIndex := func(vals structs.CheckServiceNodes) uint64 {
|
|
|
|
var max uint64
|
|
|
|
for _, v := range vals {
|
|
|
|
if i := v.Service.RaftIndex.ModifyIndex; i > max {
|
|
|
|
max = i
|
|
|
|
}
|
2020-03-09 20:59:02 +00:00
|
|
|
}
|
2021-11-09 16:45:36 +00:00
|
|
|
return max
|
|
|
|
}
|
|
|
|
|
|
|
|
endpoints := cfgSnap.MeshGateway.GatewayGroups[key.String()]
|
|
|
|
fedStateEndpoints := cfgSnap.MeshGateway.FedStateGateways[key.String()]
|
|
|
|
|
|
|
|
if maxModifyIndex(fedStateEndpoints) > maxModifyIndex(endpoints) {
|
|
|
|
endpoints = fedStateEndpoints
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(endpoints) == 0 {
|
|
|
|
s.Logger.Error("skipping mesh gateway endpoints because no definition found", "datacenter", key)
|
|
|
|
continue
|
2020-03-09 20:59:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
{ // standard connect
|
2021-10-24 15:16:28 +00:00
|
|
|
clusterName := connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain)
|
2020-03-09 20:59:02 +00:00
|
|
|
|
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
|
|
|
[]loadAssignmentEndpointGroup{
|
|
|
|
{Endpoints: endpoints},
|
|
|
|
},
|
2021-10-29 00:47:42 +00:00
|
|
|
cfgSnap.Locality,
|
2020-03-09 20:59:02 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, la)
|
|
|
|
}
|
|
|
|
|
2021-10-26 22:10:30 +00:00
|
|
|
if cfgSnap.ProxyID.InDefaultPartition() &&
|
2021-10-26 21:58:23 +00:00
|
|
|
cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" &&
|
|
|
|
cfgSnap.ServerSNIFn != nil {
|
2020-03-09 20:59:02 +00:00
|
|
|
|
2021-10-26 21:58:23 +00:00
|
|
|
clusterName := cfgSnap.ServerSNIFn(key.Datacenter, "")
|
2020-03-09 20:59:02 +00:00
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
|
|
|
[]loadAssignmentEndpointGroup{
|
|
|
|
{Endpoints: endpoints},
|
|
|
|
},
|
2021-10-29 00:47:42 +00:00
|
|
|
cfgSnap.Locality,
|
2020-03-09 20:59:02 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, la)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-03 21:28:45 +00:00
|
|
|
// generate endpoints for our servers if WAN federation is enabled
|
2021-10-26 22:10:30 +00:00
|
|
|
if cfgSnap.ProxyID.InDefaultPartition() &&
|
2021-10-26 21:58:23 +00:00
|
|
|
cfgSnap.ServiceMeta[structs.MetaWANFederationKey] == "1" &&
|
|
|
|
cfgSnap.ServerSNIFn != nil {
|
2021-02-26 22:23:15 +00:00
|
|
|
var allServersLbEndpoints []*envoy_endpoint_v3.LbEndpoint
|
2020-03-09 20:59:02 +00:00
|
|
|
|
|
|
|
for _, srv := range cfgSnap.MeshGateway.ConsulServers {
|
|
|
|
clusterName := cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node)
|
|
|
|
|
2022-01-28 06:49:06 +00:00
|
|
|
_, addr, port := srv.BestAddress(false /*wan*/)
|
2020-03-09 20:59:02 +00:00
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
lbEndpoint := &envoy_endpoint_v3.LbEndpoint{
|
|
|
|
HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{
|
|
|
|
Endpoint: &envoy_endpoint_v3.Endpoint{
|
2020-06-23 20:19:56 +00:00
|
|
|
Address: makeAddress(addr, port),
|
2020-03-09 20:59:02 +00:00
|
|
|
},
|
|
|
|
},
|
2021-02-26 22:23:15 +00:00
|
|
|
HealthStatus: envoy_core_v3.HealthStatus_UNKNOWN,
|
2020-03-09 20:59:02 +00:00
|
|
|
}
|
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
cla := &envoy_endpoint_v3.ClusterLoadAssignment{
|
2020-03-09 20:59:02 +00:00
|
|
|
ClusterName: clusterName,
|
2021-02-26 22:23:15 +00:00
|
|
|
Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{{
|
|
|
|
LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{lbEndpoint},
|
2020-03-09 20:59:02 +00:00
|
|
|
}},
|
|
|
|
}
|
|
|
|
allServersLbEndpoints = append(allServersLbEndpoints, lbEndpoint)
|
|
|
|
|
|
|
|
resources = append(resources, cla)
|
|
|
|
}
|
|
|
|
|
|
|
|
// And add one catch all so that remote datacenters can dial ANY server
|
|
|
|
// in this datacenter without knowing its name.
|
2021-02-26 22:23:15 +00:00
|
|
|
resources = append(resources, &envoy_endpoint_v3.ClusterLoadAssignment{
|
2020-03-09 20:59:02 +00:00
|
|
|
ClusterName: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, ""),
|
2021-02-26 22:23:15 +00:00
|
|
|
Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{{
|
2020-03-09 20:59:02 +00:00
|
|
|
LbEndpoints: allServersLbEndpoints,
|
|
|
|
}},
|
|
|
|
})
|
2019-06-24 19:05:36 +00:00
|
|
|
}
|
2019-06-18 00:52:01 +00:00
|
|
|
|
2020-02-19 16:57:55 +00:00
|
|
|
// Generate the endpoints for each service and its subsets
|
2020-04-14 14:59:23 +00:00
|
|
|
e, err := s.endpointsFromServicesAndResolvers(cfgSnap, cfgSnap.MeshGateway.ServiceGroups, cfgSnap.MeshGateway.ServiceResolvers)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
resources = append(resources, e...)
|
|
|
|
|
|
|
|
return resources, nil
|
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromServicesAndResolvers(
|
2020-04-14 14:59:23 +00:00
|
|
|
cfgSnap *proxycfg.ConfigSnapshot,
|
2020-06-12 14:57:41 +00:00
|
|
|
services map[structs.ServiceName]structs.CheckServiceNodes,
|
2021-04-29 18:54:05 +00:00
|
|
|
resolvers map[structs.ServiceName]*structs.ServiceResolverConfigEntry,
|
|
|
|
) ([]proto.Message, error) {
|
2020-04-14 14:59:23 +00:00
|
|
|
resources := make([]proto.Message, 0, len(services))
|
|
|
|
|
|
|
|
// generate the endpoints for the linked service groups
|
|
|
|
for svc, endpoints := range services {
|
2020-06-03 21:28:45 +00:00
|
|
|
// Skip creating endpoints for services that have hostnames as addresses
|
|
|
|
// EDS cannot resolve hostnames so we provide them through CDS instead
|
|
|
|
if cfgSnap.Kind == structs.ServiceKindTerminatingGateway && len(cfgSnap.TerminatingGateway.HostnameServices[svc]) > 0 {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-04-14 14:59:23 +00:00
|
|
|
clusterEndpoints := make(map[string][]loadAssignmentEndpointGroup)
|
|
|
|
clusterEndpoints[UnnamedSubset] = []loadAssignmentEndpointGroup{{Endpoints: endpoints, OnlyPassing: false}}
|
2020-02-19 16:57:55 +00:00
|
|
|
|
|
|
|
// Collect all of the loadAssignmentEndpointGroups for the various subsets. We do this before generating
|
|
|
|
// the endpoints for the default/unnamed subset so that we can take into account the DefaultSubset on the
|
|
|
|
// service-resolver which may prevent the default/unnamed cluster from creating endpoints for all service
|
|
|
|
// instances.
|
2020-04-14 14:59:23 +00:00
|
|
|
if resolver, hasResolver := resolvers[svc]; hasResolver {
|
2020-02-19 16:57:55 +00:00
|
|
|
for subsetName, subset := range resolver.Subsets {
|
|
|
|
subsetEndpoints, err := s.filterSubsetEndpoints(&subset, endpoints)
|
2019-07-02 13:43:35 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2020-04-14 14:59:23 +00:00
|
|
|
groups := []loadAssignmentEndpointGroup{{Endpoints: subsetEndpoints, OnlyPassing: subset.OnlyPassing}}
|
|
|
|
clusterEndpoints[subsetName] = groups
|
2019-07-02 13:43:35 +00:00
|
|
|
|
2020-02-19 16:57:55 +00:00
|
|
|
// if this subset is the default then override the unnamed subset with this configuration
|
|
|
|
if subsetName == resolver.DefaultSubset {
|
2020-04-14 14:59:23 +00:00
|
|
|
clusterEndpoints[UnnamedSubset] = groups
|
2019-07-02 13:43:35 +00:00
|
|
|
}
|
|
|
|
}
|
2020-02-19 16:57:55 +00:00
|
|
|
}
|
2019-07-02 13:43:35 +00:00
|
|
|
|
2020-02-19 16:57:55 +00:00
|
|
|
// now generate the load assignment for all subsets
|
2020-04-14 14:59:23 +00:00
|
|
|
for subsetName, groups := range clusterEndpoints {
|
2021-09-01 14:35:39 +00:00
|
|
|
clusterName := connect.ServiceSNI(svc.Name, subsetName, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
|
2019-07-02 13:43:35 +00:00
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
2020-04-14 14:59:23 +00:00
|
|
|
groups,
|
2021-10-29 00:47:42 +00:00
|
|
|
cfgSnap.Locality,
|
2019-07-02 13:43:35 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, la)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-03 18:18:55 +00:00
|
|
|
return resources, nil
|
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
|
2020-04-16 21:00:48 +00:00
|
|
|
var resources []proto.Message
|
2022-01-20 16:12:04 +00:00
|
|
|
createdClusters := make(map[proxycfg.UpstreamID]bool)
|
2020-04-16 23:24:11 +00:00
|
|
|
for _, upstreams := range cfgSnap.IngressGateway.Upstreams {
|
|
|
|
for _, u := range upstreams {
|
2022-01-20 16:12:04 +00:00
|
|
|
uid := proxycfg.NewUpstreamID(&u)
|
2020-04-16 21:00:48 +00:00
|
|
|
|
2020-04-21 21:06:23 +00:00
|
|
|
// If we've already created endpoints for this upstream, skip it. Multiple listeners may
|
|
|
|
// reference the same upstream, so we don't need to create duplicate endpoints in that case.
|
2022-01-20 16:12:04 +00:00
|
|
|
if createdClusters[uid] {
|
2020-04-21 21:06:23 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-04-16 23:24:11 +00:00
|
|
|
es := s.endpointsFromDiscoveryChain(
|
2022-01-20 16:12:04 +00:00
|
|
|
uid,
|
|
|
|
cfgSnap.IngressGateway.DiscoveryChain[uid],
|
2021-10-22 21:30:42 +00:00
|
|
|
proxycfg.GatewayKey{Datacenter: cfgSnap.Datacenter, Partition: u.DestinationPartition},
|
2021-03-17 19:40:49 +00:00
|
|
|
&u,
|
2022-01-20 16:12:04 +00:00
|
|
|
cfgSnap.IngressGateway.WatchedUpstreamEndpoints[uid],
|
|
|
|
cfgSnap.IngressGateway.WatchedGatewayEndpoints[uid],
|
2020-04-16 23:24:11 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, es...)
|
2022-01-20 16:12:04 +00:00
|
|
|
createdClusters[uid] = true
|
2020-04-16 23:24:11 +00:00
|
|
|
}
|
2020-04-16 21:00:48 +00:00
|
|
|
}
|
|
|
|
return resources, nil
|
|
|
|
}
|
|
|
|
|
2021-05-04 04:43:55 +00:00
|
|
|
// used in clusters.go
|
2021-02-26 22:23:15 +00:00
|
|
|
func makeEndpoint(host string, port int) *envoy_endpoint_v3.LbEndpoint {
|
|
|
|
return &envoy_endpoint_v3.LbEndpoint{
|
|
|
|
HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{
|
|
|
|
Endpoint: &envoy_endpoint_v3.Endpoint{
|
2020-06-23 20:19:56 +00:00
|
|
|
Address: makeAddress(host, port),
|
2019-07-02 03:10:51 +00:00
|
|
|
},
|
2018-10-03 18:18:55 +00:00
|
|
|
},
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
2018-10-03 18:18:55 +00:00
|
|
|
}
|
|
|
|
|
2021-05-04 04:43:55 +00:00
|
|
|
func makePipeEndpoint(path string) *envoy_endpoint_v3.LbEndpoint {
|
|
|
|
return &envoy_endpoint_v3.LbEndpoint{
|
|
|
|
HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{
|
|
|
|
Endpoint: &envoy_endpoint_v3.Endpoint{
|
|
|
|
Address: makePipeAddress(path, 0),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-29 18:54:05 +00:00
|
|
|
func (s *ResourceGenerator) endpointsFromDiscoveryChain(
|
2022-01-20 16:12:04 +00:00
|
|
|
uid proxycfg.UpstreamID,
|
2020-04-16 21:00:48 +00:00
|
|
|
chain *structs.CompiledDiscoveryChain,
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayKey proxycfg.GatewayKey,
|
2021-03-17 19:40:49 +00:00
|
|
|
upstream *structs.Upstream,
|
2021-10-22 21:30:42 +00:00
|
|
|
upstreamEndpoints map[string]structs.CheckServiceNodes,
|
|
|
|
gatewayEndpoints map[string]structs.CheckServiceNodes,
|
2020-04-16 21:00:48 +00:00
|
|
|
) []proto.Message {
|
|
|
|
var resources []proto.Message
|
|
|
|
|
|
|
|
if chain == nil {
|
|
|
|
return resources
|
|
|
|
}
|
|
|
|
|
2021-03-17 19:40:49 +00:00
|
|
|
configMap := make(map[string]interface{})
|
|
|
|
if upstream != nil {
|
|
|
|
configMap = upstream.Config
|
|
|
|
}
|
|
|
|
cfg, err := structs.ParseUpstreamConfigNoDefaults(configMap)
|
2020-05-26 08:57:22 +00:00
|
|
|
if err != nil {
|
|
|
|
// Don't hard fail on a config typo, just warn. The parse func returns
|
|
|
|
// default config if there is an error so it's safe to continue.
|
2022-01-20 16:12:04 +00:00
|
|
|
s.Logger.Warn("failed to parse", "upstream", uid,
|
2020-05-26 08:57:22 +00:00
|
|
|
"error", err)
|
|
|
|
}
|
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
var escapeHatchCluster *envoy_cluster_v3.Cluster
|
2021-03-15 20:12:57 +00:00
|
|
|
if cfg.EnvoyClusterJSON != "" {
|
2022-03-30 15:04:18 +00:00
|
|
|
if chain.Default {
|
2020-05-26 08:57:22 +00:00
|
|
|
// If you haven't done anything to setup the discovery chain, then
|
|
|
|
// you can use the envoy_cluster_json escape hatch.
|
2021-03-15 20:12:57 +00:00
|
|
|
escapeHatchCluster, err = makeClusterFromUserConfig(cfg.EnvoyClusterJSON)
|
2020-05-26 08:57:22 +00:00
|
|
|
if err != nil {
|
|
|
|
return resources
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
s.Logger.Warn("ignoring escape hatch setting, because a discovery chain is configued for",
|
2022-01-20 16:12:04 +00:00
|
|
|
"discovery chain", chain.ServiceName, "upstream", uid,
|
2020-05-26 08:57:22 +00:00
|
|
|
"envoy_cluster_json", chain.ServiceName)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-16 21:00:48 +00:00
|
|
|
// Find all resolver nodes.
|
|
|
|
for _, node := range chain.Nodes {
|
|
|
|
if node.Type != structs.DiscoveryGraphNodeTypeResolver {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
failover := node.Resolver.Failover
|
|
|
|
targetID := node.Resolver.Target
|
|
|
|
|
|
|
|
target := chain.Targets[targetID]
|
|
|
|
|
|
|
|
clusterName := CustomizeClusterName(target.Name, chain)
|
2020-05-26 08:57:22 +00:00
|
|
|
if escapeHatchCluster != nil {
|
|
|
|
clusterName = escapeHatchCluster.Name
|
|
|
|
}
|
|
|
|
s.Logger.Debug("generating endpoints for", "cluster", clusterName)
|
2020-04-16 21:00:48 +00:00
|
|
|
|
|
|
|
// Determine if we have to generate the entire cluster differently.
|
|
|
|
failoverThroughMeshGateway := chain.WillFailoverThroughMeshGateway(node)
|
|
|
|
|
|
|
|
if failoverThroughMeshGateway {
|
|
|
|
actualTargetID := firstHealthyTarget(
|
|
|
|
chain.Targets,
|
|
|
|
upstreamEndpoints,
|
|
|
|
targetID,
|
|
|
|
failover.Targets,
|
|
|
|
)
|
|
|
|
if actualTargetID != targetID {
|
|
|
|
targetID = actualTargetID
|
|
|
|
}
|
|
|
|
|
|
|
|
failover = nil
|
|
|
|
}
|
|
|
|
|
|
|
|
primaryGroup, valid := makeLoadAssignmentEndpointGroup(
|
|
|
|
chain.Targets,
|
|
|
|
upstreamEndpoints,
|
|
|
|
gatewayEndpoints,
|
|
|
|
targetID,
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayKey,
|
2020-04-16 21:00:48 +00:00
|
|
|
)
|
|
|
|
if !valid {
|
|
|
|
continue // skip the cluster if we're still populating the snapshot
|
|
|
|
}
|
|
|
|
|
|
|
|
var endpointGroups []loadAssignmentEndpointGroup
|
|
|
|
|
|
|
|
if failover != nil && len(failover.Targets) > 0 {
|
|
|
|
endpointGroups = make([]loadAssignmentEndpointGroup, 0, len(failover.Targets)+1)
|
|
|
|
|
|
|
|
endpointGroups = append(endpointGroups, primaryGroup)
|
|
|
|
|
|
|
|
for _, failTargetID := range failover.Targets {
|
|
|
|
failoverGroup, valid := makeLoadAssignmentEndpointGroup(
|
|
|
|
chain.Targets,
|
|
|
|
upstreamEndpoints,
|
|
|
|
gatewayEndpoints,
|
|
|
|
failTargetID,
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayKey,
|
2020-04-16 21:00:48 +00:00
|
|
|
)
|
|
|
|
if !valid {
|
|
|
|
continue // skip the failover target if we're still populating the snapshot
|
|
|
|
}
|
|
|
|
endpointGroups = append(endpointGroups, failoverGroup)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
endpointGroups = append(endpointGroups, primaryGroup)
|
|
|
|
}
|
|
|
|
|
|
|
|
la := makeLoadAssignment(
|
|
|
|
clusterName,
|
|
|
|
endpointGroups,
|
2021-10-29 00:41:58 +00:00
|
|
|
gatewayKey,
|
2020-04-16 21:00:48 +00:00
|
|
|
)
|
|
|
|
resources = append(resources, la)
|
|
|
|
}
|
|
|
|
|
|
|
|
return resources
|
|
|
|
}
|
|
|
|
|
2019-07-24 01:20:24 +00:00
|
|
|
type loadAssignmentEndpointGroup struct {
|
2019-08-05 18:30:35 +00:00
|
|
|
Endpoints structs.CheckServiceNodes
|
|
|
|
OnlyPassing bool
|
2021-02-26 22:23:15 +00:00
|
|
|
OverrideHealth envoy_core_v3.HealthStatus
|
2019-07-24 01:20:24 +00:00
|
|
|
}
|
|
|
|
|
2021-10-29 00:41:58 +00:00
|
|
|
func makeLoadAssignment(clusterName string, endpointGroups []loadAssignmentEndpointGroup, localKey proxycfg.GatewayKey) *envoy_endpoint_v3.ClusterLoadAssignment {
|
2021-02-26 22:23:15 +00:00
|
|
|
cla := &envoy_endpoint_v3.ClusterLoadAssignment{
|
2019-07-02 03:10:51 +00:00
|
|
|
ClusterName: clusterName,
|
2021-02-26 22:23:15 +00:00
|
|
|
Endpoints: make([]*envoy_endpoint_v3.LocalityLbEndpoints, 0, len(endpointGroups)),
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
2019-08-02 20:34:54 +00:00
|
|
|
|
|
|
|
if len(endpointGroups) > 1 {
|
2021-02-26 22:23:15 +00:00
|
|
|
cla.Policy = &envoy_endpoint_v3.ClusterLoadAssignment_Policy{
|
2019-08-02 20:34:54 +00:00
|
|
|
// We choose such a large value here that the failover math should
|
|
|
|
// in effect not happen until zero instances are healthy.
|
|
|
|
OverprovisioningFactor: makeUint32Value(100000),
|
2019-03-22 19:37:14 +00:00
|
|
|
}
|
2019-07-02 03:10:51 +00:00
|
|
|
}
|
|
|
|
|
2019-07-24 01:20:24 +00:00
|
|
|
for priority, endpointGroup := range endpointGroups {
|
|
|
|
endpoints := endpointGroup.Endpoints
|
2021-02-26 22:23:15 +00:00
|
|
|
es := make([]*envoy_endpoint_v3.LbEndpoint, 0, len(endpoints))
|
2019-07-02 03:10:51 +00:00
|
|
|
|
|
|
|
for _, ep := range endpoints {
|
|
|
|
// TODO (mesh-gateway) - should we respect the translate_wan_addrs configuration here or just always use the wan for cross-dc?
|
2022-01-28 06:49:06 +00:00
|
|
|
_, addr, port := ep.BestAddress(!localKey.Matches(ep.Node.Datacenter, ep.Node.PartitionOrDefault()))
|
2019-08-05 18:30:35 +00:00
|
|
|
healthStatus, weight := calculateEndpointHealthAndWeight(ep, endpointGroup.OnlyPassing)
|
2019-03-22 19:37:14 +00:00
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
if endpointGroup.OverrideHealth != envoy_core_v3.HealthStatus_UNKNOWN {
|
2019-08-05 18:30:35 +00:00
|
|
|
healthStatus = endpointGroup.OverrideHealth
|
2019-03-22 19:37:14 +00:00
|
|
|
}
|
2019-08-05 18:30:35 +00:00
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
es = append(es, &envoy_endpoint_v3.LbEndpoint{
|
|
|
|
HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{
|
|
|
|
Endpoint: &envoy_endpoint_v3.Endpoint{
|
2020-06-23 20:19:56 +00:00
|
|
|
Address: makeAddress(addr, port),
|
2019-07-02 03:10:51 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
HealthStatus: healthStatus,
|
|
|
|
LoadBalancingWeight: makeUint32Value(weight),
|
|
|
|
})
|
2019-03-22 19:37:14 +00:00
|
|
|
}
|
2019-07-02 03:10:51 +00:00
|
|
|
|
2021-02-26 22:23:15 +00:00
|
|
|
cla.Endpoints = append(cla.Endpoints, &envoy_endpoint_v3.LocalityLbEndpoints{
|
2019-07-02 03:10:51 +00:00
|
|
|
Priority: uint32(priority),
|
2018-10-03 18:18:55 +00:00
|
|
|
LbEndpoints: es,
|
2019-07-02 03:10:51 +00:00
|
|
|
})
|
2018-10-03 18:18:55 +00:00
|
|
|
}
|
2019-07-02 03:10:51 +00:00
|
|
|
|
|
|
|
return cla
|
2018-10-03 18:18:55 +00:00
|
|
|
}
|
2019-08-05 18:30:35 +00:00
|
|
|
|
|
|
|
func makeLoadAssignmentEndpointGroup(
|
|
|
|
targets map[string]*structs.DiscoveryTarget,
|
|
|
|
targetHealth map[string]structs.CheckServiceNodes,
|
|
|
|
gatewayHealth map[string]structs.CheckServiceNodes,
|
|
|
|
targetID string,
|
2021-10-22 21:30:42 +00:00
|
|
|
localKey proxycfg.GatewayKey,
|
2019-08-05 18:30:35 +00:00
|
|
|
) (loadAssignmentEndpointGroup, bool) {
|
|
|
|
realEndpoints, ok := targetHealth[targetID]
|
|
|
|
if !ok {
|
|
|
|
// skip the cluster if we're still populating the snapshot
|
|
|
|
return loadAssignmentEndpointGroup{}, false
|
|
|
|
}
|
|
|
|
target := targets[targetID]
|
|
|
|
|
2021-10-22 21:30:42 +00:00
|
|
|
var gatewayKey proxycfg.GatewayKey
|
|
|
|
|
2019-08-05 18:30:35 +00:00
|
|
|
switch target.MeshGateway.Mode {
|
|
|
|
case structs.MeshGatewayModeRemote:
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayKey.Datacenter = target.Datacenter
|
|
|
|
gatewayKey.Partition = target.Partition
|
|
|
|
|
2019-08-05 18:30:35 +00:00
|
|
|
case structs.MeshGatewayModeLocal:
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayKey = localKey
|
2019-08-05 18:30:35 +00:00
|
|
|
}
|
|
|
|
|
2022-04-05 21:10:06 +00:00
|
|
|
if gatewayKey.IsEmpty() || (acl.EqualPartitions(localKey.Partition, target.Partition) && localKey.Datacenter == target.Datacenter) {
|
2021-10-22 21:30:42 +00:00
|
|
|
// Gateways are not needed if the request isn't for a remote DC or partition.
|
2019-08-05 18:30:35 +00:00
|
|
|
return loadAssignmentEndpointGroup{
|
|
|
|
Endpoints: realEndpoints,
|
|
|
|
OnlyPassing: target.Subset.OnlyPassing,
|
|
|
|
}, true
|
|
|
|
}
|
|
|
|
|
|
|
|
// If using a mesh gateway we need to pull those endpoints instead.
|
2021-10-22 21:30:42 +00:00
|
|
|
gatewayEndpoints, ok := gatewayHealth[gatewayKey.String()]
|
2019-08-05 18:30:35 +00:00
|
|
|
if !ok {
|
|
|
|
// skip the cluster if we're still populating the snapshot
|
|
|
|
return loadAssignmentEndpointGroup{}, false
|
|
|
|
}
|
|
|
|
|
|
|
|
// But we will use the health from the actual backend service.
|
2021-02-26 22:23:15 +00:00
|
|
|
overallHealth := envoy_core_v3.HealthStatus_UNHEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
for _, ep := range realEndpoints {
|
|
|
|
health, _ := calculateEndpointHealthAndWeight(ep, target.Subset.OnlyPassing)
|
2021-02-26 22:23:15 +00:00
|
|
|
if health == envoy_core_v3.HealthStatus_HEALTHY {
|
|
|
|
overallHealth = envoy_core_v3.HealthStatus_HEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return loadAssignmentEndpointGroup{
|
|
|
|
Endpoints: gatewayEndpoints,
|
|
|
|
OverrideHealth: overallHealth,
|
|
|
|
}, true
|
|
|
|
}
|
|
|
|
|
|
|
|
func calculateEndpointHealthAndWeight(
|
|
|
|
ep structs.CheckServiceNode,
|
|
|
|
onlyPassing bool,
|
2021-02-26 22:23:15 +00:00
|
|
|
) (envoy_core_v3.HealthStatus, int) {
|
|
|
|
healthStatus := envoy_core_v3.HealthStatus_HEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
weight := 1
|
|
|
|
if ep.Service.Weights != nil {
|
|
|
|
weight = ep.Service.Weights.Passing
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, chk := range ep.Checks {
|
|
|
|
if chk.Status == api.HealthCritical {
|
2021-02-26 22:23:15 +00:00
|
|
|
healthStatus = envoy_core_v3.HealthStatus_UNHEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
}
|
|
|
|
if onlyPassing && chk.Status != api.HealthPassing {
|
2021-02-26 22:23:15 +00:00
|
|
|
healthStatus = envoy_core_v3.HealthStatus_UNHEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
}
|
|
|
|
if chk.Status == api.HealthWarning && ep.Service.Weights != nil {
|
|
|
|
weight = ep.Service.Weights.Warning
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Make weights fit Envoy's limits. A zero weight means that either Warning
|
|
|
|
// (likely) or Passing (weirdly) weight has been set to 0 effectively making
|
|
|
|
// this instance unhealthy and should not be sent traffic.
|
|
|
|
if weight < 1 {
|
2021-02-26 22:23:15 +00:00
|
|
|
healthStatus = envoy_core_v3.HealthStatus_UNHEALTHY
|
2019-08-05 18:30:35 +00:00
|
|
|
weight = 1
|
|
|
|
}
|
|
|
|
if weight > 128 {
|
|
|
|
weight = 128
|
|
|
|
}
|
|
|
|
return healthStatus, weight
|
|
|
|
}
|