status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: tgtwy xDS generation for destinations 

Signed-off-by: Dhia Ayachi <dhia@hashicorp.com>
This commit is contained in:
DanStough 2022-05-24 12:51:52 -04:00 committed by Dan Stough
parent bd4ddb3720
commit 4b402e3119
12 changed files with 720 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
agent/consul/config_endpoint_test.go
View File

@ -1133,6 +1133,31 @@ func TestConfigEntry_ResolveServiceConfig_TransparentProxy(t *testing.T) {
TransparentProxy: structs.TransparentProxyConfig{OutboundListenerPort: 808},
},
},
{
name: "from service-defaults with endpoint",
entries: []structs.ConfigEntry{
&structs.ServiceConfigEntry{
Kind: structs.ServiceDefaults,
Name: "foo",
Mode: structs.ProxyModeTransparent,
Destination: &structs.DestinationConfig{
Address: "hello.world.com",
Port: 443,
},
},
},
request: structs.ServiceConfigRequest{
Name: "foo",
Datacenter: "dc1",
},
expect: structs.ServiceConfigResponse{
Mode: structs.ProxyModeTransparent,
Destination: structs.DestinationConfig{
Address: "hello.world.com",
Port: 443,
},
},
},
{
name: "service-defaults overrides proxy-defaults",
entries: []structs.ConfigEntry{
@ -1207,11 +1232,10 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
wildcard := structs.NewServiceID(structs.WildcardSpecifier, structs.WildcardEnterpriseMetaInDefaultPartition())
tt := []struct {
name string
entries []structs.ConfigEntry
request structs.ServiceConfigRequest
proxyCfg structs.ConnectProxyConfig
expect structs.ServiceConfigResponse
name string
entries []structs.ConfigEntry
request structs.ServiceConfigRequest
expect structs.ServiceConfigResponse
}{
{
name: "upstream config entries from Upstreams and service-defaults",

3
agent/consul/merge_service_config.go
View File

@ -155,6 +155,9 @@ func computeResolvedServiceConfig(
if serviceConf.Mode != structs.ProxyModeDefault {
thisReply.Mode = serviceConf.Mode
}
if serviceConf.Destination != nil {
thisReply.Destination = *serviceConf.Destination
}
thisReply.Meta = serviceConf.Meta
}

34
agent/proxycfg/snapshot.go
View File

@ -230,8 +230,14 @@ type configSnapshotTerminatingGateway struct {
// GatewayServices is a map of service name to the config entry association
// between the gateway and a service. TLS configuration stored here is
// used for TLS origination from the gateway to the linked service.
// This map does not include GatewayServices that represent Endpoints to external
// destinations.
GatewayServices map[structs.ServiceName]structs.GatewayService
// DestinationServices is a map of service name to GatewayServices that represent
// a destination to an external destination of the service mesh.
DestinationServices map[structs.ServiceName]structs.GatewayService
// HostnameServices is a map of service name to service instances with a hostname as the address.
// If hostnames are configured they must be provided to Envoy via CDS not EDS.
HostnameServices map[structs.ServiceName]structs.CheckServiceNodes
@ -269,6 +275,33 @@ func (c *configSnapshotTerminatingGateway) ValidServices() []structs.ServiceName
return out
}
// ValidDestinations returns the list of service keys (that represent exclusively endpoints) that have enough data to be emitted.
func (c *configSnapshotTerminatingGateway) ValidDestinations() []structs.ServiceName {
out := make([]structs.ServiceName, 0, len(c.DestinationServices))
for svc := range c.DestinationServices {
// It only counts if ALL of our watches have come back (with data or not).
// Skip the service if we don't have a cert to present for mTLS.
if cert, ok := c.ServiceLeaves[svc]; !ok || cert == nil {
continue
}
// Skip the service if we haven't gotten our intentions yet.
if _, intentionsSet := c.Intentions[svc]; !intentionsSet {
continue
}
// Skip the service if we haven't gotten our service config yet to know
// the protocol.
if _, ok := c.ServiceConfigs[svc]; !ok || c.ServiceConfigs[svc].Destination.Address == "" {
continue
}
out = append(out, svc)
}
return out
}
// isEmpty is a test helper
func (c *configSnapshotTerminatingGateway) isEmpty() bool {
if c == nil {
@ -286,6 +319,7 @@ func (c *configSnapshotTerminatingGateway) isEmpty() bool {
len(c.ServiceConfigs) == 0 &&
len(c.WatchedConfigs) == 0 &&
len(c.GatewayServices) == 0 &&
len(c.DestinationServices) == 0 &&
len(c.HostnameServices) == 0 &&
!c.MeshConfigSet
}

20
agent/proxycfg/terminating_gateway.go
View File

@ -63,6 +63,7 @@ func (s *handlerTerminatingGateway) initialize(ctx context.Context) (ConfigSnaps
snap.TerminatingGateway.ServiceResolversSet = make(map[structs.ServiceName]bool)
snap.TerminatingGateway.ServiceGroups = make(map[structs.ServiceName]structs.CheckServiceNodes)
snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceName]structs.GatewayService)
snap.TerminatingGateway.DestinationServices = make(map[structs.ServiceName]structs.GatewayService)
snap.TerminatingGateway.HostnameServices = make(map[structs.ServiceName]structs.CheckServiceNodes)
return snap, nil
}
@ -111,10 +112,15 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
svcMap[svc.Service] = struct{}{}
// Store the gateway <-> service mapping for TLS origination
snap.TerminatingGateway.GatewayServices[svc.Service] = *svc
if svc.ServiceKind == structs.GatewayServiceKindDestination {
snap.TerminatingGateway.DestinationServices[svc.Service] = *svc
} else {
snap.TerminatingGateway.GatewayServices[svc.Service] = *svc
}
// Watch the health endpoint to discover endpoints for the service
if _, ok := snap.TerminatingGateway.WatchedServices[svc.Service]; !ok {
if _, ok := snap.TerminatingGateway.WatchedServices[svc.Service]; !ok && !(svc.ServiceKind == structs.GatewayServiceKindDestination) {
ctx, cancel := context.WithCancel(ctx)
err := s.dataSources.Health.Notify(ctx, &structs.ServiceSpecificRequest{
Datacenter: s.source.Datacenter,
@ -213,7 +219,8 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
// Watch service resolvers for the service
// These are used to create clusters and endpoints for the service subsets
if _, ok := snap.TerminatingGateway.WatchedResolvers[svc.Service]; !ok {
if _, ok := snap.TerminatingGateway.WatchedResolvers[svc.Service]; !ok && !(svc.ServiceKind == structs.GatewayServiceKindDestination) {
ctx, cancel := context.WithCancel(ctx)
err := s.dataSources.ConfigEntry.Notify(ctx, &structs.ConfigEntryQuery{
Datacenter: s.source.Datacenter,
@ -242,6 +249,13 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
}
}
// Delete endpoint service mapping for services that were not in the update
for sn := range snap.TerminatingGateway.DestinationServices {
if _, ok := svcMap[sn]; !ok {
delete(snap.TerminatingGateway.DestinationServices, sn)
}
}
// Clean up services with hostname mapping for services that were not in the update
for sn := range snap.TerminatingGateway.HostnameServices {
if _, ok := svcMap[sn]; !ok {

163
agent/proxycfg/testing_terminating_gateway.go
View File

@ -6,12 +6,7 @@ import (
"github.com/hashicorp/consul/agent/structs"
)
func TestConfigSnapshotTerminatingGateway(
t testing.T,
populateServices bool,
nsFn func(ns *structs.NodeService),
extraUpdates []UpdateEvent,
) *ConfigSnapshot {
func TestConfigSnapshotTerminatingGateway(t testing.T, populateServices bool, nsFn func(ns *structs.NodeService), extraUpdates []UpdateEvent) *ConfigSnapshot {
roots, _ := TestCerts(t)
var (
@ -34,6 +29,7 @@ func TestConfigSnapshotTerminatingGateway(
},
}
tgtwyServices := []*structs.GatewayService{}
if populateServices {
webNodes := TestUpstreamNodes(t, web.Name)
webNodes[0].Service.Meta = map[string]string{"version": "1"}
@ -156,28 +152,30 @@ func TestConfigSnapshotTerminatingGateway(
},
}
tgtwyServices = append(tgtwyServices,
&structs.GatewayService{
Service: web,
CAFile: "ca.cert.pem",
},
&structs.GatewayService{
Service: api,
CAFile: "ca.cert.pem",
CertFile: "api.cert.pem",
KeyFile: "api.key.pem",
},
&structs.GatewayService{
Service: db,
},
&structs.GatewayService{
Service: cache,
},
)
baseEvents = testSpliceEvents(baseEvents, []UpdateEvent{
{
CorrelationID: gatewayServicesWatchID,
Result: &structs.IndexedGatewayServices{
Services: []*structs.GatewayService{
{
Service: web,
CAFile: "ca.cert.pem",
},
{
Service: api,
CAFile: "ca.cert.pem",
CertFile: "api.cert.pem",
KeyFile: "api.key.pem",
},
{
Service: db,
},
{
Service: cache,
},
},
Services: tgtwyServices,
},
},
{
@ -342,6 +340,123 @@ func TestConfigSnapshotTerminatingGateway(
}, nsFn, nil, testSpliceEvents(baseEvents, extraUpdates))
}
func TestConfigSnapshotTerminatingGatewayDestinations(t testing.T, populateDestinations bool, extraUpdates []UpdateEvent) *ConfigSnapshot {
roots, _ := TestCerts(t)
var (
externalIPTCP = structs.NewServiceName("external-IP-TCP", nil)
externalHostnameTCP = structs.NewServiceName("external-hostname-TCP", nil)
)
baseEvents := []UpdateEvent{
{
CorrelationID: rootsWatchID,
Result: roots,
},
{
CorrelationID: gatewayServicesWatchID,
Result: &structs.IndexedGatewayServices{
Services: nil,
},
},
}
tgtwyServices := []*structs.GatewayService{}
if populateDestinations {
tgtwyServices = append(tgtwyServices,
&structs.GatewayService{
Service: externalIPTCP,
ServiceKind: structs.GatewayServiceKindDestination,
},
&structs.GatewayService{
Service: externalHostnameTCP,
ServiceKind: structs.GatewayServiceKindDestination,
},
)
baseEvents = testSpliceEvents(baseEvents, []UpdateEvent{
{
CorrelationID: gatewayServicesWatchID,
Result: &structs.IndexedGatewayServices{
Services: tgtwyServices,
},
},
// no intentions defined for these services
{
CorrelationID: serviceIntentionsIDPrefix + externalIPTCP.String(),
Result: &structs.IndexedIntentionMatches{
Matches: []structs.Intentions{
nil,
},
},
},
{
CorrelationID: serviceIntentionsIDPrefix + externalHostnameTCP.String(),
Result: &structs.IndexedIntentionMatches{
Matches: []structs.Intentions{
nil,
},
},
},
// ========
{
CorrelationID: serviceLeafIDPrefix + externalIPTCP.String(),
Result: &structs.IssuedCert{
CertPEM: "placeholder.crt",
PrivateKeyPEM: "placeholder.key",
},
},
{
CorrelationID: serviceLeafIDPrefix + externalHostnameTCP.String(),
Result: &structs.IssuedCert{
CertPEM: "placeholder.crt",
PrivateKeyPEM: "placeholder.key",
},
},
// ========
{
CorrelationID: serviceConfigIDPrefix + externalIPTCP.String(),
Result: &structs.ServiceConfigResponse{
Mode: structs.ProxyModeTransparent,
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
Destination: structs.DestinationConfig{
Address: "192.168.0.1",
Port: 80,
},
},
},
{
CorrelationID: serviceConfigIDPrefix + externalHostnameTCP.String(),
Result: &structs.ServiceConfigResponse{
Mode: structs.ProxyModeTransparent,
ProxyConfig: map[string]interface{}{"protocol": "tcp"},
Destination: structs.DestinationConfig{
Address: "*.hashicorp.com",
Port: 8089,
},
},
},
})
}
return testConfigSnapshotFixture(t, &structs.NodeService{
Kind: structs.ServiceKindTerminatingGateway,
Service: "terminating-gateway",
Address: "1.2.3.4",
Port: 8443,
Proxy: structs.ConnectProxyConfig{
Mode: structs.ProxyModeTransparent,
},
TaggedAddresses: map[string]structs.ServiceAddress{
structs.TaggedAddressWAN: {
Address: "198.18.0.1",
Port: 443,
},
},
}, nil, nil, testSpliceEvents(baseEvents, extraUpdates))
}
func TestConfigSnapshotTerminatingGatewayServiceSubsets(t testing.T) *ConfigSnapshot {
return testConfigSnapshotTerminatingGatewayServiceSubsets(t, false)
}

11
agent/structs/config_entry.go
View File

@ -301,6 +301,16 @@ type DestinationConfig struct {
Port int `json:",omitempty"`
}
func (d *DestinationConfig) HasHostname() bool {
ip := net.ParseIP(d.Address)
return ip == nil
}
func (d *DestinationConfig) HasIP() bool {
ip := net.ParseIP(d.Address)
return ip != nil
}
// ProxyConfigEntry is the top-level struct for global proxy configuration defaults.
type ProxyConfigEntry struct {
Kind string
@ -1043,6 +1053,7 @@ type ServiceConfigResponse struct {
Expose ExposeConfig `json:",omitempty"`
TransparentProxy TransparentProxyConfig `json:",omitempty"`
Mode ProxyMode `json:",omitempty"`
Destination DestinationConfig `json:",omitempty"`
Meta map[string]string `json:",omitempty"`
QueryMeta
}

167
agent/xds/clusters.go
View File

@ -9,6 +9,8 @@ import (
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_cluster_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3"
envoy_common_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
envoy_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
@ -27,6 +29,12 @@ import (
"github.com/hashicorp/consul/agent/structs"
)
const (
dynamicForwardProxyClusterName = "dynamic_forward_proxy_cluster"
dynamicForwardProxyClusterTypeName = "envoy.clusters.dynamic_forward_proxy"
dynamicForwardProxyClusterDNSCacheName = "dynamic_forward_proxy_cache_config"
)
// clustersFromSnapshot returns the xDS API representation of the "clusters" in the snapshot.
func (s *ResourceGenerator) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
if cfgSnap == nil {
@ -37,7 +45,7 @@ func (s *ResourceGenerator) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapsho
case structs.ServiceKindConnectProxy:
return s.clustersFromSnapshotConnectProxy(cfgSnap)
case structs.ServiceKindTerminatingGateway:
res, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers)
res, err := s.clustersFromSnapshotTerminatingGateway(cfgSnap)
if err != nil {
return nil, err
}
@ -276,7 +284,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
continue // skip local
}
opts := gatewayClusterOpts{
opts := clusterOpts{
name: connect.GatewaySNI(key.Datacenter, key.Partition, cfgSnap.Roots.TrustDomain),
hostnameEndpoints: cfgSnap.MeshGateway.HostnameDatacenters[key.String()],
isRemote: true,
@ -298,7 +306,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
if key.Datacenter == cfgSnap.Datacenter {
hostnameEndpoints = nil
}
opts := gatewayClusterOpts{
opts := clusterOpts{
name: cfgSnap.ServerSNIFn(key.Datacenter, ""),
hostnameEndpoints: hostnameEndpoints,
isRemote: !key.Matches(cfgSnap.Datacenter, cfgSnap.ProxyID.PartitionOrDefault()),
@ -309,7 +317,7 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
// And for the current datacenter, send all flavors appropriately.
for _, srv := range cfgSnap.MeshGateway.ConsulServers {
opts := gatewayClusterOpts{
opts := clusterOpts{
name: cfgSnap.ServerSNIFn(cfgSnap.Datacenter, srv.Node.Node),
}
cluster := s.makeGatewayCluster(cfgSnap, opts)
@ -327,6 +335,25 @@ func (s *ResourceGenerator) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.Co
return clusters, nil
}
// clustersFromSnapshotTerminatingGateway returns the xDS API representation of the "clusters"
// for a terminating gateway. This will include 1 cluster per Destination associated with this terminating gateway.
func (s *ResourceGenerator) clustersFromSnapshotTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
res := []proto.Message{}
gwClusters, err := s.makeGatewayServiceClusters(cfgSnap, cfgSnap.TerminatingGateway.ServiceGroups, cfgSnap.TerminatingGateway.ServiceResolvers)
if err != nil {
return nil, err
}
res = append(res, gwClusters...)
destClusters, err := s.makeDestinationClusters(cfgSnap)
if err != nil {
return nil, err
}
res = append(res, destClusters...)
return res, nil
}
func (s *ResourceGenerator) makeGatewayServiceClusters(
cfgSnap *proxycfg.ConfigSnapshot,
services map[structs.ServiceName]structs.CheckServiceNodes,
@ -367,7 +394,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
isRemote = !cfgSnap.Locality.Matches(services[svc][0].Node.Datacenter, services[svc][0].Node.PartitionOrDefault())
}
opts := gatewayClusterOpts{
opts := clusterOpts{
name: clusterName,
hostnameEndpoints: hostnameEndpoints,
connectTimeout: resolver.ConnectTimeout,
@ -387,7 +414,7 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
return nil, err
}
opts := gatewayClusterOpts{
opts := clusterOpts{
name: connect.ServiceSNI(svc.Name, name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain),
hostnameEndpoints: subsetHostnameEndpoints,
onlyPassing: subset.OnlyPassing,
@ -406,6 +433,46 @@ func (s *ResourceGenerator) makeGatewayServiceClusters(
return clusters, nil
}
func (s *ResourceGenerator) makeDestinationClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var createDynamicForwardProxy bool
serviceConfigs := cfgSnap.TerminatingGateway.ServiceConfigs
clusters := make([]proto.Message, 0, len(cfgSnap.TerminatingGateway.DestinationServices))
for _, svcName := range cfgSnap.TerminatingGateway.ValidDestinations() {
svcConfig, _ := serviceConfigs[svcName]
dest := svcConfig.Destination
// If IP, create a cluster with the fake name.
if dest.HasIP() {
opts := clusterOpts{
name: connect.ServiceSNI(svcName.Name, "", svcName.NamespaceOrDefault(), svcName.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain),
addressEndpoint: dest,
}
cluster := s.makeTerminatingIPCluster(cfgSnap, opts)
clusters = append(clusters, cluster)
continue
}
// TODO (dans): clusters will need to be customized later when we figure out how to manage a TLS segment from the terminating gateway to the Destination.
createDynamicForwardProxy = true
}
if createDynamicForwardProxy {
opts := clusterOpts{
name: dynamicForwardProxyClusterName,
}
cluster := s.makeDynamicForwardProxyCluster(cfgSnap, opts)
// TODO (dans): might be relevant later for TLS addons like CA validation
//if err := s.injectGatewayServiceAddons(cfgSnap, cluster, svc, loadBalancer); err != nil {
// return nil, err
//}
clusters = append(clusters, cluster)
}
return clusters, nil
}
func (s *ResourceGenerator) injectGatewayServiceAddons(cfgSnap *proxycfg.ConfigSnapshot, c *envoy_cluster_v3.Cluster, svc structs.ServiceName, lb *structs.LoadBalancer) error {
switch cfgSnap.Kind {
case structs.ServiceKindMeshGateway:
@ -1023,7 +1090,7 @@ func makeClusterFromUserConfig(configJSON string) (*envoy_cluster_v3.Cluster, er
return &c, err
}
type gatewayClusterOpts struct {
type clusterOpts struct {
// name for the cluster
name string
@ -1038,10 +1105,13 @@ type gatewayClusterOpts struct {
// hostnameEndpoints is a list of endpoints with a hostname as their address
hostnameEndpoints structs.CheckServiceNodes
// addressEndpoint is a singular ip/port endpoint
addressEndpoint structs.DestinationConfig
}
// makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway
func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayClusterOpts) *envoy_cluster_v3.Cluster {
func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster {
cfg, err := ParseGatewayConfig(snap.Proxy.Config)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
@ -1165,6 +1235,87 @@ func configureClusterWithHostnames(
}
}
// makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway
func (s *ResourceGenerator) makeTerminatingIPCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster {
cfg, err := ParseGatewayConfig(snap.Proxy.Config)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
// default config if there is an error so it's safe to continue.
s.Logger.Warn("failed to parse gateway config", "error", err)
}
if opts.connectTimeout <= 0 {
opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond
}
cluster := &envoy_cluster_v3.Cluster{
Name: opts.name,
ConnectTimeout: durationpb.New(opts.connectTimeout),
// Having an empty config enables outlier detection with default config.
OutlierDetection: &envoy_cluster_v3.OutlierDetection{},
}
discoveryType := envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC}
cluster.ClusterDiscoveryType = &discoveryType
endpoints := []*envoy_endpoint_v3.LbEndpoint{
makeEndpoint(opts.addressEndpoint.Address, opts.addressEndpoint.Port),
}
cluster.LoadAssignment = &envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: cluster.Name,
Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: endpoints,
},
},
}
return cluster
}
// makeDynamicForwardProxyCluster creates an Envoy cluster for that routes based on the SNI header received at the listener
func (s *ResourceGenerator) makeDynamicForwardProxyCluster(snap *proxycfg.ConfigSnapshot, opts clusterOpts) *envoy_cluster_v3.Cluster {
cfg, err := ParseGatewayConfig(snap.Proxy.Config)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
// default config if there is an error so it's safe to continue.
s.Logger.Warn("failed to parse gateway config", "error", err)
}
if opts.connectTimeout <= 0 {
opts.connectTimeout = time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond
}
cluster := &envoy_cluster_v3.Cluster{
Name: opts.name,
ConnectTimeout: durationpb.New(opts.connectTimeout),
}
dynamicForwardProxyCluster, err := anypb.New(&envoy_cluster_dynamic_forward_proxy_v3.ClusterConfig{
DnsCacheConfig: getCommonDNSCacheConfiguration(),
})
if err != nil {
// we should never get here since this message is static
s.Logger.Error("failed serialize dynamic forward proxy cluster config", "error", err)
}
cluster.LbPolicy = envoy_cluster_v3.Cluster_CLUSTER_PROVIDED
cluster.ClusterDiscoveryType = &envoy_cluster_v3.Cluster_ClusterType{
ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{
Name: dynamicForwardProxyClusterTypeName,
TypedConfig: dynamicForwardProxyCluster,
},
}
return cluster
}
func getCommonDNSCacheConfiguration() *envoy_common_dynamic_forward_proxy_v3.DnsCacheConfig {
return &envoy_common_dynamic_forward_proxy_v3.DnsCacheConfig{
Name: dynamicForwardProxyClusterDNSCacheName,
DnsLookupFamily: envoy_cluster_v3.Cluster_AUTO,
}
}
func makeThresholdsIfNeeded(limits *structs.UpstreamLimits) []*envoy_cluster_v3.CircuitBreakers_Thresholds {
if limits == nil {
return nil

6
agent/xds/clusters_test.go
View File

@ -613,6 +613,12 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "transparent-proxy-dial-instances-directly",
create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly,
},
{
name: "transparent-proxy-terminating-gateway-destinations-only",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil)
},
},
}
latestEnvoyVersion := proxysupport.EnvoyVersions[0]

145
agent/xds/listeners.go
View File

@ -22,6 +22,7 @@ import (
envoy_connection_limit_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/connection_limit/v3"
envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
envoy_sni_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_cluster/v3"
envoy_sni_dynamic_forward_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3"
envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
@ -1185,13 +1186,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
)
}
clusterChain, err := s.makeFilterChainTerminatingGateway(
cfgSnap,
clusterName,
svc,
intentions,
cfg.Protocol,
)
clusterChain, err := s.makeFilterChainTerminatingGateway(cfgSnap, clusterName, svc, intentions, cfg.Protocol, nil)
if err != nil {
return nil, fmt.Errorf("failed to make filter chain for cluster %q: %v", clusterName, err)
}
@ -1203,13 +1198,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
for subsetName := range resolver.Subsets {
subsetClusterName := connect.ServiceSNI(svc.Name, subsetName, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
subsetClusterChain, err := s.makeFilterChainTerminatingGateway(
cfgSnap,
subsetClusterName,
svc,
intentions,
cfg.Protocol,
)
subsetClusterChain, err := s.makeFilterChainTerminatingGateway(cfgSnap, subsetClusterName, svc, intentions, cfg.Protocol, nil)
if err != nil {
return nil, fmt.Errorf("failed to make filter chain for cluster %q: %v", subsetClusterName, err)
}
@ -1218,6 +1207,36 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
}
}
for _, svc := range cfgSnap.TerminatingGateway.ValidDestinations() {
clusterName := connect.ServiceSNI(svc.Name, "", svc.NamespaceOrDefault(), svc.PartitionOrDefault(), cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain)
intentions := cfgSnap.TerminatingGateway.Intentions[svc]
svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc]
cfg, err := ParseProxyConfig(svcConfig.ProxyConfig)
if err != nil {
// Don't hard fail on a config typo, just warn. The parse func returns
// default config if there is an error so it's safe to continue.
s.Logger.Warn(
"failed to parse Connect.Proxy.Config for linked destination",
"destination", svc.String(),
"error", err,
)
}
var dest *structs.DestinationConfig
if cfgSnap.TerminatingGateway.DestinationServices[svc].ServiceKind == structs.GatewayServiceKindDestination {
dest = &svcConfig.Destination
} else {
return nil, fmt.Errorf("invalid gateway service for destination %s", svc.Name)
}
clusterChain, err := s.makeFilterChainTerminatingGateway(cfgSnap, clusterName, svc, intentions, cfg.Protocol, dest)
if err != nil {
return nil, fmt.Errorf("failed to make filter chain for cluster %q: %v", clusterName, err)
}
l.FilterChains = append(l.FilterChains, clusterChain)
}
// Before we add the fallback, sort these chains by the matched name. All
// of these filter chains are independent, but envoy requires them to be in
// some order. If we put them in a random order then every xDS iteration
@ -1251,13 +1270,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
return l, nil
}
func (s *ResourceGenerator) makeFilterChainTerminatingGateway(
cfgSnap *proxycfg.ConfigSnapshot,
cluster string,
service structs.ServiceName,
intentions structs.Intentions,
protocol string,
) (*envoy_listener_v3.FilterChain, error) {
func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, cluster string, service structs.ServiceName, intentions structs.Intentions, protocol string, dest *structs.DestinationConfig) (*envoy_listener_v3.FilterChain, error) {
tlsContext := &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContext(
cfgSnap.TerminatingGateway.ServiceLeaves[service],
@ -1271,10 +1284,19 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(
return nil, err
}
filterChain := &envoy_listener_v3.FilterChain{
FilterChainMatch: makeSNIFilterChainMatch(cluster),
Filters: make([]*envoy_listener_v3.Filter, 0, 3),
TransportSocket: transportSocket,
var filterChain *envoy_listener_v3.FilterChain
if dest != nil {
filterChain = &envoy_listener_v3.FilterChain{
FilterChainMatch: makeDestinationFilterChainMatch(cluster, dest),
Filters: make([]*envoy_listener_v3.Filter, 0, 3),
TransportSocket: transportSocket,
}
} else {
filterChain = &envoy_listener_v3.FilterChain{
FilterChainMatch: makeSNIFilterChainMatch(cluster),
Filters: make([]*envoy_listener_v3.Filter, 0, 3),
TransportSocket: transportSocket,
}
}
// This controls if we do L4 or L7 intention checks.
@ -1293,16 +1315,28 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(
filterChain.Filters = append(filterChain.Filters, authFilter)
}
// For Destinations of Hostname types, we use the dynamic forward proxy filter since this could be
// a wildcard match. We also send to the dynamic forward cluster
if dest != nil && dest.HasHostname() {
dynamicFilter, err := makeSNIDynamicForwardProxyFilter(dest.Port)
if err != nil {
return nil, err
}
filterChain.Filters = append(filterChain.Filters, dynamicFilter)
cluster = dynamicForwardProxyClusterName
}
// Lastly we setup the actual proxying component. For L4 this is a straight
// tcp proxy. For L7 this is a very hands-off HTTP proxy just to inject an
// HTTP filter to do intention checks here instead.
opts := listenerFilterOpts{
protocol: protocol,
filterName: fmt.Sprintf("%s.%s.%s.%s", service.Name, service.NamespaceOrDefault(), service.PartitionOrDefault(), cfgSnap.Datacenter),
routeName: cluster, // Set cluster name for route config since each will have its own
cluster: cluster,
statPrefix: "upstream.",
routePath: "",
protocol: protocol,
filterName: fmt.Sprintf("%s.%s.%s.%s", service.Name, service.NamespaceOrDefault(), service.PartitionOrDefault(), cfgSnap.Datacenter),
routeName: cluster, // Set cluster name for route config since each will have its own
cluster: cluster,
statPrefix: "upstream.",
routePath: "",
useDynamicForwardProxy: dest != nil && dest.HasHostname(),
}
if useHTTPFilter {
@ -1335,6 +1369,23 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(
return filterChain, nil
}
func makeDestinationFilterChainMatch(cluster string, dest *structs.DestinationConfig) *envoy_listener_v3.FilterChainMatch {
// For hostname and wildcard destinations, we match on the address.
// For IP Destinations, use the alias SNI name to match
ip := net.ParseIP(dest.Address)
if ip != nil {
return &envoy_listener_v3.FilterChainMatch{
ServerNames: []string{cluster},
}
}
// For hostname and wildcard destinations, we match on the address in the Destination
return &envoy_listener_v3.FilterChainMatch{
ServerNames: []string{dest.Address},
}
}
func (s *ResourceGenerator) makeMeshGatewayListener(name, addr string, port int, cfgSnap *proxycfg.ConfigSnapshot) (*envoy_listener_v3.Listener, error) {
tlsInspector, err := makeTLSInspectorListenerFilter()
if err != nil {
@ -1658,18 +1709,19 @@ func (s *ResourceGenerator) getAndModifyUpstreamConfigForPeeredListener(
}
type listenerFilterOpts struct {
useRDS bool
protocol string
filterName string
routeName string
cluster string
statPrefix string
routePath string
requestTimeoutMs *int
ingressGateway bool
httpAuthzFilter *envoy_http_v3.HttpFilter
forwardClientDetails bool
forwardClientPolicy envoy_http_v3.HttpConnectionManager_ForwardClientCertDetails
useRDS bool
protocol string
filterName string
routeName string
cluster string
statPrefix string
routePath string
requestTimeoutMs *int
ingressGateway bool
httpAuthzFilter *envoy_http_v3.HttpFilter
forwardClientDetails bool
forwardClientPolicy envoy_http_v3.HttpConnectionManager_ForwardClientCertDetails
useDynamicForwardProxy bool
}
func makeListenerFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) {
@ -1702,6 +1754,13 @@ func makeSNIClusterFilter() (*envoy_listener_v3.Filter, error) {
return makeFilter("envoy.filters.network.sni_cluster", &envoy_sni_cluster_v3.SniCluster{})
}
func makeSNIDynamicForwardProxyFilter(upstreamPort int) (*envoy_listener_v3.Filter, error) {
return makeFilter("envoy.filters.network.sni_dynamic_forward_proxy", &envoy_sni_dynamic_forward_proxy_v3.FilterConfig{
DnsCacheConfig: getCommonDNSCacheConfiguration(),
PortSpecifier: &envoy_sni_dynamic_forward_proxy_v3.FilterConfig_PortValue{PortValue: uint32(upstreamPort)},
})
}
func makeTCPProxyFilter(filterName, cluster, statPrefix string) (*envoy_listener_v3.Filter, error) {
cfg := &envoy_tcp_proxy_v3.TcpProxy{
StatPrefix: makeStatPrefix(statPrefix, filterName),

6
agent/xds/listeners_test.go
View File

@ -776,6 +776,12 @@ func TestListenersFromSnapshot(t *testing.T) {
name: "transparent-proxy-terminating-gateway",
create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly,
},
{
name: "transparent-proxy-terminating-gateway-destinations-only",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTerminatingGatewayDestinations(t, true, nil)
},
},
}
latestEnvoyVersion := proxysupport.EnvoyVersions[0]

50
agent/xds/testdata/clusters/transparent-proxy-terminating-gateway-destinations-only.latest.golden vendored Normal file
View File

@ -0,0 +1,50 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dynamic_forward_proxy_cluster",
"clusterType": {
"name": "envoy.clusters.dynamic_forward_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig",
"dnsCacheConfig": {
"name": "dynamic_forward_proxy_cache_config"
}
}
},
"connectTimeout": "5s",
"lbPolicy": "CLUSTER_PROVIDED"
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "STATIC",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "192.168.0.1",
"portValue": 80
}
}
}
}
]
}
]
},
"outlierDetection": {
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}

164
agent/xds/testdata/listeners/transparent-proxy-terminating-gateway-destinations-only.latest.golden vendored Normal file
View File

@ -0,0 +1,164 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "default:1.2.3.4:8443",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8443
}
},
"filterChains": [
{
"filterChainMatch": {
"serverNames": [
"*.hashicorp.com"
]
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.sni_dynamic_forward_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3.FilterConfig",
"dnsCacheConfig": {
"name": "dynamic_forward_proxy_cache_config"
},
"portValue": 8089
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.external-hostname-TCP.default.default.dc1",
"cluster": "dynamic_forward_proxy_cluster"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "placeholder.crt\n"
},
"privateKey": {
"inlineString": "placeholder.key\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
},
{
"filterChainMatch": {
"serverNames": [
"external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
]
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
},
"statPrefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "upstream.external-IP-TCP.default.default.dc1",
"cluster": "external-IP-TCP.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
],
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"commonTlsContext": {
"tlsParams": {
},
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "placeholder.crt\n"
},
"privateKey": {
"inlineString": "placeholder.key\n"
}
}
],
"validationContext": {
"trustedCa": {
"inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n"
}
}
},
"requireClientCertificate": true
}
}
},
{
"filters": [
{
"name": "envoy.filters.network.sni_cluster",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.sni_cluster.v3.SniCluster"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"statPrefix": "terminating_gateway.default",
"cluster": ""
}
}
]
}
],
"listenerFilters": [
{
"name": "envoy.filters.listener.tls_inspector",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector"
}
}
],
"trafficDirection": "INBOUND"
}
],
"typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener",
"nonce": "00000001"
}