Refactor Ingress-specific lister code to separate file

This commit is contained in:
Paul Banks 2021-09-22 16:03:07 +01:00
parent 136928a90f
commit 5cfd030d03
2 changed files with 241 additions and 228 deletions

View File

@ -26,7 +26,6 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/any"
"github.com/golang/protobuf/ptypes/duration"
"github.com/golang/protobuf/ptypes/wrappers" "github.com/golang/protobuf/ptypes/wrappers"
"github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/connect"
@ -555,201 +554,6 @@ func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey prox
return &mergedCfg, nil return &mergedCfg, nil
} }
func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var resources []proto.Message
for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
var tlsContext *envoy_tls_v3.DownstreamTlsContext
sdsCfg, err := resolveListenerSDSConfig(cfgSnap, listenerKey)
if err != nil {
return nil, err
}
if sdsCfg != nil {
// Set up listener TLS from SDS
tlsContext = &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromSDS(*sdsCfg),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
} else if cfgSnap.IngressGateway.TLSConfig.Enabled {
tlsContext = &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
}
if listenerKey.Protocol == "tcp" {
// We rely on the invariant of upstreams slice always having at least 1
// member, because this key/value pair is created only when a
// GatewayService is returned in the RPC
u := upstreams[0]
id := u.Identifier()
chain := cfgSnap.IngressGateway.DiscoveryChain[id]
var upstreamListener proto.Message
upstreamListener, err := s.makeUpstreamListenerForDiscoveryChain(
&u,
address,
chain,
cfgSnap,
tlsContext,
)
if err != nil {
return nil, err
}
resources = append(resources, upstreamListener)
} else {
// If multiple upstreams share this port, make a special listener for the protocol.
listener := makePortListener(listenerKey.Protocol, address, listenerKey.Port, envoy_core_v3.TrafficDirection_OUTBOUND)
opts := listenerFilterOpts{
useRDS: true,
protocol: listenerKey.Protocol,
filterName: listenerKey.RouteName(),
routeName: listenerKey.RouteName(),
cluster: "",
statPrefix: "ingress_upstream_",
routePath: "",
httpAuthzFilter: nil,
}
// Generate any filter chains needed for services with custom TLS certs
// via SDS.
sniFilterChains, err := makeSDSOverrideFilterChains(cfgSnap, listenerKey, opts)
if err != nil {
return nil, err
}
// If there are any sni filter chains, we need a TLS inspector filter!
if len(sniFilterChains) > 0 {
tlsInspector, err := makeTLSInspectorListenerFilter()
if err != nil {
return nil, err
}
listener.ListenerFilters = []*envoy_listener_v3.ListenerFilter{tlsInspector}
}
listener.FilterChains = sniFilterChains
// See if there are other services that didn't have specific SNI-matching
// filter chains. If so add a default filterchain to serve them.
if len(sniFilterChains) < len(upstreams) {
defaultFilter, err := makeListenerFilter(opts)
if err != nil {
return nil, err
}
transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}
listener.FilterChains = append(listener.FilterChains,
&envoy_listener_v3.FilterChain{
Filters: []*envoy_listener_v3.Filter{
defaultFilter,
},
TransportSocket: transportSocket,
})
}
resources = append(resources, listener)
}
}
return resources, nil
}
func routeNameForUpstream(l structs.IngressListener, s structs.IngressService) string {
key := proxycfg.IngressListenerKeyFromListener(l)
// If the upstream service doesn't have any TLS overrides then it can just use
// the combined filterchain with all the merged routes.
if !ingressServiceHasSDSOverrides(s) {
return key.RouteName()
}
// Return a specific route for this service as it needs a custom FilterChain
// to serve its custom cert so we should attach its routes to a separate Route
// too. We need this to be consistent between OSS and Enterprise to avoid xDS
// config golden files in tests conflicting so we can't use ServiceID.String()
// which normalizes to included all identifiers in Enterprise.
sn := s.ToServiceName()
svcIdentifier := sn.Name
if !sn.InDefaultPartition() || !sn.InDefaultNamespace() {
// Non-default partition/namespace, use a full identifier
svcIdentifier = sn.String()
}
return fmt.Sprintf("%s_%s", key.RouteName(), svcIdentifier)
}
func ingressServiceHasSDSOverrides(s structs.IngressService) bool {
return s.TLS != nil &&
s.TLS.SDS != nil &&
s.TLS.SDS.CertResource != ""
}
// ingress services that specify custom TLS certs via SDS overrides need to get
// their own filter chain and routes. This will generate all the extra filter
// chains an ingress listener needs. It may be empty and expects the default
// catch-all chain and route to contain all the other services that share the
// default TLS config.
func makeSDSOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
listenerKey proxycfg.IngressListenerKey,
filterOpts listenerFilterOpts) ([]*envoy_listener_v3.FilterChain, error) {
listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey]
if !ok {
return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port)
}
var chains []*envoy_listener_v3.FilterChain
for _, svc := range listenerCfg.Services {
if ingressServiceHasSDSOverrides(svc) {
if len(svc.Hosts) < 1 {
// Shouldn't be possible with validation but be careful
return nil, fmt.Errorf("no hosts specified with SDS certificate (service %q on listener on port %d)",
svc.ToServiceName().ToServiceID().String(), listenerKey.Port)
}
// Service has a certificate resource override. Return a new filter chain
// with the right TLS cert and a filter that will load only the routes for
// this service.
routeName := routeNameForUpstream(listenerCfg, svc)
filterOpts.filterName = routeName
filterOpts.routeName = routeName
filter, err := makeListenerFilter(filterOpts)
if err != nil {
return nil, err
}
tlsContext := &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromSDS(*svc.TLS.SDS),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}
chain := &envoy_listener_v3.FilterChain{
// Only match traffic for this service's hosts.
FilterChainMatch: makeSNIFilterChainMatch(svc.Hosts...),
Filters: []*envoy_listener_v3.Filter{
filter,
},
TransportSocket: transportSocket,
}
chains = append(chains, chain)
}
}
return chains, nil
}
// makeListener returns a listener with name and bind details set. Filters must // makeListener returns a listener with name and bind details set. Filters must
// be added before it's useful. // be added before it's useful.
// //
@ -1965,38 +1769,6 @@ func makeCommonTLSContextFromLeaf(cfgSnap *proxycfg.ConfigSnapshot, leaf *struct
} }
} }
func makeCommonTLSContextFromSDS(sdsCfg structs.GatewayTLSSDSConfig) *envoy_tls_v3.CommonTlsContext {
return &envoy_tls_v3.CommonTlsContext{
TlsParams: &envoy_tls_v3.TlsParameters{},
TlsCertificateSdsSecretConfigs: []*envoy_tls_v3.SdsSecretConfig{
{
Name: sdsCfg.CertResource,
SdsConfig: &envoy_core_v3.ConfigSource{
ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_ApiConfigSource{
ApiConfigSource: &envoy_core_v3.ApiConfigSource{
ApiType: envoy_core_v3.ApiConfigSource_GRPC,
TransportApiVersion: envoy_core_v3.ApiVersion_V3,
// Note ClusterNames can't be set here - that's only for REST type
// we need a full GRPC config instead.
GrpcServices: []*envoy_core_v3.GrpcService{
{
TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{
ClusterName: sdsCfg.ClusterName,
},
},
Timeout: &duration.Duration{Seconds: 5},
},
},
},
},
ResourceApiVersion: envoy_core_v3.ApiVersion_V3,
},
},
},
}
}
func makeDownstreamTLSTransportSocket(tlsContext *envoy_tls_v3.DownstreamTlsContext) (*envoy_core_v3.TransportSocket, error) { func makeDownstreamTLSTransportSocket(tlsContext *envoy_tls_v3.DownstreamTlsContext) (*envoy_core_v3.TransportSocket, error) {
if tlsContext == nil { if tlsContext == nil {
return nil, nil return nil, nil

View File

@ -0,0 +1,241 @@
package xds
import (
"fmt"
envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes/duration"
"github.com/golang/protobuf/ptypes/wrappers"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
)
func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var resources []proto.Message
for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
var tlsContext *envoy_tls_v3.DownstreamTlsContext
sdsCfg, err := resolveListenerSDSConfig(cfgSnap, listenerKey)
if err != nil {
return nil, err
}
if sdsCfg != nil {
// Set up listener TLS from SDS
tlsContext = &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromSDS(*sdsCfg),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
} else if cfgSnap.IngressGateway.TLSConfig.Enabled {
tlsContext = &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
}
if listenerKey.Protocol == "tcp" {
// We rely on the invariant of upstreams slice always having at least 1
// member, because this key/value pair is created only when a
// GatewayService is returned in the RPC
u := upstreams[0]
id := u.Identifier()
chain := cfgSnap.IngressGateway.DiscoveryChain[id]
var upstreamListener proto.Message
upstreamListener, err := s.makeUpstreamListenerForDiscoveryChain(
&u,
address,
chain,
cfgSnap,
tlsContext,
)
if err != nil {
return nil, err
}
resources = append(resources, upstreamListener)
} else {
// If multiple upstreams share this port, make a special listener for the protocol.
listener := makePortListener(listenerKey.Protocol, address, listenerKey.Port, envoy_core_v3.TrafficDirection_OUTBOUND)
opts := listenerFilterOpts{
useRDS: true,
protocol: listenerKey.Protocol,
filterName: listenerKey.RouteName(),
routeName: listenerKey.RouteName(),
cluster: "",
statPrefix: "ingress_upstream_",
routePath: "",
httpAuthzFilter: nil,
}
// Generate any filter chains needed for services with custom TLS certs
// via SDS.
sniFilterChains, err := makeSDSOverrideFilterChains(cfgSnap, listenerKey, opts)
if err != nil {
return nil, err
}
// If there are any sni filter chains, we need a TLS inspector filter!
if len(sniFilterChains) > 0 {
tlsInspector, err := makeTLSInspectorListenerFilter()
if err != nil {
return nil, err
}
listener.ListenerFilters = []*envoy_listener_v3.ListenerFilter{tlsInspector}
}
listener.FilterChains = sniFilterChains
// See if there are other services that didn't have specific SNI-matching
// filter chains. If so add a default filterchain to serve them.
if len(sniFilterChains) < len(upstreams) {
defaultFilter, err := makeListenerFilter(opts)
if err != nil {
return nil, err
}
transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}
listener.FilterChains = append(listener.FilterChains,
&envoy_listener_v3.FilterChain{
Filters: []*envoy_listener_v3.Filter{
defaultFilter,
},
TransportSocket: transportSocket,
})
}
resources = append(resources, listener)
}
}
return resources, nil
}
func routeNameForUpstream(l structs.IngressListener, s structs.IngressService) string {
key := proxycfg.IngressListenerKeyFromListener(l)
// If the upstream service doesn't have any TLS overrides then it can just use
// the combined filterchain with all the merged routes.
if !ingressServiceHasSDSOverrides(s) {
return key.RouteName()
}
// Return a specific route for this service as it needs a custom FilterChain
// to serve its custom cert so we should attach its routes to a separate Route
// too. We need this to be consistent between OSS and Enterprise to avoid xDS
// config golden files in tests conflicting so we can't use ServiceID.String()
// which normalizes to included all identifiers in Enterprise.
sn := s.ToServiceName()
svcIdentifier := sn.Name
if !sn.InDefaultPartition() || !sn.InDefaultNamespace() {
// Non-default partition/namespace, use a full identifier
svcIdentifier = sn.String()
}
return fmt.Sprintf("%s_%s", key.RouteName(), svcIdentifier)
}
func ingressServiceHasSDSOverrides(s structs.IngressService) bool {
return s.TLS != nil &&
s.TLS.SDS != nil &&
s.TLS.SDS.CertResource != ""
}
// ingress services that specify custom TLS certs via SDS overrides need to get
// their own filter chain and routes. This will generate all the extra filter
// chains an ingress listener needs. It may be empty and expects the default
// catch-all chain and route to contain all the other services that share the
// default TLS config.
func makeSDSOverrideFilterChains(cfgSnap *proxycfg.ConfigSnapshot,
listenerKey proxycfg.IngressListenerKey,
filterOpts listenerFilterOpts) ([]*envoy_listener_v3.FilterChain, error) {
listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey]
if !ok {
return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port)
}
var chains []*envoy_listener_v3.FilterChain
for _, svc := range listenerCfg.Services {
if ingressServiceHasSDSOverrides(svc) {
if len(svc.Hosts) < 1 {
// Shouldn't be possible with validation but be careful
return nil, fmt.Errorf("no hosts specified with SDS certificate (service %q on listener on port %d)",
svc.ToServiceName().ToServiceID().String(), listenerKey.Port)
}
// Service has a certificate resource override. Return a new filter chain
// with the right TLS cert and a filter that will load only the routes for
// this service.
routeName := routeNameForUpstream(listenerCfg, svc)
filterOpts.filterName = routeName
filterOpts.routeName = routeName
filter, err := makeListenerFilter(filterOpts)
if err != nil {
return nil, err
}
tlsContext := &envoy_tls_v3.DownstreamTlsContext{
CommonTlsContext: makeCommonTLSContextFromSDS(*svc.TLS.SDS),
RequireClientCertificate: &wrappers.BoolValue{Value: false},
}
transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext)
if err != nil {
return nil, err
}
chain := &envoy_listener_v3.FilterChain{
// Only match traffic for this service's hosts.
FilterChainMatch: makeSNIFilterChainMatch(svc.Hosts...),
Filters: []*envoy_listener_v3.Filter{
filter,
},
TransportSocket: transportSocket,
}
chains = append(chains, chain)
}
}
return chains, nil
}
func makeCommonTLSContextFromSDS(sdsCfg structs.GatewayTLSSDSConfig) *envoy_tls_v3.CommonTlsContext {
return &envoy_tls_v3.CommonTlsContext{
TlsParams: &envoy_tls_v3.TlsParameters{},
TlsCertificateSdsSecretConfigs: []*envoy_tls_v3.SdsSecretConfig{
{
Name: sdsCfg.CertResource,
SdsConfig: &envoy_core_v3.ConfigSource{
ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_ApiConfigSource{
ApiConfigSource: &envoy_core_v3.ApiConfigSource{
ApiType: envoy_core_v3.ApiConfigSource_GRPC,
TransportApiVersion: envoy_core_v3.ApiVersion_V3,
// Note ClusterNames can't be set here - that's only for REST type
// we need a full GRPC config instead.
GrpcServices: []*envoy_core_v3.GrpcService{
{
TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{
EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{
ClusterName: sdsCfg.ClusterName,
},
},
Timeout: &duration.Duration{Seconds: 5},
},
},
},
},
ResourceApiVersion: envoy_core_v3.ApiVersion_V3,
},
},
},
}
}