status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Allow ingress gateways to route traffic based on Host header 

This commit adds the necessary changes to allow an ingress gateway to
route traffic from a single defined port to multiple different upstream
services in the Consul mesh.

To do this, we now require all HTTP requests coming into the ingress
gateway to specify a Host header that matches "<service-name>.*" in
order to correctly route traffic to the correct service.

- Differentiate multiple listener's route names by port
- Adds a case in xds for allowing default discovery chains to create a
  route configuration when on an ingress gateway. This allows default
  services to easily use host header routing
- ingress-gateways have a single route config for each listener
  that utilizes domain matching to route to different services.
This commit is contained in:
Kyle Havlovitz 2020-04-16 16:24:11 -07:00 committed by Chris Piraino
parent a854e4d9c5
commit 247f9eaf13
29 changed files with 638 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
agent/consul/internal_endpoint_test.go
View File

@ -935,6 +935,7 @@ func TestInternal_GatewayServices_BothGateways(t *testing.T) {
Service: structs.NewServiceID("db", nil),
Gateway: structs.NewServiceID("ingress", nil),
GatewayKind: structs.ServiceKindIngressGateway,
Protocol: "tcp",
Port: 8888,
},
}

1
agent/consul/state/catalog.go
View File

@ -2540,6 +2540,7 @@ func (s *Store) ingressConfigGatewayServices(tx *memdb.Txn, gateway structs.Serv
Service: service.ToServiceID(),
GatewayKind: structs.ServiceKindIngressGateway,
Port: listener.Port,
Protocol: listener.Protocol,
}
gatewayServices = append(gatewayServices, mapping)

13
agent/proxycfg/snapshot.go
View File

@ -2,6 +2,8 @@ package proxycfg
import (
"context"
"fmt"
"github.com/hashicorp/consul/agent/structs"
"github.com/mitchellh/copystructure"
)
@ -194,7 +196,7 @@ type configSnapshotIngressGateway struct {
// Upstreams is a list of upstreams this ingress gateway should serve traffic
// to. This is constructed from the ingress-gateway config entry, and uses
// the GatewayServices RPC to retrieve them.
Upstreams []structs.Upstream
Upstreams map[IngressListenerKey]structs.Upstreams
// WatchedDiscoveryChains is a map of upstream.Identifier() -> CancelFunc's
// in order to cancel any watches when the ingress gateway configuration is
@ -214,6 +216,15 @@ func (c *configSnapshotIngressGateway) IsEmpty() bool {
len(c.WatchedUpstreamEndpoints) == 0
}
type IngressListenerKey struct {
Protocol string
Port int
}
func (k *IngressListenerKey) RouteName() string {
return fmt.Sprintf("%s_%d", k.Protocol, k.Port)
}
// ConfigSnapshot captures all the resulting config needed for a proxy instance.
// It is meant to be point-in-time coherent and is used to deliver the current
// config state to observers who need it to be pushed in (e.g. XDS server).

8
agent/proxycfg/state.go
View File

@ -1320,8 +1320,8 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap
return fmt.Errorf("invalid type for response: %T", u.Result)
}
var upstreams structs.Upstreams
watchedSvcs := make(map[string]struct{})
upstreamsMap := make(map[IngressListenerKey]structs.Upstreams)
for _, service := range services.Services {
u := makeUpstream(service, s.address)
@ -1330,9 +1330,11 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap
return err
}
watchedSvcs[u.Identifier()] = struct{}{}
upstreams = append(upstreams, u)
id := IngressListenerKey{Protocol: service.Protocol, Port: service.Port}
upstreamsMap[id] = append(upstreamsMap[id], u)
}
snap.IngressGateway.Upstreams = upstreams
snap.IngressGateway.Upstreams = upstreamsMap
for id, cancelFn := range snap.IngressGateway.WatchedDiscoveryChains {
if _, ok := watchedSvcs[id]; !ok {

60
agent/proxycfg/state_test.go
View File

@ -810,6 +810,66 @@ func TestState_WatchesAndUpdates(t *testing.T) {
},
},
},
"ingress-gateway-update-upstreams": testCase{
ns: structs.NodeService{
Kind: structs.ServiceKindIngressGateway,
ID: "ingress-gateway",
Service: "ingress-gateway",
Address: "10.0.1.1",
},
sourceDC: "dc1",
stages: []verificationStage{
verificationStage{
requiredWatches: map[string]verifyWatchRequest{
rootsWatchID: genVerifyRootsWatch("dc1"),
leafWatchID: genVerifyLeafWatch("ingress-gateway", "dc1"),
},
events: []cache.UpdateEvent{
rootWatchEvent(),
cache.UpdateEvent{
CorrelationID: leafWatchID,
Result: issuedCert,
Err: nil,
},
cache.UpdateEvent{
CorrelationID: gatewayServicesWatchID,
Result: &structs.IndexedGatewayServices{
Services: structs.GatewayServices{
{
Gateway: structs.NewServiceID("ingress-gateway", nil),
Service: structs.NewServiceID("api", nil),
Port: 9999,
},
},
},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.Len(t, snap.IngressGateway.Upstreams, 1)
require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 1)
require.Contains(t, snap.IngressGateway.WatchedDiscoveryChains, "api")
},
},
verificationStage{
requiredWatches: map[string]verifyWatchRequest{},
events: []cache.UpdateEvent{
cache.UpdateEvent{
CorrelationID: gatewayServicesWatchID,
Result: &structs.IndexedGatewayServices{},
Err: nil,
},
},
verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
require.True(t, snap.Valid())
require.Len(t, snap.IngressGateway.Upstreams, 0)
require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 0)
require.NotContains(t, snap.IngressGateway.WatchedDiscoveryChains, "api")
},
},
},
},
"terminating-gateway-initial": testCase{
ns: structs.NodeService{
Kind: structs.ServiceKindTerminatingGateway,

66
agent/proxycfg/testing.go
View File

@ -1143,6 +1143,7 @@ func setupTestVariationConfigEntriesAndSnapshot(
},
},
)
case "http-multiple-services":
default:
t.Fatalf("unexpected variation: %q", variation)
return ConfigSnapshotUpstreams{}
@ -1233,6 +1234,13 @@ func setupTestVariationConfigEntriesAndSnapshot(
case "chain-and-splitter":
case "grpc-router":
case "chain-and-router":
case "http-multiple-services":
snap.WatchedUpstreamEndpoints["foo"] = map[string]structs.CheckServiceNodes{
"foo.default.dc1": TestUpstreamNodes(t),
}
snap.WatchedUpstreamEndpoints["bar"] = map[string]structs.CheckServiceNodes{
"bar.default.dc1": TestUpstreamNodesAlternate(t),
}
default:
t.Fatalf("unexpected variation: %q", variation)
return ConfigSnapshotUpstreams{}
@ -1312,82 +1320,86 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera
}
func TestConfigSnapshotIngress(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "simple")
return testConfigSnapshotIngressGateway(t, true, "tcp", "simple")
}
func TestConfigSnapshotIngressWithOverrides(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "simple-with-overrides")
return testConfigSnapshotIngressGateway(t, true, "tcp", "simple-with-overrides")
}
func TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "splitter-with-resolver-redirect-multidc")
return testConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc")
}
func TestConfigSnapshotIngress_HTTPMultipleServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "http", "http-multiple-services")
}
func TestConfigSnapshotIngressExternalSNI(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "external-sni")
return testConfigSnapshotIngressGateway(t, true, "tcp", "external-sni")
}
func TestConfigSnapshotIngressWithFailover(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover")
}
func TestConfigSnapshotIngressWithFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway")
}
func TestConfigSnapshotIngressWithFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway-triggered")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway-triggered")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered")
}
func TestConfigSnapshotIngressWithFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway")
}
func TestConfigSnapshotIngressWithFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway-triggered")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway")
}
func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway-triggered")
return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered")
}
func TestConfigSnapshotIngressWithSplitter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "chain-and-splitter")
return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-splitter")
}
func TestConfigSnapshotIngressWithGRPCRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "grpc-router")
return testConfigSnapshotIngressGateway(t, true, "http", "grpc-router")
}
func TestConfigSnapshotIngressWithRouter(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "chain-and-router")
return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-router")
}
func TestConfigSnapshotIngressGateway(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "default")
return testConfigSnapshotIngressGateway(t, true, "tcp", "default")
}
func TestConfigSnapshotIngressGatewayNoServices(t testing.T) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, false, "default")
return testConfigSnapshotIngressGateway(t, false, "tcp", "default")
}
func TestConfigSnapshotIngressDiscoveryChainWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot {
return testConfigSnapshotIngressGateway(t, true, "simple", additionalEntries...)
return testConfigSnapshotIngressGateway(t, true, "http", "simple", additionalEntries...)
}
func testConfigSnapshotIngressGateway(
t testing.T, populateServices bool, variation string,
t testing.T, populateServices bool, protocol, variation string,
additionalEntries ...structs.ConfigEntry,
) *ConfigSnapshot {
roots, leaf := TestCerts(t)
@ -1404,12 +1416,14 @@ func testConfigSnapshotIngressGateway(
ConfigSnapshotUpstreams: setupTestVariationConfigEntriesAndSnapshot(
t, variation, leaf, additionalEntries...,
),
Upstreams: structs.Upstreams{
{
// We rely on this one having default type in a few tests...
DestinationName: "db",
LocalBindPort: 9191,
LocalBindAddress: "2.3.4.5",
Upstreams: map[IngressListenerKey]structs.Upstreams{
IngressListenerKey{protocol, 9191}: structs.Upstreams{
{
// We rely on this one having default type in a few tests...
DestinationName: "db",
LocalBindPort: 9191,
LocalBindAddress: "2.3.4.5",
},
},
},
}

3
agent/structs/config_entry_gateways.go
View File

@ -300,6 +300,7 @@ type GatewayService struct {
Service ServiceID
GatewayKind ServiceKind
Port int
Protocol string
CAFile string
CertFile string
KeyFile string
@ -315,6 +316,7 @@ func (g *GatewayService) IsSame(o *GatewayService) bool {
g.Service.Matches(&o.Service) &&
g.GatewayKind == o.GatewayKind &&
g.Port == o.Port &&
g.Protocol == o.Protocol &&
g.CAFile == o.CAFile &&
g.CertFile == o.CertFile &&
g.KeyFile == o.KeyFile &&
@ -328,6 +330,7 @@ func (g *GatewayService) Clone() *GatewayService {
Service: g.Service,
GatewayKind: g.GatewayKind,
Port: g.Port,
Protocol: g.Protocol,
CAFile: g.CAFile,
CertFile: g.CertFile,
KeyFile: g.KeyFile,

38
agent/xds/clusters.go
View File

@ -236,27 +236,29 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([
func (s *Server) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var clusters []proto.Message
for _, u := range cfgSnap.IngressGateway.Upstreams {
id := u.Identifier()
chain, ok := cfgSnap.IngressGateway.DiscoveryChain[id]
if !ok {
// this should not happen
return nil, fmt.Errorf("no discovery chain for upstream %q", id)
}
for _, upstreams := range cfgSnap.IngressGateway.Upstreams {
for _, u := range upstreams {
id := u.Identifier()
chain, ok := cfgSnap.IngressGateway.DiscoveryChain[id]
if !ok {
// this should not happen
return nil, fmt.Errorf("no discovery chain for upstream %q", id)
}
chainEndpoints, ok := cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id]
if !ok {
// this should not happen
return nil, fmt.Errorf("no endpoint map for upstream %q", id)
}
chainEndpoints, ok := cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id]
if !ok {
// this should not happen
return nil, fmt.Errorf("no endpoint map for upstream %q", id)
}
upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain(u, chain, chainEndpoints, cfgSnap)
if err != nil {
return nil, err
}
upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain(u, chain, chainEndpoints, cfgSnap)
if err != nil {
return nil, err
}
for _, c := range upstreamClusters {
clusters = append(clusters, c)
for _, c := range upstreamClusters {
clusters = append(clusters, c)
}
}
}
return clusters, nil

20
agent/xds/endpoints.go
View File

@ -255,16 +255,18 @@ func (s *Server) endpointsFromServicesAndResolvers(
func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var resources []proto.Message
for _, u := range cfgSnap.IngressGateway.Upstreams {
id := u.Identifier()
for _, upstreams := range cfgSnap.IngressGateway.Upstreams {
for _, u := range upstreams {
id := u.Identifier()
es := s.endpointsFromDiscoveryChain(
cfgSnap.IngressGateway.DiscoveryChain[id],
cfgSnap.Datacenter,
cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id],
cfgSnap.IngressGateway.WatchedGatewayEndpoints[id],
)
resources = append(resources, es...)
es := s.endpointsFromDiscoveryChain(
cfgSnap.IngressGateway.DiscoveryChain[id],
cfgSnap.Datacenter,
cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id],
cfgSnap.IngressGateway.WatchedGatewayEndpoints[id],
)
resources = append(resources, es...)
}
}
return resources, nil
}

52
agent/xds/listeners.go
View File

@ -270,25 +270,47 @@ func (s *Server) listenersFromSnapshotGateway(cfgSnap *proxycfg.ConfigSnapshot,
// See: https://www.consul.io/docs/connect/proxies/envoy.html#mesh-gateway-options
func (s *Server) listenersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
var resources []proto.Message
// TODO(ingress): We give each upstream a distinct listener at the moment,
// for http listeners we will need to multiplex upstreams on a single
// listener.
for _, u := range cfgSnap.IngressGateway.Upstreams {
id := u.Identifier()
for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
if listenerKey.Protocol == "tcp" {
u := upstreams[0]
id := u.Identifier()
chain := cfgSnap.IngressGateway.DiscoveryChain[id]
chain := cfgSnap.IngressGateway.DiscoveryChain[id]
var upstreamListener proto.Message
var err error
if chain == nil || chain.IsDefault() {
upstreamListener, err = s.makeUpstreamListenerIgnoreDiscoveryChain(&u, chain, cfgSnap)
var upstreamListener proto.Message
var err error
if chain == nil || chain.IsDefault() {
upstreamListener, err = s.makeUpstreamListenerIgnoreDiscoveryChain(&u, chain, cfgSnap)
} else {
upstreamListener, err = s.makeUpstreamListenerForDiscoveryChain(&u, chain, cfgSnap)
}
if err != nil {
return nil, err
}
resources = append(resources, upstreamListener)
} else {
upstreamListener, err = s.makeUpstreamListenerForDiscoveryChain(&u, chain, cfgSnap)
// If multiple upstreams share this port, make a special listener for the protocol.
addr := cfgSnap.Address
if addr == "" {
addr = "0.0.0.0"
}
listener := makeListener(listenerKey.Protocol, addr, listenerKey.Port)
filter, err := makeListenerFilter(
true, listenerKey.Protocol, listenerKey.RouteName(), "", "ingress_upstream_", "", false)
if err != nil {
return nil, err
}
listener.FilterChains = []envoylistener.FilterChain{
{
Filters: []envoylistener.Filter{
filter,
},
},
}
resources = append(resources, listener)
}
if err != nil {
return nil, err
}
resources = append(resources, upstreamListener)
}
return resources, nil

28
agent/xds/listeners_test.go
View File

@ -363,6 +363,34 @@ func TestListenersFromSnapshot(t *testing.T) {
}
},
},
{
name: "ingress-http-multiple-services",
create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{
{
DestinationName: "foo",
LocalBindPort: 8080,
},
{
DestinationName: "bar",
LocalBindPort: 8080,
},
},
proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{
{
DestinationName: "baz",
LocalBindPort: 443,
},
{
DestinationName: "qux",
LocalBindPort: 443,
},
},
}
},
},
{
name: "terminating-gateway-no-api-cert",
create: proxycfg.TestConfigSnapshotTerminatingGateway,

114
agent/xds/routes.go
View File

@ -37,7 +37,34 @@ func routesFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.M
return nil, errors.New("nil config given")
}
return routesFromUpstreams(cfgSnap.ConnectProxy.ConfigSnapshotUpstreams, cfgSnap.Proxy.Upstreams)
var resources []proto.Message
for _, u := range cfgSnap.Proxy.Upstreams {
upstreamID := u.Identifier()
var chain *structs.CompiledDiscoveryChain
if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
chain = cfgSnap.ConnectProxy.DiscoveryChain[upstreamID]
}
if chain == nil || chain.IsDefault() {
// TODO(rb): make this do the old school stuff too
} else {
virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, "*")
if err != nil {
return nil, err
}
route := &envoy.RouteConfiguration{
Name: upstreamID,
VirtualHosts: []envoyroute.VirtualHost{*virtualHost},
ValidateClusters: makeBoolValue(true),
}
resources = append(resources, route)
}
}
// TODO(rb): make sure we don't generate an empty result
return resources, nil
}
// routesFromSnapshotIngressGateway returns the xDS API representation of the
@ -47,45 +74,51 @@ func routesFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto
return nil, errors.New("nil config given")
}
return routesFromUpstreams(cfgSnap.IngressGateway.ConfigSnapshotUpstreams, cfgSnap.IngressGateway.Upstreams)
}
func routesFromUpstreams(snap proxycfg.ConfigSnapshotUpstreams, upstreams structs.Upstreams) ([]proto.Message, error) {
var resources []proto.Message
for _, u := range upstreams {
upstreamID := u.Identifier()
var chain *structs.CompiledDiscoveryChain
if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
chain = snap.DiscoveryChain[upstreamID]
var result []proto.Message
for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams {
// Do not create any route configuration for TCP listeners
if listenerKey.Protocol == "tcp" {
continue
}
if chain == nil || chain.IsDefault() {
// TODO(rb): make this do the old school stuff too
} else {
upstreamRoute, err := makeUpstreamRouteForDiscoveryChain(&u, chain)
if err != nil {
return nil, err
}
if upstreamRoute != nil {
resources = append(resources, upstreamRoute)
upstreamRoute := &envoy.RouteConfiguration{
Name: listenerKey.RouteName(),
// ValidateClusters defaults to true when defined statically and false
// when done via RDS. Re-set the sane value of true to prevent
// null-routing traffic.
ValidateClusters: makeBoolValue(true),
}
for _, u := range upstreams {
upstreamID := u.Identifier()
chain := cfgSnap.IngressGateway.DiscoveryChain[upstreamID]
if chain != nil {
domain := fmt.Sprintf("%s.*", chain.ServiceName)
// Don't require a service prefix on the domain if there is only 1
// upstream. This makes it a smoother experience when only having a
// single service associated to a listener, which is probably a common
// case when demoing/testing
if len(upstreams) == 1 {
domain = "*"
}
virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, domain)
if err != nil {
return nil, err
}
upstreamRoute.VirtualHosts = append(upstreamRoute.VirtualHosts, *virtualHost)
}
}
result = append(result, upstreamRoute)
}
// TODO(rb): make sure we don't generate an empty result
return resources, nil
return result, nil
}
func makeUpstreamRouteForDiscoveryChain(
u *structs.Upstream,
routeName string,
chain *structs.CompiledDiscoveryChain,
) (*envoy.RouteConfiguration, error) {
upstreamID := u.Identifier()
routeName := upstreamID
serviceDomain string,
) (*envoyroute.VirtualHost, error) {
var routes []envoyroute.Route
startNode := chain.Nodes[chain.StartNode]
@ -188,20 +221,13 @@ func makeUpstreamRouteForDiscoveryChain(
panic("unknown first node in discovery chain of type: " + startNode.Type)
}
return &envoy.RouteConfiguration{
Name: routeName,
VirtualHosts: []envoyroute.VirtualHost{
envoyroute.VirtualHost{
Name: routeName,
Domains: []string{"*"},
Routes: routes,
},
},
// ValidateClusters defaults to true when defined statically and false
// when done via RDS. Re-set the sane value of true to prevent
// null-routing traffic.
ValidateClusters: makeBoolValue(true),
}, nil
host := &envoyroute.VirtualHost{
Name: routeName,
Domains: []string{serviceDomain},
Routes: routes,
}
return host, nil
}
func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute, protocol string) envoyroute.RouteMatch {

64
agent/xds/routes_test.go
View File

@ -4,9 +4,13 @@ import (
"path"
"sort"
"testing"
"time"
envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
"github.com/hashicorp/consul/agent/connect"
"github.com/hashicorp/consul/agent/consul/discoverychain"
"github.com/hashicorp/consul/agent/proxycfg"
"github.com/hashicorp/consul/agent/structs"
testinf "github.com/mitchellh/go-testing-interface"
"github.com/stretchr/testify/require"
)
@ -104,6 +108,66 @@ func TestRoutesFromSnapshot(t *testing.T) {
create: proxycfg.TestConfigSnapshotIngressWithRouter,
setup: nil,
},
{
name: "ingress-http-multiple-services",
create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{
proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{
{
DestinationName: "foo",
LocalBindPort: 8080,
},
{
DestinationName: "bar",
LocalBindPort: 8080,
},
},
proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{
{
DestinationName: "baz",
LocalBindPort: 443,
},
{
DestinationName: "qux",
LocalBindPort: 443,
},
},
}
// We do not add baz/qux here so that we test the chain.IsDefault() case
entries := []structs.ConfigEntry{
&structs.ProxyConfigEntry{
Kind: structs.ProxyDefaults,
Name: structs.ProxyConfigGlobal,
Config: map[string]interface{}{
"protocol": "http",
},
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "foo",
ConnectTimeout: 22 * time.Second,
},
&structs.ServiceResolverConfigEntry{
Kind: structs.ServiceResolver,
Name: "bar",
ConnectTimeout: 22 * time.Second,
},
}
fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
bazChain := discoverychain.TestCompileConfigEntries(t, "baz", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
quxChain := discoverychain.TestCompileConfigEntries(t, "qux", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...)
snap.IngressGateway.DiscoveryChain = map[string]*structs.CompiledDiscoveryChain{
"foo": fooChain,
"bar": barChain,
"baz": bazChain,
"qux": quxChain,
}
},
},
}
for _, tt := range tests {

85
agent/xds/testdata/listeners/ingress-http-multiple-services.golden vendored Normal file
View File

@ -0,0 +1,85 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "http:1.2.3.4:443",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 443
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"rds": {
"config_source": {
"ads": {
}
},
"route_config_name": "http_443"
},
"stat_prefix": "ingress_upstream_http_443_http",
"tracing": {
"operation_name": "EGRESS",
"random_sampling": {
}
}
}
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "http:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"rds": {
"config_source": {
"ads": {
}
},
"route_config_name": "http_8080"
},
"stat_prefix": "ingress_upstream_http_8080_http",
"tracing": {
"operation_name": "EGRESS",
"random_sampling": {
}
}
}
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}

8
agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden vendored
View File

@ -3,10 +3,10 @@
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "db:2.3.4.5:9191",
"name": "http:1.2.3.4:9191",
"address": {
"socketAddress": {
"address": "2.3.4.5",
"address": "1.2.3.4",
"portValue": 9191
}
},
@ -26,9 +26,9 @@
"ads": {
}
},
"route_config_name": "db"
"route_config_name": "http_9191"
},
"stat_prefix": "upstream_db_http",
"stat_prefix": "ingress_upstream_http_9191_http",
"tracing": {
"operation_name": "EGRESS",
"random_sampling": {

85
agent/xds/testdata/routes/ingress-http-multiple-services.golden vendored Normal file
View File

@ -0,0 +1,85 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "http_443",
"virtualHosts": [
{
"name": "baz",
"domains": [
"baz.*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "baz.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "qux",
"domains": [
"qux.*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "qux.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
},
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "http_8080",
"virtualHosts": [
{
"name": "foo",
"domains": [
"foo.*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
},
{
"name": "bar",
"domains": [
"bar.*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"
}

2
agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden vendored
View File

@ -3,7 +3,7 @@
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"name": "http_9191",
"virtualHosts": [
{
"name": "db",

23
agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden vendored
View File

@ -1,29 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"virtualHosts": [
{
"name": "db",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"

2
agent/xds/testdata/routes/ingress-with-chain-and-router.golden vendored
View File

@ -3,7 +3,7 @@
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"name": "http_9191",
"virtualHosts": [
{
"name": "db",

2
agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden vendored
View File

@ -3,7 +3,7 @@
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"name": "http_9191",
"virtualHosts": [
{
"name": "db",

23
agent/xds/testdata/routes/ingress-with-chain-external-sni.golden vendored
View File

@ -1,29 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"virtualHosts": [
{
"name": "db",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"

23
agent/xds/testdata/routes/ingress-with-chain.golden vendored
View File

@ -1,29 +1,6 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"virtualHosts": [
{
"name": "db",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul"
}
}
]
}
],
"validateClusters": true
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"nonce": "00000001"

2
agent/xds/testdata/routes/ingress-with-grpc-router.golden vendored
View File

@ -3,7 +3,7 @@
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "db",
"name": "http_9191",
"virtualHosts": [
{
"name": "db",

3
test/integration/connect/envoy/case-ingress-gateway-multiple-services/capture.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
snapshot_envoy_admin localhost:20000 ingress-gateway primary || true

29
test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl Normal file
View File

@ -0,0 +1,29 @@
enable_central_service_config = true
config_entries {
bootstrap = [
{
kind = "ingress-gateway"
name = "ingress-gateway"
listeners = [
{
port = 9999
protocol = "http"
services = [
{
name = "*"
}
]
}
]
},
{
kind = "proxy-defaults"
name = "global"
config {
protocol = "http"
}
}
]
}

4
test/integration/connect/envoy/case-ingress-gateway-multiple-services/gateway.hcl Normal file
View File

@ -0,0 +1,4 @@
services {
name = "ingress-gateway"
kind = "ingress-gateway"
}

11
test/integration/connect/envoy/case-ingress-gateway-multiple-services/setup.sh Normal file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -euo pipefail
# wait for bootstrap to apply config entries
wait_for_config_entry ingress-gateway ingress-gateway
wait_for_config_entry proxy-defaults global
gen_envoy_bootstrap ingress-gateway 20000 primary true
gen_envoy_bootstrap s1 19000
gen_envoy_bootstrap s2 19001

3
test/integration/connect/envoy/case-ingress-gateway-multiple-services/vars.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
export REQUIRED_SERVICES="$DEFAULT_REQUIRED_SERVICES ingress-gateway-primary"

58
test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats Normal file
View File

@ -0,0 +1,58 @@
#!/usr/bin/env bats
load helpers
@test "ingress proxy admin is up on :20000" {
retry_default curl -f -s localhost:20000/stats -o /dev/null
}
@test "s1 proxy admin is up on :19000" {
retry_default curl -f -s localhost:19000/stats -o /dev/null
}
@test "s2 proxy admin is up on :19001" {
retry_default curl -f -s localhost:19001/stats -o /dev/null
}
@test "s1 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21000 s1
}
@test "s2 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21001 s2
}
@test "ingress-gateway should have healthy endpoints for s1" {
assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1
}
@test "ingress-gateway should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1
}
@test "ingress should be able to connect to s1 using Host header" {
run retry_default curl -H"Host: s1.example.consul" -s -f localhost:9999/debug?env=dump
[ "$status" -eq 0 ]
GOT=$(echo "$output" | grep -E "^FORTIO_NAME=")
EXPECT_NAME="s1"
if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then
echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2
return 1
fi
}
@test "ingress should be able to connect to s2 using Host header" {
run retry_default curl -H"Host: s2.example.consul" -s -f localhost:9999/debug?env=dump
[ "$status" -eq 0 ]
GOT=$(echo "$output" | grep -E "^FORTIO_NAME=")
EXPECT_NAME="s2"
if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then
echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2
return 1
fi
}