diff --git a/agent/consul/internal_endpoint_test.go b/agent/consul/internal_endpoint_test.go index 9645865b1c..f36ea42d15 100644 --- a/agent/consul/internal_endpoint_test.go +++ b/agent/consul/internal_endpoint_test.go @@ -935,6 +935,7 @@ func TestInternal_GatewayServices_BothGateways(t *testing.T) { Service: structs.NewServiceID("db", nil), Gateway: structs.NewServiceID("ingress", nil), GatewayKind: structs.ServiceKindIngressGateway, + Protocol: "tcp", Port: 8888, }, } diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index cd01e69aa2..ad03f466b0 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -2540,6 +2540,7 @@ func (s *Store) ingressConfigGatewayServices(tx *memdb.Txn, gateway structs.Serv Service: service.ToServiceID(), GatewayKind: structs.ServiceKindIngressGateway, Port: listener.Port, + Protocol: listener.Protocol, } gatewayServices = append(gatewayServices, mapping) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index f46f104b87..ed7b879c8c 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -2,6 +2,8 @@ package proxycfg import ( "context" + "fmt" + "github.com/hashicorp/consul/agent/structs" "github.com/mitchellh/copystructure" ) @@ -194,7 +196,7 @@ type configSnapshotIngressGateway struct { // Upstreams is a list of upstreams this ingress gateway should serve traffic // to. This is constructed from the ingress-gateway config entry, and uses // the GatewayServices RPC to retrieve them. - Upstreams []structs.Upstream + Upstreams map[IngressListenerKey]structs.Upstreams // WatchedDiscoveryChains is a map of upstream.Identifier() -> CancelFunc's // in order to cancel any watches when the ingress gateway configuration is @@ -214,6 +216,15 @@ func (c *configSnapshotIngressGateway) IsEmpty() bool { len(c.WatchedUpstreamEndpoints) == 0 } +type IngressListenerKey struct { + Protocol string + Port int +} + +func (k *IngressListenerKey) RouteName() string { + return fmt.Sprintf("%s_%d", k.Protocol, k.Port) +} + // ConfigSnapshot captures all the resulting config needed for a proxy instance. // It is meant to be point-in-time coherent and is used to deliver the current // config state to observers who need it to be pushed in (e.g. XDS server). diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 1a9898108c..d87988448e 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -1320,8 +1320,8 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap return fmt.Errorf("invalid type for response: %T", u.Result) } - var upstreams structs.Upstreams watchedSvcs := make(map[string]struct{}) + upstreamsMap := make(map[IngressListenerKey]structs.Upstreams) for _, service := range services.Services { u := makeUpstream(service, s.address) @@ -1330,9 +1330,11 @@ func (s *state) handleUpdateIngressGateway(u cache.UpdateEvent, snap *ConfigSnap return err } watchedSvcs[u.Identifier()] = struct{}{} - upstreams = append(upstreams, u) + + id := IngressListenerKey{Protocol: service.Protocol, Port: service.Port} + upstreamsMap[id] = append(upstreamsMap[id], u) } - snap.IngressGateway.Upstreams = upstreams + snap.IngressGateway.Upstreams = upstreamsMap for id, cancelFn := range snap.IngressGateway.WatchedDiscoveryChains { if _, ok := watchedSvcs[id]; !ok { diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 75e7bfaaf6..5c4292c94c 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -810,6 +810,66 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, }, }, + "ingress-gateway-update-upstreams": testCase{ + ns: structs.NodeService{ + Kind: structs.ServiceKindIngressGateway, + ID: "ingress-gateway", + Service: "ingress-gateway", + Address: "10.0.1.1", + }, + sourceDC: "dc1", + stages: []verificationStage{ + verificationStage{ + requiredWatches: map[string]verifyWatchRequest{ + rootsWatchID: genVerifyRootsWatch("dc1"), + leafWatchID: genVerifyLeafWatch("ingress-gateway", "dc1"), + }, + events: []cache.UpdateEvent{ + rootWatchEvent(), + cache.UpdateEvent{ + CorrelationID: leafWatchID, + Result: issuedCert, + Err: nil, + }, + cache.UpdateEvent{ + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{ + Services: structs.GatewayServices{ + { + Gateway: structs.NewServiceID("ingress-gateway", nil), + Service: structs.NewServiceID("api", nil), + Port: 9999, + }, + }, + }, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.Len(t, snap.IngressGateway.Upstreams, 1) + require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 1) + require.Contains(t, snap.IngressGateway.WatchedDiscoveryChains, "api") + }, + }, + verificationStage{ + requiredWatches: map[string]verifyWatchRequest{}, + events: []cache.UpdateEvent{ + cache.UpdateEvent{ + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{}, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.Len(t, snap.IngressGateway.Upstreams, 0) + require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 0) + require.NotContains(t, snap.IngressGateway.WatchedDiscoveryChains, "api") + }, + }, + }, + }, "terminating-gateway-initial": testCase{ ns: structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, diff --git a/agent/proxycfg/testing.go b/agent/proxycfg/testing.go index 7f6afc99c4..7c5d478f4e 100644 --- a/agent/proxycfg/testing.go +++ b/agent/proxycfg/testing.go @@ -1143,6 +1143,7 @@ func setupTestVariationConfigEntriesAndSnapshot( }, }, ) + case "http-multiple-services": default: t.Fatalf("unexpected variation: %q", variation) return ConfigSnapshotUpstreams{} @@ -1233,6 +1234,13 @@ func setupTestVariationConfigEntriesAndSnapshot( case "chain-and-splitter": case "grpc-router": case "chain-and-router": + case "http-multiple-services": + snap.WatchedUpstreamEndpoints["foo"] = map[string]structs.CheckServiceNodes{ + "foo.default.dc1": TestUpstreamNodes(t), + } + snap.WatchedUpstreamEndpoints["bar"] = map[string]structs.CheckServiceNodes{ + "bar.default.dc1": TestUpstreamNodesAlternate(t), + } default: t.Fatalf("unexpected variation: %q", variation) return ConfigSnapshotUpstreams{} @@ -1312,82 +1320,86 @@ func testConfigSnapshotMeshGateway(t testing.T, populateServices bool, useFedera } func TestConfigSnapshotIngress(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "simple") + return testConfigSnapshotIngressGateway(t, true, "tcp", "simple") } func TestConfigSnapshotIngressWithOverrides(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "simple-with-overrides") + return testConfigSnapshotIngressGateway(t, true, "tcp", "simple-with-overrides") } func TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "splitter-with-resolver-redirect-multidc") + return testConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc") +} + +func TestConfigSnapshotIngress_HTTPMultipleServices(t testing.T) *ConfigSnapshot { + return testConfigSnapshotIngressGateway(t, true, "http", "http-multiple-services") } func TestConfigSnapshotIngressExternalSNI(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "external-sni") + return testConfigSnapshotIngressGateway(t, true, "tcp", "external-sni") } func TestConfigSnapshotIngressWithFailover(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover") } func TestConfigSnapshotIngressWithFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway") } func TestConfigSnapshotIngressWithFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-remote-gateway-triggered") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered") } func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGateway(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway") } func TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGatewayTriggered(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-double-remote-gateway-triggered") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered") } func TestConfigSnapshotIngressWithFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway") } func TestConfigSnapshotIngressWithFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-local-gateway-triggered") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered") } func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGateway(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway") } func TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGatewayTriggered(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "failover-through-double-local-gateway-triggered") + return testConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered") } func TestConfigSnapshotIngressWithSplitter(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "chain-and-splitter") + return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-splitter") } func TestConfigSnapshotIngressWithGRPCRouter(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "grpc-router") + return testConfigSnapshotIngressGateway(t, true, "http", "grpc-router") } func TestConfigSnapshotIngressWithRouter(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "chain-and-router") + return testConfigSnapshotIngressGateway(t, true, "http", "chain-and-router") } func TestConfigSnapshotIngressGateway(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "default") + return testConfigSnapshotIngressGateway(t, true, "tcp", "default") } func TestConfigSnapshotIngressGatewayNoServices(t testing.T) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, false, "default") + return testConfigSnapshotIngressGateway(t, false, "tcp", "default") } func TestConfigSnapshotIngressDiscoveryChainWithEntries(t testing.T, additionalEntries ...structs.ConfigEntry) *ConfigSnapshot { - return testConfigSnapshotIngressGateway(t, true, "simple", additionalEntries...) + return testConfigSnapshotIngressGateway(t, true, "http", "simple", additionalEntries...) } func testConfigSnapshotIngressGateway( - t testing.T, populateServices bool, variation string, + t testing.T, populateServices bool, protocol, variation string, additionalEntries ...structs.ConfigEntry, ) *ConfigSnapshot { roots, leaf := TestCerts(t) @@ -1404,12 +1416,14 @@ func testConfigSnapshotIngressGateway( ConfigSnapshotUpstreams: setupTestVariationConfigEntriesAndSnapshot( t, variation, leaf, additionalEntries..., ), - Upstreams: structs.Upstreams{ - { - // We rely on this one having default type in a few tests... - DestinationName: "db", - LocalBindPort: 9191, - LocalBindAddress: "2.3.4.5", + Upstreams: map[IngressListenerKey]structs.Upstreams{ + IngressListenerKey{protocol, 9191}: structs.Upstreams{ + { + // We rely on this one having default type in a few tests... + DestinationName: "db", + LocalBindPort: 9191, + LocalBindAddress: "2.3.4.5", + }, }, }, } diff --git a/agent/structs/config_entry_gateways.go b/agent/structs/config_entry_gateways.go index 646a2768d3..86374a7122 100644 --- a/agent/structs/config_entry_gateways.go +++ b/agent/structs/config_entry_gateways.go @@ -300,6 +300,7 @@ type GatewayService struct { Service ServiceID GatewayKind ServiceKind Port int + Protocol string CAFile string CertFile string KeyFile string @@ -315,6 +316,7 @@ func (g *GatewayService) IsSame(o *GatewayService) bool { g.Service.Matches(&o.Service) && g.GatewayKind == o.GatewayKind && g.Port == o.Port && + g.Protocol == o.Protocol && g.CAFile == o.CAFile && g.CertFile == o.CertFile && g.KeyFile == o.KeyFile && @@ -328,6 +330,7 @@ func (g *GatewayService) Clone() *GatewayService { Service: g.Service, GatewayKind: g.GatewayKind, Port: g.Port, + Protocol: g.Protocol, CAFile: g.CAFile, CertFile: g.CertFile, KeyFile: g.KeyFile, diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index 85837c70d6..55ebe54eea 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -236,27 +236,29 @@ func (s *Server) makeGatewayServiceClusters(cfgSnap *proxycfg.ConfigSnapshot) ([ func (s *Server) clustersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var clusters []proto.Message - for _, u := range cfgSnap.IngressGateway.Upstreams { - id := u.Identifier() - chain, ok := cfgSnap.IngressGateway.DiscoveryChain[id] - if !ok { - // this should not happen - return nil, fmt.Errorf("no discovery chain for upstream %q", id) - } + for _, upstreams := range cfgSnap.IngressGateway.Upstreams { + for _, u := range upstreams { + id := u.Identifier() + chain, ok := cfgSnap.IngressGateway.DiscoveryChain[id] + if !ok { + // this should not happen + return nil, fmt.Errorf("no discovery chain for upstream %q", id) + } - chainEndpoints, ok := cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id] - if !ok { - // this should not happen - return nil, fmt.Errorf("no endpoint map for upstream %q", id) - } + chainEndpoints, ok := cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id] + if !ok { + // this should not happen + return nil, fmt.Errorf("no endpoint map for upstream %q", id) + } - upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain(u, chain, chainEndpoints, cfgSnap) - if err != nil { - return nil, err - } + upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain(u, chain, chainEndpoints, cfgSnap) + if err != nil { + return nil, err + } - for _, c := range upstreamClusters { - clusters = append(clusters, c) + for _, c := range upstreamClusters { + clusters = append(clusters, c) + } } } return clusters, nil diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index b82fb16cbd..f5dc0517fa 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -255,16 +255,18 @@ func (s *Server) endpointsFromServicesAndResolvers( func (s *Server) endpointsFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var resources []proto.Message - for _, u := range cfgSnap.IngressGateway.Upstreams { - id := u.Identifier() + for _, upstreams := range cfgSnap.IngressGateway.Upstreams { + for _, u := range upstreams { + id := u.Identifier() - es := s.endpointsFromDiscoveryChain( - cfgSnap.IngressGateway.DiscoveryChain[id], - cfgSnap.Datacenter, - cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id], - cfgSnap.IngressGateway.WatchedGatewayEndpoints[id], - ) - resources = append(resources, es...) + es := s.endpointsFromDiscoveryChain( + cfgSnap.IngressGateway.DiscoveryChain[id], + cfgSnap.Datacenter, + cfgSnap.IngressGateway.WatchedUpstreamEndpoints[id], + cfgSnap.IngressGateway.WatchedGatewayEndpoints[id], + ) + resources = append(resources, es...) + } } return resources, nil } diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 29f13150ab..f39895b712 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -270,25 +270,47 @@ func (s *Server) listenersFromSnapshotGateway(cfgSnap *proxycfg.ConfigSnapshot, // See: https://www.consul.io/docs/connect/proxies/envoy.html#mesh-gateway-options func (s *Server) listenersFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { var resources []proto.Message - // TODO(ingress): We give each upstream a distinct listener at the moment, - // for http listeners we will need to multiplex upstreams on a single - // listener. - for _, u := range cfgSnap.IngressGateway.Upstreams { - id := u.Identifier() + for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams { + if listenerKey.Protocol == "tcp" { + u := upstreams[0] + id := u.Identifier() - chain := cfgSnap.IngressGateway.DiscoveryChain[id] + chain := cfgSnap.IngressGateway.DiscoveryChain[id] - var upstreamListener proto.Message - var err error - if chain == nil || chain.IsDefault() { - upstreamListener, err = s.makeUpstreamListenerIgnoreDiscoveryChain(&u, chain, cfgSnap) + var upstreamListener proto.Message + var err error + if chain == nil || chain.IsDefault() { + upstreamListener, err = s.makeUpstreamListenerIgnoreDiscoveryChain(&u, chain, cfgSnap) + } else { + upstreamListener, err = s.makeUpstreamListenerForDiscoveryChain(&u, chain, cfgSnap) + } + if err != nil { + return nil, err + } + resources = append(resources, upstreamListener) } else { - upstreamListener, err = s.makeUpstreamListenerForDiscoveryChain(&u, chain, cfgSnap) + // If multiple upstreams share this port, make a special listener for the protocol. + addr := cfgSnap.Address + if addr == "" { + addr = "0.0.0.0" + } + + listener := makeListener(listenerKey.Protocol, addr, listenerKey.Port) + filter, err := makeListenerFilter( + true, listenerKey.Protocol, listenerKey.RouteName(), "", "ingress_upstream_", "", false) + if err != nil { + return nil, err + } + + listener.FilterChains = []envoylistener.FilterChain{ + { + Filters: []envoylistener.Filter{ + filter, + }, + }, + } + resources = append(resources, listener) } - if err != nil { - return nil, err - } - resources = append(resources, upstreamListener) } return resources, nil diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 8a79aca8f8..7a2507eae1 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -363,6 +363,34 @@ func TestListenersFromSnapshot(t *testing.T) { } }, }, + { + name: "ingress-http-multiple-services", + create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{ + proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{ + { + DestinationName: "foo", + LocalBindPort: 8080, + }, + { + DestinationName: "bar", + LocalBindPort: 8080, + }, + }, + proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{ + { + DestinationName: "baz", + LocalBindPort: 443, + }, + { + DestinationName: "qux", + LocalBindPort: 443, + }, + }, + } + }, + }, { name: "terminating-gateway-no-api-cert", create: proxycfg.TestConfigSnapshotTerminatingGateway, diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 83215a4593..5036d2a82d 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -37,7 +37,34 @@ func routesFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.M return nil, errors.New("nil config given") } - return routesFromUpstreams(cfgSnap.ConnectProxy.ConfigSnapshotUpstreams, cfgSnap.Proxy.Upstreams) + var resources []proto.Message + for _, u := range cfgSnap.Proxy.Upstreams { + upstreamID := u.Identifier() + + var chain *structs.CompiledDiscoveryChain + if u.DestinationType != structs.UpstreamDestTypePreparedQuery { + chain = cfgSnap.ConnectProxy.DiscoveryChain[upstreamID] + } + + if chain == nil || chain.IsDefault() { + // TODO(rb): make this do the old school stuff too + } else { + virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, "*") + if err != nil { + return nil, err + } + + route := &envoy.RouteConfiguration{ + Name: upstreamID, + VirtualHosts: []envoyroute.VirtualHost{*virtualHost}, + ValidateClusters: makeBoolValue(true), + } + resources = append(resources, route) + } + } + + // TODO(rb): make sure we don't generate an empty result + return resources, nil } // routesFromSnapshotIngressGateway returns the xDS API representation of the @@ -47,45 +74,51 @@ func routesFromSnapshotIngressGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto return nil, errors.New("nil config given") } - return routesFromUpstreams(cfgSnap.IngressGateway.ConfigSnapshotUpstreams, cfgSnap.IngressGateway.Upstreams) -} - -func routesFromUpstreams(snap proxycfg.ConfigSnapshotUpstreams, upstreams structs.Upstreams) ([]proto.Message, error) { - var resources []proto.Message - - for _, u := range upstreams { - upstreamID := u.Identifier() - - var chain *structs.CompiledDiscoveryChain - if u.DestinationType != structs.UpstreamDestTypePreparedQuery { - chain = snap.DiscoveryChain[upstreamID] + var result []proto.Message + for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams { + // Do not create any route configuration for TCP listeners + if listenerKey.Protocol == "tcp" { + continue } - if chain == nil || chain.IsDefault() { - // TODO(rb): make this do the old school stuff too - } else { - upstreamRoute, err := makeUpstreamRouteForDiscoveryChain(&u, chain) - if err != nil { - return nil, err - } - if upstreamRoute != nil { - resources = append(resources, upstreamRoute) + upstreamRoute := &envoy.RouteConfiguration{ + Name: listenerKey.RouteName(), + // ValidateClusters defaults to true when defined statically and false + // when done via RDS. Re-set the sane value of true to prevent + // null-routing traffic. + ValidateClusters: makeBoolValue(true), + } + for _, u := range upstreams { + upstreamID := u.Identifier() + chain := cfgSnap.IngressGateway.DiscoveryChain[upstreamID] + if chain != nil { + domain := fmt.Sprintf("%s.*", chain.ServiceName) + // Don't require a service prefix on the domain if there is only 1 + // upstream. This makes it a smoother experience when only having a + // single service associated to a listener, which is probably a common + // case when demoing/testing + if len(upstreams) == 1 { + domain = "*" + } + virtualHost, err := makeUpstreamRouteForDiscoveryChain(upstreamID, chain, domain) + if err != nil { + return nil, err + } + upstreamRoute.VirtualHosts = append(upstreamRoute.VirtualHosts, *virtualHost) } } + + result = append(result, upstreamRoute) } - // TODO(rb): make sure we don't generate an empty result - - return resources, nil + return result, nil } func makeUpstreamRouteForDiscoveryChain( - u *structs.Upstream, + routeName string, chain *structs.CompiledDiscoveryChain, -) (*envoy.RouteConfiguration, error) { - upstreamID := u.Identifier() - routeName := upstreamID - + serviceDomain string, +) (*envoyroute.VirtualHost, error) { var routes []envoyroute.Route startNode := chain.Nodes[chain.StartNode] @@ -188,20 +221,13 @@ func makeUpstreamRouteForDiscoveryChain( panic("unknown first node in discovery chain of type: " + startNode.Type) } - return &envoy.RouteConfiguration{ - Name: routeName, - VirtualHosts: []envoyroute.VirtualHost{ - envoyroute.VirtualHost{ - Name: routeName, - Domains: []string{"*"}, - Routes: routes, - }, - }, - // ValidateClusters defaults to true when defined statically and false - // when done via RDS. Re-set the sane value of true to prevent - // null-routing traffic. - ValidateClusters: makeBoolValue(true), - }, nil + host := &envoyroute.VirtualHost{ + Name: routeName, + Domains: []string{serviceDomain}, + Routes: routes, + } + + return host, nil } func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute, protocol string) envoyroute.RouteMatch { diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index bb016e8103..c0c0fa7a2f 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -4,9 +4,13 @@ import ( "path" "sort" "testing" + "time" envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/consul/discoverychain" "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" testinf "github.com/mitchellh/go-testing-interface" "github.com/stretchr/testify/require" ) @@ -104,6 +108,66 @@ func TestRoutesFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotIngressWithRouter, setup: nil, }, + { + name: "ingress-http-multiple-services", + create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices, + setup: func(snap *proxycfg.ConfigSnapshot) { + snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{ + proxycfg.IngressListenerKey{Protocol: "http", Port: 8080}: structs.Upstreams{ + { + DestinationName: "foo", + LocalBindPort: 8080, + }, + { + DestinationName: "bar", + LocalBindPort: 8080, + }, + }, + proxycfg.IngressListenerKey{Protocol: "http", Port: 443}: structs.Upstreams{ + { + DestinationName: "baz", + LocalBindPort: 443, + }, + { + DestinationName: "qux", + LocalBindPort: 443, + }, + }, + } + + // We do not add baz/qux here so that we test the chain.IsDefault() case + entries := []structs.ConfigEntry{ + &structs.ProxyConfigEntry{ + Kind: structs.ProxyDefaults, + Name: structs.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "http", + }, + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "foo", + ConnectTimeout: 22 * time.Second, + }, + &structs.ServiceResolverConfigEntry{ + Kind: structs.ServiceResolver, + Name: "bar", + ConnectTimeout: 22 * time.Second, + }, + } + fooChain := discoverychain.TestCompileConfigEntries(t, "foo", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + barChain := discoverychain.TestCompileConfigEntries(t, "bar", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + bazChain := discoverychain.TestCompileConfigEntries(t, "baz", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + quxChain := discoverychain.TestCompileConfigEntries(t, "qux", "default", "dc1", connect.TestClusterID+".consul", "dc1", nil, entries...) + + snap.IngressGateway.DiscoveryChain = map[string]*structs.CompiledDiscoveryChain{ + "foo": fooChain, + "bar": barChain, + "baz": bazChain, + "qux": quxChain, + } + }, + }, } for _, tt := range tests { diff --git a/agent/xds/testdata/listeners/ingress-http-multiple-services.golden b/agent/xds/testdata/listeners/ingress-http-multiple-services.golden new file mode 100644 index 0000000000..fed7659316 --- /dev/null +++ b/agent/xds/testdata/listeners/ingress-http-multiple-services.golden @@ -0,0 +1,85 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "http:1.2.3.4:443", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 443 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "rds": { + "config_source": { + "ads": { + } + }, + "route_config_name": "http_443" + }, + "stat_prefix": "ingress_upstream_http_443_http", + "tracing": { + "operation_name": "EGRESS", + "random_sampling": { + } + } + } + } + ] + } + ] + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.http_connection_manager", + "config": { + "http_filters": [ + { + "name": "envoy.router" + } + ], + "rds": { + "config_source": { + "ads": { + } + }, + "route_config_name": "http_8080" + }, + "stat_prefix": "ingress_upstream_http_8080_http", + "tracing": { + "operation_name": "EGRESS", + "random_sampling": { + } + } + } + } + ] + } + ] + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden b/agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden index ff17ba57d2..872b2dc062 100644 --- a/agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden +++ b/agent/xds/testdata/listeners/ingress-splitter-with-resolver-redirect.golden @@ -3,10 +3,10 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.Listener", - "name": "db:2.3.4.5:9191", + "name": "http:1.2.3.4:9191", "address": { "socketAddress": { - "address": "2.3.4.5", + "address": "1.2.3.4", "portValue": 9191 } }, @@ -26,9 +26,9 @@ "ads": { } }, - "route_config_name": "db" + "route_config_name": "http_9191" }, - "stat_prefix": "upstream_db_http", + "stat_prefix": "ingress_upstream_http_9191_http", "tracing": { "operation_name": "EGRESS", "random_sampling": { diff --git a/agent/xds/testdata/routes/ingress-http-multiple-services.golden b/agent/xds/testdata/routes/ingress-http-multiple-services.golden new file mode 100644 index 0000000000..d458dc1107 --- /dev/null +++ b/agent/xds/testdata/routes/ingress-http-multiple-services.golden @@ -0,0 +1,85 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "http_443", + "virtualHosts": [ + { + "name": "baz", + "domains": [ + "baz.*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "baz.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "qux", + "domains": [ + "qux.*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "qux.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + }, + { + "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "name": "http_8080", + "virtualHosts": [ + { + "name": "foo", + "domains": [ + "foo.*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + }, + { + "name": "bar", + "domains": [ + "bar.*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + } + } + ] + } + ], + "validateClusters": true + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden b/agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden index 8c2f58a985..a521f8377d 100644 --- a/agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden +++ b/agent/xds/testdata/routes/ingress-splitter-with-resolver-redirect.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", + "name": "http_9191", "virtualHosts": [ { "name": "db", diff --git a/agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden b/agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden index 357c4a3c35..a065cc73d0 100644 --- a/agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden +++ b/agent/xds/testdata/routes/ingress-with-chain-and-overrides.golden @@ -1,29 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", - "virtualHosts": [ - { - "name": "db", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" - } - } - ] - } - ], - "validateClusters": true - } ], "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", "nonce": "00000001" diff --git a/agent/xds/testdata/routes/ingress-with-chain-and-router.golden b/agent/xds/testdata/routes/ingress-with-chain-and-router.golden index abd0c8330f..306f3545db 100644 --- a/agent/xds/testdata/routes/ingress-with-chain-and-router.golden +++ b/agent/xds/testdata/routes/ingress-with-chain-and-router.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", + "name": "http_9191", "virtualHosts": [ { "name": "db", diff --git a/agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden b/agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden index efe364bad5..90a78f1ba6 100644 --- a/agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden +++ b/agent/xds/testdata/routes/ingress-with-chain-and-splitter.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", + "name": "http_9191", "virtualHosts": [ { "name": "db", diff --git a/agent/xds/testdata/routes/ingress-with-chain-external-sni.golden b/agent/xds/testdata/routes/ingress-with-chain-external-sni.golden index a0b6cb832b..a065cc73d0 100644 --- a/agent/xds/testdata/routes/ingress-with-chain-external-sni.golden +++ b/agent/xds/testdata/routes/ingress-with-chain-external-sni.golden @@ -1,29 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", - "virtualHosts": [ - { - "name": "db", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" - } - } - ] - } - ], - "validateClusters": true - } ], "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", "nonce": "00000001" diff --git a/agent/xds/testdata/routes/ingress-with-chain.golden b/agent/xds/testdata/routes/ingress-with-chain.golden index a0b6cb832b..a065cc73d0 100644 --- a/agent/xds/testdata/routes/ingress-with-chain.golden +++ b/agent/xds/testdata/routes/ingress-with-chain.golden @@ -1,29 +1,6 @@ { "versionInfo": "00000001", "resources": [ - { - "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", - "virtualHosts": [ - { - "name": "db", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" - } - } - ] - } - ], - "validateClusters": true - } ], "typeUrl": "type.googleapis.com/envoy.api.v2.RouteConfiguration", "nonce": "00000001" diff --git a/agent/xds/testdata/routes/ingress-with-grpc-router.golden b/agent/xds/testdata/routes/ingress-with-grpc-router.golden index 244a2e7ef5..3b1bf9b859 100644 --- a/agent/xds/testdata/routes/ingress-with-grpc-router.golden +++ b/agent/xds/testdata/routes/ingress-with-grpc-router.golden @@ -3,7 +3,7 @@ "resources": [ { "@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration", - "name": "db", + "name": "http_9191", "virtualHosts": [ { "name": "db", diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/capture.sh b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/capture.sh new file mode 100644 index 0000000000..41ea5cb24f --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/capture.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +snapshot_envoy_admin localhost:20000 ingress-gateway primary || true \ No newline at end of file diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl new file mode 100644 index 0000000000..b99168d2b7 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/config_entries.hcl @@ -0,0 +1,29 @@ +enable_central_service_config = true + +config_entries { + bootstrap = [ + { + kind = "ingress-gateway" + name = "ingress-gateway" + + listeners = [ + { + port = 9999 + protocol = "http" + services = [ + { + name = "*" + } + ] + } + ] + }, + { + kind = "proxy-defaults" + name = "global" + config { + protocol = "http" + } + } + ] +} diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/gateway.hcl b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/gateway.hcl new file mode 100644 index 0000000000..781ef1851b --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/gateway.hcl @@ -0,0 +1,4 @@ +services { + name = "ingress-gateway" + kind = "ingress-gateway" +} diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/setup.sh b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/setup.sh new file mode 100644 index 0000000000..9a2c4ab9ed --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +# wait for bootstrap to apply config entries +wait_for_config_entry ingress-gateway ingress-gateway +wait_for_config_entry proxy-defaults global + +gen_envoy_bootstrap ingress-gateway 20000 primary true +gen_envoy_bootstrap s1 19000 +gen_envoy_bootstrap s2 19001 diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/vars.sh b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/vars.sh new file mode 100644 index 0000000000..c97ad2ea54 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/vars.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +export REQUIRED_SERVICES="$DEFAULT_REQUIRED_SERVICES ingress-gateway-primary" diff --git a/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats new file mode 100644 index 0000000000..26a6427003 --- /dev/null +++ b/test/integration/connect/envoy/case-ingress-gateway-multiple-services/verify.bats @@ -0,0 +1,58 @@ +#!/usr/bin/env bats + +load helpers + +@test "ingress proxy admin is up on :20000" { + retry_default curl -f -s localhost:20000/stats -o /dev/null +} + +@test "s1 proxy admin is up on :19000" { + retry_default curl -f -s localhost:19000/stats -o /dev/null +} + +@test "s2 proxy admin is up on :19001" { + retry_default curl -f -s localhost:19001/stats -o /dev/null +} + +@test "s1 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21000 s1 +} + +@test "s2 proxy listener should be up and have right cert" { + assert_proxy_presents_cert_uri localhost:21001 s2 +} + +@test "ingress-gateway should have healthy endpoints for s1" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s1 HEALTHY 1 +} + +@test "ingress-gateway should have healthy endpoints for s2" { + assert_upstream_has_endpoints_in_status 127.0.0.1:20000 s2 HEALTHY 1 +} + +@test "ingress should be able to connect to s1 using Host header" { + run retry_default curl -H"Host: s1.example.consul" -s -f localhost:9999/debug?env=dump + [ "$status" -eq 0 ] + + GOT=$(echo "$output" | grep -E "^FORTIO_NAME=") + EXPECT_NAME="s1" + + if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then + echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2 + return 1 + fi +} + +@test "ingress should be able to connect to s2 using Host header" { + run retry_default curl -H"Host: s2.example.consul" -s -f localhost:9999/debug?env=dump + [ "$status" -eq 0 ] + + GOT=$(echo "$output" | grep -E "^FORTIO_NAME=") + EXPECT_NAME="s2" + + if [ "$GOT" != "FORTIO_NAME=${EXPECT_NAME}" ]; then + echo "expected name: $EXPECT_NAME, actual name: $GOT" 1>&2 + return 1 + fi +} +