xds: generate routes directly from API gateway snapshot (#17392)

* xds generation for routes api gateway

* Update gateway.go

* move buildHttpRoute into xds package

* Update agent/consul/discoverychain/gateway.go

* remove unneeded function

* convert http route code to only run for http protocol to future proof code path

* Update agent/consul/discoverychain/gateway.go

Co-authored-by: Mike Morris <mikemorris@users.noreply.github.com>

* fix tests, clean up http check logic

* clean up todo

* Fix casing in docstring

* Fix import block, adjust docstrings

* update name and comment

* use constant value

* use constant

---------

Co-authored-by: Mike Morris <mikemorris@users.noreply.github.com>
Co-authored-by: Nathan Coleman <nathan.coleman@hashicorp.com>

agent/consul/discoverychain/gateway.go

@@ -59,10 +59,10 @@ func (l *GatewayChainSynthesizer) SetHostname(hostname string) {
 // single hostname can be specified in multiple routes. Routing for a given
 // hostname must behave based on the aggregate of all rules that apply to it.
 func (l *GatewayChainSynthesizer) AddHTTPRoute(route structs.HTTPRouteConfigEntry) {
-	l.matchesByHostname = getHostMatches(l.hostname, &route, l.matchesByHostname)
+	l.matchesByHostname = initHostMatches(l.hostname, &route, l.matchesByHostname)
 }
 
-func getHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, currentMatches map[string][]hostnameMatch) map[string][]hostnameMatch {
+func initHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, currentMatches map[string][]hostnameMatch) map[string][]hostnameMatch {
 	hostnames := route.FilteredHostnames(hostname)
 	for _, host := range hostnames {
 		matches, ok := currentMatches[host]
@@ -196,16 +196,22 @@ func (l *GatewayChainSynthesizer) consolidateHTTPRoutes() []structs.HTTPRouteCon
 	return consolidateHTTPRoutes(l.matchesByHostname, l.suffix, l.gateway)
 }
 
+// ReformatHTTPRoute takes in an HTTPRoute and reformats it to match the discovery chains generated by the gateway chain synthesizer
+func ReformatHTTPRoute(route *structs.HTTPRouteConfigEntry, listener *structs.APIGatewayListener, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry {
+	matches := initHostMatches(listener.GetHostname(), route, map[string][]hostnameMatch{})
+	return consolidateHTTPRoutes(matches, listener.Name, gateway)
+}
+
 // consolidateHTTPRoutes combines all rules into the shortest possible list of routes
 // with one route per hostname containing all rules for that hostname.
-func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, suffix string, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry {
+func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, listenerName string, gateway *structs.APIGatewayConfigEntry) []structs.HTTPRouteConfigEntry {
 	var routes []structs.HTTPRouteConfigEntry
 
 	for hostname, rules := range matchesByHostname {
 		// Create route for this hostname
 		route := structs.HTTPRouteConfigEntry{
 			Kind:           structs.HTTPRoute,
-			Name:           fmt.Sprintf("%s-%s-%s", gateway.Name, suffix, hostsKey(hostname)),
+			Name:           fmt.Sprintf("%s-%s-%s", gateway.Name, listenerName, hostsKey(hostname)),
 			Hostnames:      []string{hostname},
 			Rules:          make([]structs.HTTPRouteRule, 0, len(rules)),
 			Meta:           gateway.Meta,

agent/xds/listeners_apigateway.go

@@ -43,7 +43,7 @@ func (s *ResourceGenerator) makeAPIGatewayListeners(address string, cfgSnap *pro
 			return nil, err
 		}
 
-		if listenerKey.Protocol == "tcp" {
+		if listenerCfg.Protocol == structs.ListenerProtocolTCP {
 			// Find the upstream matching this listener
 
 			// We rely on the invariant of upstreams slice always having at least 1

agent/xds/routes.go

@@ -19,6 +19,7 @@ import (
 	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/consul/discoverychain"
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 )
@@ -36,13 +37,7 @@ func (s *ResourceGenerator) routesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot)
 	case structs.ServiceKindIngressGateway:
 		return s.routesForIngressGateway(cfgSnap)
 	case structs.ServiceKindAPIGateway:
-		// TODO Find a cleaner solution, can't currently pass unexported property types
+		return s.routesForAPIGateway(cfgSnap)
-		var err error
-		cfgSnap.IngressGateway, err = cfgSnap.APIGateway.ToIngress(cfgSnap.Datacenter)
-		if err != nil {
-			return nil, err
-		}
-		return s.routesForIngressGateway(cfgSnap)
 	case structs.ServiceKindTerminatingGateway:
 		return s.routesForTerminatingGateway(cfgSnap)
 	case structs.ServiceKindMeshGateway:
@@ -430,6 +425,85 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap
 	return result, nil
 }
 
+// routesForAPIGateway returns the xDS API representation of the
+// "routes" in the snapshot.
+func (s *ResourceGenerator) routesForAPIGateway(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
+	var result []proto.Message
+
+	readyUpstreamsList := getReadyUpstreams(cfgSnap)
+
+	for _, readyUpstreams := range readyUpstreamsList {
+		listenerCfg := readyUpstreams.listenerCfg
+		// Do not create any route configuration for TCP listeners
+		if listenerCfg.Protocol != structs.ListenerProtocolHTTP {
+			continue
+		}
+
+		routeRef := readyUpstreams.routeReference
+		listenerKey := readyUpstreams.listenerKey
+
+		defaultRoute := &envoy_route_v3.RouteConfiguration{
+			Name: listenerKey.RouteName(),
+			// ValidateClusters defaults to true when defined statically and false
+			// when done via RDS. Re-set the reasonable value of true to prevent
+			// null-routing traffic.
+			ValidateClusters: makeBoolValue(true),
+		}
+
+		route, ok := cfgSnap.APIGateway.HTTPRoutes.Get(routeRef)
+		if !ok {
+			return nil, fmt.Errorf("missing route for route reference %s:%s", routeRef.Name, routeRef.Kind)
+		}
+
+		// Reformat the route here since discovery chains were indexed earlier using the
+		// specific naming convention in discoverychain.consolidateHTTPRoutes. If we don't
+		// convert our route to use the same naming convention, we won't find any chains below.
+		reformatedRoutes := discoverychain.ReformatHTTPRoute(route, &listenerCfg, cfgSnap.APIGateway.GatewayConfig)
+
+		for _, reformatedRoute := range reformatedRoutes {
+			reformatedRoute := reformatedRoute
+
+			upstream := buildHTTPRouteUpstream(reformatedRoute, listenerCfg)
+			uid := proxycfg.NewUpstreamID(&upstream)
+			chain := cfgSnap.APIGateway.DiscoveryChain[uid]
+			if chain == nil {
+				s.Logger.Debug("Discovery chain not found for flattened route", "discovery chain ID", uid)
+				continue
+			}
+
+			domains := generateUpstreamAPIsDomains(listenerKey, upstream, reformatedRoute.Hostnames)
+
+			virtualHost, err := s.makeUpstreamRouteForDiscoveryChain(cfgSnap, uid, chain, domains, false)
+			if err != nil {
+				return nil, err
+			}
+
+			injectHeaderManipToVirtualHostAPIGateway(&reformatedRoute, virtualHost)
+
+			defaultRoute.VirtualHosts = append(defaultRoute.VirtualHosts, virtualHost)
+		}
+
+		if len(defaultRoute.VirtualHosts) > 0 {
+			result = append(result, defaultRoute)
+		}
+	}
+
+	return result, nil
+}
+
+func buildHTTPRouteUpstream(route structs.HTTPRouteConfigEntry, listener structs.APIGatewayListener) structs.Upstream {
+	return structs.Upstream{
+		DestinationName:      route.GetName(),
+		DestinationNamespace: route.NamespaceOrDefault(),
+		DestinationPartition: route.PartitionOrDefault(),
+		IngressHosts:         route.Hostnames,
+		LocalBindPort:        listener.Port,
+		Config: map[string]interface{}{
+			"protocol": string(listener.Protocol),
+		},
+	}
+}
+
 func makeHeadersValueOptions(vals map[string]string, add bool) []*envoy_core_v3.HeaderValueOption {
 	opts := make([]*envoy_core_v3.HeaderValueOption, 0, len(vals))
 	for k, v := range vals {
@@ -516,6 +590,11 @@ func generateUpstreamIngressDomains(listenerKey proxycfg.IngressListenerKey, u s
 	return domains
 }
 
+func generateUpstreamAPIsDomains(listenerKey proxycfg.APIGatewayListenerKey, u structs.Upstream, hosts []string) []string {
+	u.IngressHosts = hosts
+	return generateUpstreamIngressDomains(listenerKey, u)
+}
+
 func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain(
 	cfgSnap *proxycfg.ConfigSnapshot,
 	uid proxycfg.UpstreamID,
@@ -1019,6 +1098,16 @@ func injectHeaderManipToRoute(dest *structs.ServiceRouteDestination, r *envoy_ro
 	return nil
 }
 
+func injectHeaderManipToVirtualHostAPIGateway(dest *structs.HTTPRouteConfigEntry, vh *envoy_route_v3.VirtualHost) {
+	for _, rule := range dest.Rules {
+		for _, header := range rule.Filters.Headers {
+			vh.RequestHeadersToAdd = append(vh.RequestHeadersToAdd, makeHeadersValueOptions(header.Add, true)...)
+			vh.RequestHeadersToAdd = append(vh.RequestHeadersToAdd, makeHeadersValueOptions(header.Set, false)...)
+			vh.RequestHeadersToRemove = append(vh.RequestHeadersToRemove, header.Remove...)
+		}
+	}
+}
+
 func injectHeaderManipToVirtualHost(dest *structs.IngressService, vh *envoy_route_v3.VirtualHost) error {
 	if !dest.RequestHeaders.IsZero() {
 		vh.RequestHeadersToAdd = append(