NET-5530 Support response header modifiers on http-route config entry (#18646)

* Add response header filters to http-route config entry definitions

* Map response header filters from config entry when constructing route destination

* Support response header modifiers at the service level as well

* Update protobuf definitions

* Update existing unit tests

* Add response filters to route consolidation logic

* Make existing unit tests more robust

* Add missing docstring

* Add changelog entry

* Add response filter modifiers to existing integration test

* Add more robust testing for response header modifiers in the discovery chain

* Add more robust testing for request header modifiers in the discovery chain

* Modify test to verify that service filter modifiers take precedence over rule filter modifiers

.changelog/18646.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+api-gateway: Add support for response header modifiers on http-route configuration entry
+```

agent/consul/discoverychain/gateway.go

@@ -29,6 +29,7 @@ type GatewayChainSynthesizer struct {
 type hostnameMatch struct {
 	match           structs.HTTPMatch
 	filters         structs.HTTPFilters
+	responseFilters structs.HTTPResponseFilters
 	services        []structs.HTTPService
 }
 
@@ -89,6 +90,7 @@ func initHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, curre
 				matches = append(matches, hostnameMatch{
 					match:           match,
 					filters:         rule.Filters,
+					responseFilters: rule.ResponseFilters,
 					services:        rule.Services,
 				})
 			}
@@ -228,6 +230,7 @@ func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, listene
 			route.Rules = append(route.Rules, structs.HTTPRouteRule{
 				Matches:         []structs.HTTPMatch{rule.match},
 				Filters:         rule.filters,
+				ResponseFilters: rule.responseFilters,
 				Services:        rule.services,
 			})
 		}

agent/consul/discoverychain/gateway_httproute.go

@@ -79,7 +79,8 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
 	var defaults []*structs.ServiceConfigEntry
 
 	for idx, rule := range route.Rules {
-		modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers)
+		requestModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers)
+		responseModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.ResponseFilters.Headers)
 		prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrite)
 
 		var destination structs.ServiceRouteDestination
@@ -90,16 +91,29 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
 			if service.Filters.URLRewrite == nil {
 				servicePrefixRewrite = prefixRewrite
 			}
-			serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers)
+
-			modifier.Add = mergeMaps(modifier.Add, serviceModifier.Add)
+			// Merge service request header modifier(s) onto route rule modifiers
-			modifier.Set = mergeMaps(modifier.Set, serviceModifier.Set)
+			// Note: Removals for the same header may exist on the rule + the service and
-			modifier.Remove = append(modifier.Remove, serviceModifier.Remove...)
+			//   will result in idempotent duplicate values in the modifier w/ service coming last
+			serviceRequestModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers)
+			requestModifier.Add = mergeMaps(requestModifier.Add, serviceRequestModifier.Add)
+			requestModifier.Set = mergeMaps(requestModifier.Set, serviceRequestModifier.Set)
+			requestModifier.Remove = append(requestModifier.Remove, serviceRequestModifier.Remove...)
+
+			// Merge service response header modifier(s) onto route rule modifiers
+			// Note: Removals for the same header may exist on the rule + the service and
+			//   will result in idempotent duplicate values in the modifier w/ service coming last
+			serviceResponseModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.ResponseFilters.Headers)
+			responseModifier.Add = mergeMaps(responseModifier.Add, serviceResponseModifier.Add)
+			responseModifier.Set = mergeMaps(responseModifier.Set, serviceResponseModifier.Set)
+			responseModifier.Remove = append(responseModifier.Remove, serviceResponseModifier.Remove...)
 
 			destination.Service = service.Name
 			destination.Namespace = service.NamespaceOrDefault()
 			destination.Partition = service.PartitionOrDefault()
 			destination.PrefixRewrite = servicePrefixRewrite
-			destination.RequestHeaders = modifier
+			destination.RequestHeaders = requestModifier
+			destination.ResponseHeaders = responseModifier
 
 			// since we have already validated the protocol elsewhere, we
 			// create a new service defaults here to make sure we pass validation
@@ -115,7 +129,8 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
 			destination.Namespace = route.NamespaceOrDefault()
 			destination.Partition = route.PartitionOrDefault()
 			destination.PrefixRewrite = prefixRewrite
-			destination.RequestHeaders = modifier
+			destination.RequestHeaders = requestModifier
+			destination.ResponseHeaders = responseModifier
 
 			splitter := &structs.ServiceSplitterConfigEntry{
 				Kind:           structs.ServiceSplitter,

agent/consul/discoverychain/gateway_test.go

@@ -518,8 +518,70 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
 					Kind: structs.HTTPRoute,
 					Name: "http-route",
 					Rules: []structs.HTTPRouteRule{{
+						Filters: structs.HTTPFilters{
+							Headers: []structs.HTTPHeaderFilter{
+								{
+									Add:    map[string]string{"add me to the rule request": "present"},
+									Set:    map[string]string{"set me on the rule request": "present"},
+									Remove: []string{"remove me from the rule request"},
+								},
+								{
+									Add: map[string]string{"add me to the rule and service request": "rule"},
+									Set: map[string]string{"set me on the rule and service request": "rule"},
+								},
+								{
+									Remove: []string{"remove me from the rule and service request"},
+								},
+							},
+						},
+						ResponseFilters: structs.HTTPResponseFilters{
+							Headers: []structs.HTTPHeaderFilter{{
+								Add: map[string]string{
+									"add me to the rule response":             "present",
+									"add me to the rule and service response": "rule",
+								},
+								Set: map[string]string{
+									"set me on the rule response":             "present",
+									"set me on the rule and service response": "rule",
+								},
+								Remove: []string{
+									"remove me from the rule response",
+									"remove me from the rule and service response",
+								},
+							}},
+						},
 						Services: []structs.HTTPService{{
 							Name: "foo",
+							Filters: structs.HTTPFilters{
+								Headers: []structs.HTTPHeaderFilter{
+									{
+										Add: map[string]string{"add me to the service request": "present"},
+									},
+									{
+										Set:    map[string]string{"set me on the service request": "present"},
+										Remove: []string{"remove me from the service request"},
+									},
+									{
+										Add:    map[string]string{"add me to the rule and service request": "service"},
+										Set:    map[string]string{"set me on the rule and service request": "service"},
+										Remove: []string{"remove me from the rule and service request"},
+									},
+								},
+							},
+							ResponseFilters: structs.HTTPResponseFilters{
+								Headers: []structs.HTTPHeaderFilter{
+									{
+										Add:    map[string]string{"add me to the service response": "present"},
+										Set:    map[string]string{"set me on the service response": "present"},
+										Remove: []string{"remove me from the service response"},
+									},
+									{
+										Add:    map[string]string{"add me to the rule and service response": "service"},
+										Set:    map[string]string{"set me on the rule and service response": "service"},
+										Remove: []string{"remove me from the rule and service response"},
+									},
+								},
+							},
 						}},
 					}},
 				},
@@ -557,8 +619,40 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
 									Partition: "default",
 									Namespace: "default",
 									RequestHeaders: &structs.HTTPHeaderModifiers{
-										Add: make(map[string]string),
+										Add: map[string]string{
-										Set: make(map[string]string),
+											"add me to the rule request":             "present",
+											"add me to the service request":          "present",
+											"add me to the rule and service request": "service",
+										},
+										Set: map[string]string{
+											"set me on the rule request":             "present",
+											"set me on the service request":          "present",
+											"set me on the rule and service request": "service",
+										},
+										Remove: []string{
+											"remove me from the rule request",
+											"remove me from the rule and service request",
+											"remove me from the service request",
+											"remove me from the rule and service request",
+										},
+									},
+									ResponseHeaders: &structs.HTTPHeaderModifiers{
+										Add: map[string]string{
+											"add me to the rule response":             "present",
+											"add me to the service response":          "present",
+											"add me to the rule and service response": "service",
+										},
+										Set: map[string]string{
+											"set me on the rule response":             "present",
+											"set me on the service response":          "present",
+											"set me on the rule and service response": "service",
+										},
+										Remove: []string{
+											"remove me from the rule response",
+											"remove me from the rule and service response",
+											"remove me from the service response",
+											"remove me from the rule and service response",
+										},
 									},
 								},
 							},
@@ -663,6 +757,10 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
 										Add: make(map[string]string),
 										Set: make(map[string]string),
 									},
+									ResponseHeaders: &structs.HTTPHeaderModifiers{
+										Add: make(map[string]string),
+										Set: make(map[string]string),
+									},
 								},
 							},
 							NextNode: "resolver:foo-2.default.default.dc2",
@@ -850,6 +948,10 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
 										Add: make(map[string]string),
 										Set: make(map[string]string),
 									},
+									ResponseHeaders: &structs.HTTPHeaderModifiers{
+										Add: make(map[string]string),
+										Set: make(map[string]string),
+									},
 								},
 							},
 							NextNode: "splitter:splitter-one.default.default",

agent/structs/config_entry_routes.go

@@ -457,6 +457,12 @@ type HTTPFilters struct {
 	JWT           *JWTFilter
 }
 
+// HTTPResponseFilters specifies a list of filters used to modify the
+// response returned by an upstream
+type HTTPResponseFilters struct {
+	Headers []HTTPHeaderFilter
+}
+
 // HTTPHeaderFilter specifies how HTTP headers should be modified.
 type HTTPHeaderFilter struct {
 	Add    map[string]string
@@ -486,6 +492,9 @@ type HTTPRouteRule struct {
 	// Filters is a list of HTTP-based filters used to modify a request prior
 	// to routing it to the upstream service
 	Filters HTTPFilters
+	// ResponseFilters is a list of HTTP-based filters used to modify a response
+	// returned by the upstream service
+	ResponseFilters HTTPResponseFilters
 	// Matches specified the matching criteria used in the routing table. If a
 	// request matches the given HTTPMatch configuration, then traffic is routed
 	// to services specified in the Services field.
@@ -505,6 +514,10 @@ type HTTPService struct {
 	// to routing it to the upstream service
 	Filters HTTPFilters
 
+	// ResponseFilters is a list of HTTP-based filters used to modify the
+	// response returned from the upstream service
+	ResponseFilters HTTPResponseFilters
+
 	acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
 }
 

api/config_entry_routes.go

@@ -204,6 +204,12 @@ type HTTPFilters struct {
 	JWT           *JWTFilter
 }
 
+// HTTPResponseFilters specifies a list of filters used to modify a
+// response returned by an upstream
+type HTTPResponseFilters struct {
+	Headers []HTTPHeaderFilter
+}
+
 // HTTPHeaderFilter specifies how HTTP headers should be modified.
 type HTTPHeaderFilter struct {
 	Add    map[string]string
@@ -238,6 +244,9 @@ type HTTPRouteRule struct {
 	// Filters is a list of HTTP-based filters used to modify a request prior
 	// to routing it to the upstream service
 	Filters HTTPFilters
+	// ResponseFilters is a list of HTTP-based filters used to modify a response
+	// returned by the upstream service
+	ResponseFilters HTTPResponseFilters
 	// Matches specified the matching criteria used in the routing table. If a
 	// request matches the given HTTPMatch configuration, then traffic is routed
 	// to services specified in the Services field.
@@ -253,10 +262,15 @@ type HTTPService struct {
 	// Weight is an arbitrary integer used in calculating how much
 	// traffic should be sent to the given service.
 	Weight int
+
 	// Filters is a list of HTTP-based filters used to modify a request prior
 	// to routing it to the upstream service
 	Filters HTTPFilters
 
+	// ResponseFilters is a list of HTTP-based filters used to modify the
+	// response returned from the upstream service
+	ResponseFilters HTTPResponseFilters
+
 	// Partition is the partition the config entry is associated with.
 	// Partitioning is a Consul Enterprise feature.
 	Partition string `json:",omitempty"`

proto/private/pbconfigentry/config_entry.gen.go

@@ -578,6 +578,34 @@ func HTTPQueryMatchFromStructs(t *structs.HTTPQueryMatch, s *HTTPQueryMatch) {
 	s.Name = t.Name
 	s.Value = t.Value
 }
+func HTTPResponseFiltersToStructs(s *HTTPResponseFilters, t *structs.HTTPResponseFilters) {
+	if s == nil {
+		return
+	}
+	{
+		t.Headers = make([]structs.HTTPHeaderFilter, len(s.Headers))
+		for i := range s.Headers {
+			if s.Headers[i] != nil {
+				HTTPHeaderFilterToStructs(s.Headers[i], &t.Headers[i])
+			}
+		}
+	}
+}
+func HTTPResponseFiltersFromStructs(t *structs.HTTPResponseFilters, s *HTTPResponseFilters) {
+	if s == nil {
+		return
+	}
+	{
+		s.Headers = make([]*HTTPHeaderFilter, len(t.Headers))
+		for i := range t.Headers {
+			{
+				var x HTTPHeaderFilter
+				HTTPHeaderFilterFromStructs(&t.Headers[i], &x)
+				s.Headers[i] = &x
+			}
+		}
+	}
+}
 func HTTPRouteToStructs(s *HTTPRoute, t *structs.HTTPRouteConfigEntry) {
 	if s == nil {
 		return
@@ -643,6 +671,9 @@ func HTTPRouteRuleToStructs(s *HTTPRouteRule, t *structs.HTTPRouteRule) {
 	if s.Filters != nil {
 		HTTPFiltersToStructs(s.Filters, &t.Filters)
 	}
+	if s.ResponseFilters != nil {
+		HTTPResponseFiltersToStructs(s.ResponseFilters, &t.ResponseFilters)
+	}
 	{
 		t.Matches = make([]structs.HTTPMatch, len(s.Matches))
 		for i := range s.Matches {
@@ -669,6 +700,11 @@ func HTTPRouteRuleFromStructs(t *structs.HTTPRouteRule, s *HTTPRouteRule) {
 		HTTPFiltersFromStructs(&t.Filters, &x)
 		s.Filters = &x
 	}
+	{
+		var x HTTPResponseFilters
+		HTTPResponseFiltersFromStructs(&t.ResponseFilters, &x)
+		s.ResponseFilters = &x
+	}
 	{
 		s.Matches = make([]*HTTPMatch, len(t.Matches))
 		for i := range t.Matches {
@@ -699,6 +735,9 @@ func HTTPServiceToStructs(s *HTTPService, t *structs.HTTPService) {
 	if s.Filters != nil {
 		HTTPFiltersToStructs(s.Filters, &t.Filters)
 	}
+	if s.ResponseFilters != nil {
+		HTTPResponseFiltersToStructs(s.ResponseFilters, &t.ResponseFilters)
+	}
 	t.EnterpriseMeta = enterpriseMetaToStructs(s.EnterpriseMeta)
 }
 func HTTPServiceFromStructs(t *structs.HTTPService, s *HTTPService) {
@@ -712,6 +751,11 @@ func HTTPServiceFromStructs(t *structs.HTTPService, s *HTTPService) {
 		HTTPFiltersFromStructs(&t.Filters, &x)
 		s.Filters = &x
 	}
+	{
+		var x HTTPResponseFilters
+		HTTPResponseFiltersFromStructs(&t.ResponseFilters, &x)
+		s.ResponseFilters = &x
+	}
 	s.EnterpriseMeta = enterpriseMetaFromStructs(t.EnterpriseMeta)
 }
 func HashPolicyToStructs(s *HashPolicy, t *structs.HashPolicy) {

proto/private/pbconfigentry/config_entry.pb.binary.go

@@ -687,6 +687,16 @@ func (msg *HTTPFilters) UnmarshalBinary(b []byte) error {
 	return proto.Unmarshal(b, msg)
 }
 
+// MarshalBinary implements encoding.BinaryMarshaler
+func (msg *HTTPResponseFilters) MarshalBinary() ([]byte, error) {
+	return proto.Marshal(msg)
+}
+
+// UnmarshalBinary implements encoding.BinaryUnmarshaler
+func (msg *HTTPResponseFilters) UnmarshalBinary(b []byte) error {
+	return proto.Unmarshal(b, msg)
+}
+
 // MarshalBinary implements encoding.BinaryMarshaler
 func (msg *URLRewrite) MarshalBinary() ([]byte, error) {
 	return proto.Marshal(msg)

proto/private/pbconfigentry/config_entry.proto

@@ -857,6 +857,7 @@ message HTTPRouteRule {
   HTTPFilters Filters = 1;
   repeated HTTPMatch Matches = 2;
   repeated HTTPService Services = 3;
+  HTTPResponseFilters ResponseFilters = 4;
 }
 
 // mog annotation:
@@ -954,6 +955,15 @@ message HTTPFilters {
   JWTFilter JWT = 5;
 }
 
+// mog annotation:
+//
+// target=github.com/hashicorp/consul/agent/structs.HTTPResponseFilters
+// output=config_entry.gen.go
+// name=Structs
+message HTTPResponseFilters {
+  repeated HTTPHeaderFilter Headers = 1;
+}
+
 // mog annotation:
 //
 // target=github.com/hashicorp/consul/agent/structs.URLRewrite
@@ -1014,6 +1024,7 @@ message HTTPService {
   HTTPFilters Filters = 3;
   // mog: func-to=enterpriseMetaToStructs func-from=enterpriseMetaFromStructs
   common.EnterpriseMeta EnterpriseMeta = 4;
+  HTTPResponseFilters ResponseFilters = 5;
 }
 
 // mog annotation:

test/integration/consul-container/test/gateways/gateway_endpoint_test.go

@@ -224,6 +224,7 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string
 
 type checkOptions struct {
 	debug           bool
+	responseHeaders map[string]string
 	statusCode      int
 	testName        string
 }
@@ -274,6 +275,14 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string,
 			t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode)
 			return false
 		}
+
+		for name, value := range expected.responseHeaders {
+			if res.Header.Get(name) != value {
+				t.Logf("response missing header - expected: %s=%s, actual: %s=%s", name, value, name, res.Header.Get(name))
+				return false
+			}
+		}
+
 		if expected.debug {
 			if !strings.Contains(string(body), "debug") {
 				t.Log("body does not contain 'debug'")

test/integration/consul-container/test/gateways/http_route_test.go

@@ -268,7 +268,7 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"})
 }
 
-func TestHTTPRoutePathRewrite(t *testing.T) {
+func TestHTTPRouteFilters(t *testing.T) {
 	if testing.Short() {
 		t.Skip("too slow for testing.Short")
 	}
@@ -393,6 +393,12 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 						Path: fooPath,
 					},
 				},
+				ResponseFilters: api.HTTPResponseFilters{
+					Headers: []api.HTTPHeaderFilter{{
+						Add: map[string]string{"response-filters-add": "present"},
+						Set: map[string]string{"response-filters-set": "present"},
+					}},
+				},
 				Services: []api.HTTPService{
 					{
 						Name:      fooName,
@@ -478,13 +484,23 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 	debugExpectedStatusCode := 200
 
 	// hit foo, making sure path is being rewritten by hitting the debug page
-	checkRoute(t, gatewayPort, fooUnrewritten, map[string]string{
+	// and that we get the expected response headers that we added modifiers for
-		"Host": "test.foo",
+	checkRoute(
-	}, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "foo service"})
+		t, gatewayPort, fooUnrewritten, map[string]string{"Host": "test.foo"},
+		checkOptions{
+			debug:           true,
+			responseHeaders: map[string]string{"response-filters-add": "present", "response-filters-set": "present"},
+			statusCode:      debugExpectedStatusCode,
+			testName:        "foo service"})
 	// make sure foo is being sent to proper service
-	checkRoute(t, gatewayPort, fooUnrewritten+"/foo", map[string]string{
+	// and that we get the expected response headers that we added modifiers for
-		"Host": "test.foo",
+	checkRoute(
-	}, checkOptions{debug: false, statusCode: fooStatusCode, testName: "foo service 2"})
+		t, gatewayPort, fooUnrewritten+"/foo", map[string]string{"Host": "test.foo"},
+		checkOptions{
+			debug:           false,
+			responseHeaders: map[string]string{"response-filters-add": "present", "response-filters-set": "present"},
+			statusCode:      fooStatusCode,
+			testName:        "foo service 2"})
 
 	// hit bar, making sure its been rewritten
 	checkRoute(t, gatewayPort, barUnrewritten, map[string]string{