status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

NET-5530 Support response header modifiers on http-route config entry (#18646) 

* Add response header filters to http-route config entry definitions

* Map response header filters from config entry when constructing route destination

* Support response header modifiers at the service level as well

* Update protobuf definitions

* Update existing unit tests

* Add response filters to route consolidation logic

* Make existing unit tests more robust

* Add missing docstring

* Add changelog entry

* Add response filter modifiers to existing integration test

* Add more robust testing for response header modifiers in the discovery chain

* Add more robust testing for request header modifiers in the discovery chain

* Modify test to verify that service filter modifiers take precedence over rule filter modifiers
This commit is contained in:
Nathan Coleman 2023-09-08 14:04:56 -04:00 committed by GitHub
parent 235988b3bc
commit e5d26a13cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1026 additions and 683 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/18646.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:feature
api-gateway: Add support for response header modifiers on http-route configuration entry
```

21
agent/consul/discoverychain/gateway.go
View File

@ -27,9 +27,10 @@ type GatewayChainSynthesizer struct {
}
type hostnameMatch struct {
match structs.HTTPMatch
filters structs.HTTPFilters
services []structs.HTTPService
match structs.HTTPMatch
filters structs.HTTPFilters
responseFilters structs.HTTPResponseFilters
services []structs.HTTPService
}
// NewGatewayChainSynthesizer creates a new GatewayChainSynthesizer for the
@ -87,9 +88,10 @@ func initHostMatches(hostname string, route *structs.HTTPRouteConfigEntry, curre
// Add all matches for this rule to the list for this hostname
for _, match := range rule.Matches {
matches = append(matches, hostnameMatch{
match: match,
filters: rule.Filters,
services: rule.Services,
match: match,
filters: rule.Filters,
responseFilters: rule.ResponseFilters,
services: rule.Services,
})
}
}
@ -226,9 +228,10 @@ func consolidateHTTPRoutes(matchesByHostname map[string][]hostnameMatch, listene
// Add all rules for this hostname
for _, rule := range rules {
route.Rules = append(route.Rules, structs.HTTPRouteRule{
Matches: []structs.HTTPMatch{rule.match},
Filters: rule.filters,
Services: rule.services,
Matches: []structs.HTTPMatch{rule.match},
Filters: rule.filters,
ResponseFilters: rule.responseFilters,
Services: rule.services,
})
}

29
agent/consul/discoverychain/gateway_httproute.go
View File

@ -79,7 +79,8 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
var defaults []*structs.ServiceConfigEntry
for idx, rule := range route.Rules {
modifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers)
requestModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.Filters.Headers)
responseModifier := httpRouteFiltersToServiceRouteHeaderModifier(rule.ResponseFilters.Headers)
prefixRewrite := httpRouteFiltersToDestinationPrefixRewrite(rule.Filters.URLRewrite)
var destination structs.ServiceRouteDestination
@ -90,16 +91,29 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
if service.Filters.URLRewrite == nil {
servicePrefixRewrite = prefixRewrite
}
serviceModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers)
modifier.Add = mergeMaps(modifier.Add, serviceModifier.Add)
modifier.Set = mergeMaps(modifier.Set, serviceModifier.Set)
modifier.Remove = append(modifier.Remove, serviceModifier.Remove...)
// Merge service request header modifier(s) onto route rule modifiers
// Note: Removals for the same header may exist on the rule + the service and
// will result in idempotent duplicate values in the modifier w/ service coming last
serviceRequestModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.Filters.Headers)
requestModifier.Add = mergeMaps(requestModifier.Add, serviceRequestModifier.Add)
requestModifier.Set = mergeMaps(requestModifier.Set, serviceRequestModifier.Set)
requestModifier.Remove = append(requestModifier.Remove, serviceRequestModifier.Remove...)
// Merge service response header modifier(s) onto route rule modifiers
// Note: Removals for the same header may exist on the rule + the service and
// will result in idempotent duplicate values in the modifier w/ service coming last
serviceResponseModifier := httpRouteFiltersToServiceRouteHeaderModifier(service.ResponseFilters.Headers)
responseModifier.Add = mergeMaps(responseModifier.Add, serviceResponseModifier.Add)
responseModifier.Set = mergeMaps(responseModifier.Set, serviceResponseModifier.Set)
responseModifier.Remove = append(responseModifier.Remove, serviceResponseModifier.Remove...)
destination.Service = service.Name
destination.Namespace = service.NamespaceOrDefault()
destination.Partition = service.PartitionOrDefault()
destination.PrefixRewrite = servicePrefixRewrite
destination.RequestHeaders = modifier
destination.RequestHeaders = requestModifier
destination.ResponseHeaders = responseModifier
// since we have already validated the protocol elsewhere, we
// create a new service defaults here to make sure we pass validation
@ -115,7 +129,8 @@ func httpRouteToDiscoveryChain(route structs.HTTPRouteConfigEntry) (*structs.Ser
destination.Namespace = route.NamespaceOrDefault()
destination.Partition = route.PartitionOrDefault()
destination.PrefixRewrite = prefixRewrite
destination.RequestHeaders = modifier
destination.RequestHeaders = requestModifier
destination.ResponseHeaders = responseModifier
splitter := &structs.ServiceSplitterConfigEntry{
Kind: structs.ServiceSplitter,

106
agent/consul/discoverychain/gateway_test.go
View File

@ -518,8 +518,70 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
Kind: structs.HTTPRoute,
Name: "http-route",
Rules: []structs.HTTPRouteRule{{
Filters: structs.HTTPFilters{
Headers: []structs.HTTPHeaderFilter{
{
Add: map[string]string{"add me to the rule request": "present"},
Set: map[string]string{"set me on the rule request": "present"},
Remove: []string{"remove me from the rule request"},
},
{
Add: map[string]string{"add me to the rule and service request": "rule"},
Set: map[string]string{"set me on the rule and service request": "rule"},
},
{
Remove: []string{"remove me from the rule and service request"},
},
},
},
ResponseFilters: structs.HTTPResponseFilters{
Headers: []structs.HTTPHeaderFilter{{
Add: map[string]string{
"add me to the rule response": "present",
"add me to the rule and service response": "rule",
},
Set: map[string]string{
"set me on the rule response": "present",
"set me on the rule and service response": "rule",
},
Remove: []string{
"remove me from the rule response",
"remove me from the rule and service response",
},
}},
},
Services: []structs.HTTPService{{
Name: "foo",
Filters: structs.HTTPFilters{
Headers: []structs.HTTPHeaderFilter{
{
Add: map[string]string{"add me to the service request": "present"},
},
{
Set: map[string]string{"set me on the service request": "present"},
Remove: []string{"remove me from the service request"},
},
{
Add: map[string]string{"add me to the rule and service request": "service"},
Set: map[string]string{"set me on the rule and service request": "service"},
Remove: []string{"remove me from the rule and service request"},
},
},
},
ResponseFilters: structs.HTTPResponseFilters{
Headers: []structs.HTTPHeaderFilter{
{
Add: map[string]string{"add me to the service response": "present"},
Set: map[string]string{"set me on the service response": "present"},
Remove: []string{"remove me from the service response"},
},
{
Add: map[string]string{"add me to the rule and service response": "service"},
Set: map[string]string{"set me on the rule and service response": "service"},
Remove: []string{"remove me from the rule and service response"},
},
},
},
}},
}},
},
@ -557,8 +619,40 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
Partition: "default",
Namespace: "default",
RequestHeaders: &structs.HTTPHeaderModifiers{
Add: make(map[string]string),
Set: make(map[string]string),
Add: map[string]string{
"add me to the rule request": "present",
"add me to the service request": "present",
"add me to the rule and service request": "service",
},
Set: map[string]string{
"set me on the rule request": "present",
"set me on the service request": "present",
"set me on the rule and service request": "service",
},
Remove: []string{
"remove me from the rule request",
"remove me from the rule and service request",
"remove me from the service request",
"remove me from the rule and service request",
},
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Add: map[string]string{
"add me to the rule response": "present",
"add me to the service response": "present",
"add me to the rule and service response": "service",
},
Set: map[string]string{
"set me on the rule response": "present",
"set me on the service response": "present",
"set me on the rule and service response": "service",
},
Remove: []string{
"remove me from the rule response",
"remove me from the rule and service response",
"remove me from the service response",
"remove me from the rule and service response",
},
},
},
},
@ -663,6 +757,10 @@ func TestGatewayChainSynthesizer_Synthesize(t *testing.T) {
Add: make(map[string]string),
Set: make(map[string]string),
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Add: make(map[string]string),
Set: make(map[string]string),
},
},
},
NextNode: "resolver:foo-2.default.default.dc2",
@ -850,6 +948,10 @@ func TestGatewayChainSynthesizer_ComplexChain(t *testing.T) {
Add: make(map[string]string),
Set: make(map[string]string),
},
ResponseHeaders: &structs.HTTPHeaderModifiers{
Add: make(map[string]string),
Set: make(map[string]string),
},
},
},
NextNode: "splitter:splitter-one.default.default",

13
agent/structs/config_entry_routes.go
View File

@ -457,6 +457,12 @@ type HTTPFilters struct {
JWT *JWTFilter
}
// HTTPResponseFilters specifies a list of filters used to modify the
// response returned by an upstream
type HTTPResponseFilters struct {
Headers []HTTPHeaderFilter
}
// HTTPHeaderFilter specifies how HTTP headers should be modified.
type HTTPHeaderFilter struct {
Add map[string]string
@ -486,6 +492,9 @@ type HTTPRouteRule struct {
// Filters is a list of HTTP-based filters used to modify a request prior
// to routing it to the upstream service
Filters HTTPFilters
// ResponseFilters is a list of HTTP-based filters used to modify a response
// returned by the upstream service
ResponseFilters HTTPResponseFilters
// Matches specified the matching criteria used in the routing table. If a
// request matches the given HTTPMatch configuration, then traffic is routed
// to services specified in the Services field.
@ -505,6 +514,10 @@ type HTTPService struct {
// to routing it to the upstream service
Filters HTTPFilters
// ResponseFilters is a list of HTTP-based filters used to modify the
// response returned from the upstream service
ResponseFilters HTTPResponseFilters
acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"`
}

14
api/config_entry_routes.go
View File

@ -204,6 +204,12 @@ type HTTPFilters struct {
JWT *JWTFilter
}
// HTTPResponseFilters specifies a list of filters used to modify a
// response returned by an upstream
type HTTPResponseFilters struct {
Headers []HTTPHeaderFilter
}
// HTTPHeaderFilter specifies how HTTP headers should be modified.
type HTTPHeaderFilter struct {
Add map[string]string
@ -238,6 +244,9 @@ type HTTPRouteRule struct {
// Filters is a list of HTTP-based filters used to modify a request prior
// to routing it to the upstream service
Filters HTTPFilters
// ResponseFilters is a list of HTTP-based filters used to modify a response
// returned by the upstream service
ResponseFilters HTTPResponseFilters
// Matches specified the matching criteria used in the routing table. If a
// request matches the given HTTPMatch configuration, then traffic is routed
// to services specified in the Services field.
@ -253,10 +262,15 @@ type HTTPService struct {
// Weight is an arbitrary integer used in calculating how much
// traffic should be sent to the given service.
Weight int
// Filters is a list of HTTP-based filters used to modify a request prior
// to routing it to the upstream service
Filters HTTPFilters
// ResponseFilters is a list of HTTP-based filters used to modify the
// response returned from the upstream service
ResponseFilters HTTPResponseFilters
// Partition is the partition the config entry is associated with.
// Partitioning is a Consul Enterprise feature.
Partition string `json:",omitempty"`

44
proto/private/pbconfigentry/config_entry.gen.go
View File

@ -578,6 +578,34 @@ func HTTPQueryMatchFromStructs(t *structs.HTTPQueryMatch, s *HTTPQueryMatch) {
s.Name = t.Name
s.Value = t.Value
}
func HTTPResponseFiltersToStructs(s *HTTPResponseFilters, t *structs.HTTPResponseFilters) {
if s == nil {
return
}
{
t.Headers = make([]structs.HTTPHeaderFilter, len(s.Headers))
for i := range s.Headers {
if s.Headers[i] != nil {
HTTPHeaderFilterToStructs(s.Headers[i], &t.Headers[i])
}
}
}
}
func HTTPResponseFiltersFromStructs(t *structs.HTTPResponseFilters, s *HTTPResponseFilters) {
if s == nil {
return
}
{
s.Headers = make([]*HTTPHeaderFilter, len(t.Headers))
for i := range t.Headers {
{
var x HTTPHeaderFilter
HTTPHeaderFilterFromStructs(&t.Headers[i], &x)
s.Headers[i] = &x
}
}
}
}
func HTTPRouteToStructs(s *HTTPRoute, t *structs.HTTPRouteConfigEntry) {
if s == nil {
return
@ -643,6 +671,9 @@ func HTTPRouteRuleToStructs(s *HTTPRouteRule, t *structs.HTTPRouteRule) {
if s.Filters != nil {
HTTPFiltersToStructs(s.Filters, &t.Filters)
}
if s.ResponseFilters != nil {
HTTPResponseFiltersToStructs(s.ResponseFilters, &t.ResponseFilters)
}
{
t.Matches = make([]structs.HTTPMatch, len(s.Matches))
for i := range s.Matches {
@ -669,6 +700,11 @@ func HTTPRouteRuleFromStructs(t *structs.HTTPRouteRule, s *HTTPRouteRule) {
HTTPFiltersFromStructs(&t.Filters, &x)
s.Filters = &x
}
{
var x HTTPResponseFilters
HTTPResponseFiltersFromStructs(&t.ResponseFilters, &x)
s.ResponseFilters = &x
}
{
s.Matches = make([]*HTTPMatch, len(t.Matches))
for i := range t.Matches {
@ -699,6 +735,9 @@ func HTTPServiceToStructs(s *HTTPService, t *structs.HTTPService) {
if s.Filters != nil {
HTTPFiltersToStructs(s.Filters, &t.Filters)
}
if s.ResponseFilters != nil {
HTTPResponseFiltersToStructs(s.ResponseFilters, &t.ResponseFilters)
}
t.EnterpriseMeta = enterpriseMetaToStructs(s.EnterpriseMeta)
}
func HTTPServiceFromStructs(t *structs.HTTPService, s *HTTPService) {
@ -712,6 +751,11 @@ func HTTPServiceFromStructs(t *structs.HTTPService, s *HTTPService) {
HTTPFiltersFromStructs(&t.Filters, &x)
s.Filters = &x
}
{
var x HTTPResponseFilters
HTTPResponseFiltersFromStructs(&t.ResponseFilters, &x)
s.ResponseFilters = &x
}
s.EnterpriseMeta = enterpriseMetaFromStructs(t.EnterpriseMeta)
}
func HashPolicyToStructs(s *HashPolicy, t *structs.HashPolicy) {

10
proto/private/pbconfigentry/config_entry.pb.binary.go
View File

@ -687,6 +687,16 @@ func (msg *HTTPFilters) UnmarshalBinary(b []byte) error {
return proto.Unmarshal(b, msg)
}
// MarshalBinary implements encoding.BinaryMarshaler
func (msg *HTTPResponseFilters) MarshalBinary() ([]byte, error) {
return proto.Marshal(msg)
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler
func (msg *HTTPResponseFilters) UnmarshalBinary(b []byte) error {
return proto.Unmarshal(b, msg)
}
// MarshalBinary implements encoding.BinaryMarshaler
func (msg *URLRewrite) MarshalBinary() ([]byte, error) {
return proto.Marshal(msg)

1413
proto/private/pbconfigentry/config_entry.pb.go
View File

File diff suppressed because it is too large Load Diff

11
proto/private/pbconfigentry/config_entry.proto
View File

@ -857,6 +857,7 @@ message HTTPRouteRule {
HTTPFilters Filters = 1;
repeated HTTPMatch Matches = 2;
repeated HTTPService Services = 3;
HTTPResponseFilters ResponseFilters = 4;
}
// mog annotation:
@ -954,6 +955,15 @@ message HTTPFilters {
JWTFilter JWT = 5;
}
// mog annotation:
//
// target=github.com/hashicorp/consul/agent/structs.HTTPResponseFilters
// output=config_entry.gen.go
// name=Structs
message HTTPResponseFilters {
repeated HTTPHeaderFilter Headers = 1;
}
// mog annotation:
//
// target=github.com/hashicorp/consul/agent/structs.URLRewrite
@ -1014,6 +1024,7 @@ message HTTPService {
HTTPFilters Filters = 3;
// mog: func-to=enterpriseMetaToStructs func-from=enterpriseMetaFromStructs
common.EnterpriseMeta EnterpriseMeta = 4;
HTTPResponseFilters ResponseFilters = 5;
}
// mog annotation:

15
test/integration/consul-container/test/gateways/gateway_endpoint_test.go
View File

@ -223,9 +223,10 @@ func checkTCPRouteConfigEntry(t *testing.T, client *api.Client, routeName string
}
type checkOptions struct {
debug bool
statusCode int
testName string
debug bool
responseHeaders map[string]string
statusCode int
testName string
}
// checkRoute, customized version of libassert.RouteEchos to allow for headers/distinguishing between the server instances
@ -274,6 +275,14 @@ func checkRoute(t *testing.T, port int, path string, headers map[string]string,
t.Logf("bad status code - expected: %d, actual: %d", expected.statusCode, res.StatusCode)
return false
}
for name, value := range expected.responseHeaders {
if res.Header.Get(name) != value {
t.Logf("response missing header - expected: %s=%s, actual: %s=%s", name, value, name, res.Header.Get(name))
return false
}
}
if expected.debug {
if !strings.Contains(string(body), "debug") {
t.Log("body does not contain 'debug'")

30
test/integration/consul-container/test/gateways/http_route_test.go
View File

@ -268,7 +268,7 @@ func TestHTTPRouteFlattening(t *testing.T) {
}, checkOptions{debug: false, statusCode: serviceOneResponseCode, testName: "service1, v2 path with v2 hostname"})
}
func TestHTTPRoutePathRewrite(t *testing.T) {
func TestHTTPRouteFilters(t *testing.T) {
if testing.Short() {
t.Skip("too slow for testing.Short")
}
@ -393,6 +393,12 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
Path: fooPath,
},
},
ResponseFilters: api.HTTPResponseFilters{
Headers: []api.HTTPHeaderFilter{{
Add: map[string]string{"response-filters-add": "present"},
Set: map[string]string{"response-filters-set": "present"},
}},
},
Services: []api.HTTPService{
{
Name: fooName,
@ -478,13 +484,23 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
debugExpectedStatusCode := 200
// hit foo, making sure path is being rewritten by hitting the debug page
checkRoute(t, gatewayPort, fooUnrewritten, map[string]string{
"Host": "test.foo",
}, checkOptions{debug: true, statusCode: debugExpectedStatusCode, testName: "foo service"})
// and that we get the expected response headers that we added modifiers for
checkRoute(
t, gatewayPort, fooUnrewritten, map[string]string{"Host": "test.foo"},
checkOptions{
debug: true,
responseHeaders: map[string]string{"response-filters-add": "present", "response-filters-set": "present"},
statusCode: debugExpectedStatusCode,
testName: "foo service"})
// make sure foo is being sent to proper service
checkRoute(t, gatewayPort, fooUnrewritten+"/foo", map[string]string{
"Host": "test.foo",
}, checkOptions{debug: false, statusCode: fooStatusCode, testName: "foo service 2"})
// and that we get the expected response headers that we added modifiers for
checkRoute(
t, gatewayPort, fooUnrewritten+"/foo", map[string]string{"Host": "test.foo"},
checkOptions{
debug: false,
responseHeaders: map[string]string{"response-filters-add": "present", "response-filters-set": "present"},
statusCode: fooStatusCode,
testName: "foo service 2"})
// hit bar, making sure its been rewritten
checkRoute(t, gatewayPort, barUnrewritten, map[string]string{