diff --git a/.changelog/9067.txt b/.changelog/9067.txt new file mode 100644 index 0000000000..cce35c2ba8 --- /dev/null +++ b/.changelog/9067.txt @@ -0,0 +1,3 @@ +```release-note:feature +agent: add config flag `MaxHeaderBytes` to control the maximum size of the http header per client request. +``` \ No newline at end of file diff --git a/agent/agent.go b/agent/agent.go index 952ab863b1..dc1e9c496a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -795,9 +795,10 @@ func (a *Agent) listenHTTP() ([]apiServer, error) { a.configReloaders = append(a.configReloaders, srv.ReloadConfig) a.httpHandlers = srv httpServer := &http.Server{ - Addr: l.Addr().String(), - TLSConfig: tlscfg, - Handler: srv.handler(a.config.EnableDebug), + Addr: l.Addr().String(), + TLSConfig: tlscfg, + Handler: srv.handler(a.config.EnableDebug), + MaxHeaderBytes: a.config.HTTPMaxHeaderBytes, } // Load the connlimit helper into the server diff --git a/agent/agent_test.go b/agent/agent_test.go index b1fb7e815d..1fe00626ff 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -270,6 +270,91 @@ func TestAgent_ReconnectConfigSettings(t *testing.T) { }() } +func TestAgent_HTTPMaxHeaderBytes(t *testing.T) { + tests := []struct { + name string + maxHeaderBytes int + expectedHTTPResponse int + }{ + { + "max header bytes 1 returns 431 http response when too large headers are sent", + 1, + 431, + }, + { + "max header bytes 0 returns 200 http response, as the http.DefaultMaxHeaderBytes size of 1MB is used", + 0, + 200, + }, + { + "negative maxHeaderBytes returns 200 http response, as the http.DefaultMaxHeaderBytes size of 1MB is used", + -10, + 200, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ports, err := freeport.Take(1) + require.NoError(t, err) + t.Cleanup(func() { freeport.Return(ports) }) + + caConfig := tlsutil.Config{} + tlsConf, err := tlsutil.NewConfigurator(caConfig, hclog.New(nil)) + require.NoError(t, err) + + bd := BaseDeps{ + Deps: consul.Deps{ + Logger: hclog.NewInterceptLogger(nil), + Tokens: new(token.Store), + TLSConfigurator: tlsConf, + }, + RuntimeConfig: &config.RuntimeConfig{ + HTTPAddrs: []net.Addr{ + &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: ports[0]}, + }, + HTTPMaxHeaderBytes: tt.maxHeaderBytes, + }, + Cache: cache.New(cache.Options{}), + } + a, err := New(bd) + require.NoError(t, err) + + srvs, err := a.listenHTTP() + require.NoError(t, err) + + require.Equal(t, tt.maxHeaderBytes, a.config.HTTPMaxHeaderBytes) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + t.Cleanup(cancel) + + g := new(errgroup.Group) + for _, s := range srvs { + g.Go(s.Run) + } + + require.Len(t, srvs, 1) + + client := &http.Client{} + for _, s := range srvs { + u := url.URL{Scheme: s.Protocol, Host: s.Addr.String()} + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + require.NoError(t, err) + + // This is directly pulled from the testing of request limits in the net/http source + // https://github.com/golang/go/blob/go1.15.3/src/net/http/serve_test.go#L2897-L2900 + var bytesPerHeader = len("header12345: val12345\r\n") + for i := 0; i < ((tt.maxHeaderBytes+4096)/bytesPerHeader)+1; i++ { + req.Header.Set(fmt.Sprintf("header%05d", i), fmt.Sprintf("val%05d", i)) + } + + resp, err := client.Do(req.WithContext(ctx)) + require.NoError(t, err) + require.Equal(t, tt.expectedHTTPResponse, resp.StatusCode, "expected a '%d' http response, got '%d'", tt.expectedHTTPResponse, resp.StatusCode) + } + }) + } +} + func TestAgent_ReconnectConfigWanDisabled(t *testing.T) { if testing.Short() { t.Skip("too slow for testing.Short") diff --git a/agent/config/builder.go b/agent/config/builder.go index 48a2c55bec..78ffe68388 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -920,6 +920,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { HTTPAddrs: httpAddrs, HTTPSAddrs: httpsAddrs, HTTPBlockEndpoints: c.HTTPConfig.BlockEndpoints, + HTTPMaxHeaderBytes: b.intVal(c.HTTPConfig.MaxHeaderBytes), HTTPResponseHeaders: c.HTTPConfig.ResponseHeaders, AllowWriteHTTPFrom: b.cidrsVal("allow_write_http_from", c.HTTPConfig.AllowWriteHTTPFrom), HTTPUseCache: b.boolValWithDefault(c.HTTPConfig.UseCache, true), diff --git a/agent/config/config.go b/agent/config/config.go index a6d4df8434..f8850466c7 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -614,6 +614,7 @@ type HTTPConfig struct { AllowWriteHTTPFrom []string `json:"allow_write_http_from,omitempty" hcl:"allow_write_http_from" mapstructure:"allow_write_http_from"` ResponseHeaders map[string]string `json:"response_headers,omitempty" hcl:"response_headers" mapstructure:"response_headers"` UseCache *bool `json:"use_cache,omitempty" hcl:"use_cache" mapstructure:"use_cache"` + MaxHeaderBytes *int `json:"max_header_bytes,omitempty" hcl:"max_header_bytes" mapstructure:"max_header_bytes"` } type Performance struct { diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 3cdcb0d5d6..a1e5baa0f4 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -779,6 +779,14 @@ type RuntimeConfig struct { // hcl: limits{ http_max_conns_per_client = 200 } HTTPMaxConnsPerClient int + // HTTPMaxHeaderBytes controls the maximum number of bytes the + // server will read parsing the request header's keys and + // values, including the request line. It does not limit the + // size of the request body. + // + // If zero, or negative, http.DefaultMaxHeaderBytes is used. + HTTPMaxHeaderBytes int + // HTTPSHandshakeTimeout is the time allowed for HTTPS client to complete the // TLS handshake and send first bytes of the request. // diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 1b9b65180f..a5bdc2d737 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -5270,7 +5270,8 @@ func TestFullConfig(t *testing.T) { "M6TKa9NP": "xjuxjOzQ", "JRCrHZed": "rl0mTx81" }, - "use_cache": false + "use_cache": false, + "max_header_bytes": 10 }, "key_file": "IEkkwgIA", "leave_on_terminate": true, @@ -5959,6 +5960,7 @@ func TestFullConfig(t *testing.T) { "JRCrHZed" = "rl0mTx81" } use_cache = false + max_header_bytes = 10 } key_file = "IEkkwgIA" leave_on_terminate = true @@ -6738,6 +6740,7 @@ func TestFullConfig(t *testing.T) { HTTPResponseHeaders: map[string]string{"M6TKa9NP": "xjuxjOzQ", "JRCrHZed": "rl0mTx81"}, HTTPSAddrs: []net.Addr{tcpAddr("95.17.17.19:15127")}, HTTPMaxConnsPerClient: 100, + HTTPMaxHeaderBytes: 10, HTTPSHandshakeTimeout: 2391 * time.Millisecond, HTTPSPort: 15127, HTTPUseCache: false, @@ -7665,6 +7668,7 @@ func TestSanitize(t *testing.T) { ], "HTTPBlockEndpoints": [], "HTTPMaxConnsPerClient": 0, + "HTTPMaxHeaderBytes": 0, "HTTPPort": 0, "HTTPResponseHeaders": {}, "HTTPUseCache": false, diff --git a/website/content/docs/agent/options.mdx b/website/content/docs/agent/options.mdx index 955cfbbc94..644b5bbb6b 100644 --- a/website/content/docs/agent/options.mdx +++ b/website/content/docs/agent/options.mdx @@ -1633,6 +1633,8 @@ bind_addr = "{{ GetPrivateInterfaces | include \"network\" \"10.0.0.0/8\" | attr - `use_cache` ((#http_config_use_cache)) Defaults to true. If disabled, the agent won't be using [agent caching](/api/features/caching) to answer the request. Even when the url parameter is provided. + - `max_header_bytes` This setting controls the maximum number of bytes the consul http server will read parsing the request header's keys and values, including the request line. It does not limit the size of the request body. If zero, or negative, http.DefaultMaxHeaderBytes is used, which equates to 1 Megabyte. + - `leave_on_terminate` If enabled, when the agent receives a TERM signal, it will send a `Leave` message to the rest of the cluster and gracefully leave. The default behavior for this feature varies based on whether or not the agent is running as a client or a server (prior to Consul 0.7 the default value was unconditionally set to `false`). On agents in client-mode, this defaults to `true` and for agents in server-mode, this defaults to `false`. - `limits` Available in Consul 0.9.3 and later, this is a nested