agent: configurable MaxQueryTime and DefaultQueryTime. (#3777)

agent/agent.go

@@ -1263,6 +1263,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	base.TLSMinVersion = a.config.TLSMinVersion
 	base.TLSCipherSuites = a.config.TLSCipherSuites
 	base.TLSPreferServerCipherSuites = a.config.TLSPreferServerCipherSuites
+	base.DefaultQueryTime = a.config.DefaultQueryTime
+	base.MaxQueryTime = a.config.MaxQueryTime
 
 	base.AutoEncryptAllowTLS = a.config.AutoEncryptAllowTLS
 

agent/config/builder.go

@@ -819,6 +819,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		ExposeMaxPort:                    exposeMaxPort,
 		DataDir:                          b.stringVal(c.DataDir),
 		Datacenter:                       datacenter,
+		DefaultQueryTime:                 b.durationVal("default_query_time", c.DefaultQueryTime),
 		DevMode:                          b.boolVal(b.Flags.DevMode),
 		DisableAnonymousSignature:        b.boolVal(c.DisableAnonymousSignature),
 		DisableCoordinates:               b.boolVal(c.DisableCoordinates),
@@ -850,6 +851,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		LogRotateBytes:                   b.intVal(c.LogRotateBytes),
 		LogRotateDuration:                b.durationVal("log_rotate_duration", c.LogRotateDuration),
 		LogRotateMaxFiles:                b.intVal(c.LogRotateMaxFiles),
+		MaxQueryTime:                     b.durationVal("max_query_time", c.MaxQueryTime),
 		NodeID:                           types.NodeID(b.stringVal(c.NodeID)),
 		NodeMeta:                         c.NodeMeta,
 		NodeName:                         b.nodeName(c.NodeName),

agent/config/config.go

@@ -204,6 +204,7 @@ type Config struct {
 	DNSRecursors                     []string                 `json:"recursors,omitempty" hcl:"recursors" mapstructure:"recursors"`
 	DataDir                          *string                  `json:"data_dir,omitempty" hcl:"data_dir" mapstructure:"data_dir"`
 	Datacenter                       *string                  `json:"datacenter,omitempty" hcl:"datacenter" mapstructure:"datacenter"`
+	DefaultQueryTime                 *string                  `json:"default_query_time,omitempty" hcl:"default_query_time" mapstructure:"default_query_time"`
 	DisableAnonymousSignature        *bool                    `json:"disable_anonymous_signature,omitempty" hcl:"disable_anonymous_signature" mapstructure:"disable_anonymous_signature"`
 	DisableCoordinates               *bool                    `json:"disable_coordinates,omitempty" hcl:"disable_coordinates" mapstructure:"disable_coordinates"`
 	DisableHostNodeID                *bool                    `json:"disable_host_node_id,omitempty" hcl:"disable_host_node_id" mapstructure:"disable_host_node_id"`
@@ -234,6 +235,7 @@ type Config struct {
 	LogRotateDuration                *string                  `json:"log_rotate_duration,omitempty" hcl:"log_rotate_duration" mapstructure:"log_rotate_duration"`
 	LogRotateBytes                   *int                     `json:"log_rotate_bytes,omitempty" hcl:"log_rotate_bytes" mapstructure:"log_rotate_bytes"`
 	LogRotateMaxFiles                *int                     `json:"log_rotate_max_files,omitempty" hcl:"log_rotate_max_files" mapstructure:"log_rotate_max_files"`
+	MaxQueryTime                     *string                  `json:"max_query_time,omitempty" hcl:"max_query_time" mapstructure:"max_query_time"`
 	NodeID                           *string                  `json:"node_id,omitempty" hcl:"node_id" mapstructure:"node_id"`
 	NodeMeta                         map[string]string        `json:"node_meta,omitempty" hcl:"node_meta" mapstructure:"node_meta"`
 	NodeName                         *string                  `json:"node_name,omitempty" hcl:"node_name" mapstructure:"node_name"`

agent/config/default.go

@@ -55,6 +55,7 @@ func DefaultSource() Source {
 		check_update_interval = "5m"
 		client_addr = "127.0.0.1"
 		datacenter = "` + consul.DefaultDC + `"
+		default_query_time = "300s"
 		disable_coordinates = false
 		disable_host_node_id = true
 		disable_remote_exec = true
@@ -62,6 +63,7 @@ func DefaultSource() Source {
 		encrypt_verify_incoming = true
 		encrypt_verify_outgoing = true
 		log_level = "INFO"
+		max_query_time = "600s"
 		protocol =  2
 		retry_interval = "30s"
 		retry_interval_wan = "30s"

agent/config/flags.go

@@ -66,6 +66,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) {
 	add(&f.ConfigFormat, "config-format", "Config files are in this format irrespective of their extension. Must be 'hcl' or 'json'")
 	add(&f.Config.DataDir, "data-dir", "Path to a data directory to store agent state.")
 	add(&f.Config.Datacenter, "datacenter", "Datacenter of the agent.")
+	add(&f.Config.DefaultQueryTime, "default-query-time", "the amount of time a blocking query will wait before Consul will force a response. This value can be overridden by the 'wait' query parameter.")
 	add(&f.DevMode, "dev", "Starts the agent in development mode.")
 	add(&f.Config.DisableHostNodeID, "disable-host-node-id", "Setting this to true will prevent Consul from using information from the host to generate a node ID, and will cause Consul to generate a random node ID instead.")
 	add(&f.Config.DisableKeyringFile, "disable-keyring-file", "Disables the backing up of the keyring to a file.")
@@ -85,6 +86,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) {
 	add(&f.Config.LogRotateBytes, "log-rotate-bytes", "Maximum number of bytes that should be written to a log file")
 	add(&f.Config.LogRotateDuration, "log-rotate-duration", "Time after which log rotation needs to be performed")
 	add(&f.Config.LogRotateMaxFiles, "log-rotate-max-files", "Maximum number of log file archives to keep")
+	add(&f.Config.MaxQueryTime, "max-query-time", "the maximum amount of time a blocking query can wait before Consul will force a response. Consul applies jitter to the wait time. The jittered time will be capped to MaxQueryTime.")
 	add(&f.Config.NodeName, "node", "Name of this node. Must be unique in the cluster.")
 	add(&f.Config.NodeID, "node-id", "A unique ID for this node across space and time. Defaults to a randomly-generated ID that persists in the data-dir.")
 	add(&f.Config.NodeMeta, "node-meta", "An arbitrary metadata key/value pair for this node, of the format `key:value`. Can be specified multiple times.")

agent/config/runtime.go

@@ -600,6 +600,14 @@ type RuntimeConfig struct {
 	// flag: -data-dir string
 	DataDir string
 
+	// DefaultQueryTime is the amount of time a blocking query will wait before
+	// Consul will force a response. This value can be overridden by the 'wait'
+	// query parameter.
+	//
+	// hcl: default_query_time = "duration"
+	// flag: -default-query-time string
+	DefaultQueryTime time.Duration
+
 	// DevMode enables a fast-path mode of operation to bring up an in-memory
 	// server with minimal configuration. Useful for developing Consul.
 	//
@@ -846,6 +854,14 @@ type RuntimeConfig struct {
 	// flags: -log-rotate-max-files int
 	LogRotateMaxFiles int
 
+	// MaxQueryTime is the maximum amount of time a blocking query can wait
+	// before Consul will force a response. Consul applies jitter to the wait
+	// time. The jittered time will be capped to MaxQueryTime.
+	//
+	// hcl: max_query_time = "duration"
+	// flags: -max-query-time string
+	MaxQueryTime time.Duration
+
 	// Node ID is a unique ID for this node across space and time. Defaults
 	// to a randomly-generated ID that persists in the data-dir.
 	//

agent/config/runtime_test.go

@@ -3757,6 +3757,7 @@ func TestFullConfig(t *testing.T) {
 			},
 			"data_dir": "` + dataDir + `",
 			"datacenter": "rzo029wg",
+			"default_query_time": "16743s",
 			"disable_anonymous_signature": true,
 			"disable_coordinates": true,
 			"disable_host_node_id": true,
@@ -3810,6 +3811,7 @@ func TestFullConfig(t *testing.T) {
 				"kv_max_value_size": 1234567800000000
 			},
 			"log_level": "k1zo9Spt",
+			"max_query_time": "18237s",
 			"node_id": "AsUIlw99",
 			"node_meta": {
 				"5mgGQMBk": "mJLtVMSG",
@@ -4356,6 +4358,7 @@ func TestFullConfig(t *testing.T) {
 			}
 			data_dir = "` + dataDir + `"
 			datacenter = "rzo029wg"
+			default_query_time = "16743s"
 			disable_anonymous_signature = true
 			disable_coordinates = true
 			disable_host_node_id = true
@@ -4410,6 +4413,7 @@ func TestFullConfig(t *testing.T) {
 				kv_max_value_size = 1234567800000000
 			}
 			log_level = "k1zo9Spt"
+			max_query_time = "18237s"
 			node_id = "AsUIlw99"
 			node_meta {
 				"5mgGQMBk" = "mJLtVMSG"
@@ -5064,6 +5068,7 @@ func TestFullConfig(t *testing.T) {
 		DNSCacheMaxAge:                   5 * time.Minute,
 		DataDir:                          dataDir,
 		Datacenter:                       "rzo029wg",
+		DefaultQueryTime:                 16743 * time.Second,
 		DevMode:                          true,
 		DisableAnonymousSignature:        true,
 		DisableCoordinates:               true,
@@ -5098,6 +5103,7 @@ func TestFullConfig(t *testing.T) {
 		LeaveDrainTime:                   8265 * time.Second,
 		LeaveOnTerm:                      true,
 		LogLevel:                         "k1zo9Spt",
+		MaxQueryTime:                     18237 * time.Second,
 		NodeID:                           types.NodeID("AsUIlw99"),
 		NodeMeta:                         map[string]string{"5mgGQMBk": "mJLtVMSG", "A7ynFMJB": "0Nx6RGab"},
 		NodeName:                         "otlLxGaI",
@@ -5924,6 +5930,7 @@ func TestSanitize(t *testing.T) {
 		"DNSCacheMaxAge": "0s",
 		"DataDir": "",
 		"Datacenter": "",
+		"DefaultQueryTime": "0s",
 		"DevMode": false,
 		"DisableAnonymousSignature": false,
 		"DisableCoordinates": false,
@@ -5967,6 +5974,7 @@ func TestSanitize(t *testing.T) {
 		"LogRotateBytes": 0,
 		"LogRotateDuration": "0s",
 		"LogRotateMaxFiles": 0,
+		"MaxQueryTime": "0s",
 		"NodeID": "",
 		"NodeMeta": {},
 		"NodeName": "",

agent/consul/config.go

@@ -86,6 +86,16 @@ type Config struct {
 	// DataDir is the directory to store our state in.
 	DataDir string
 
+	// DefaultQueryTime is the amount of time a blocking query will wait before
+	// Consul will force a response. This value can be overridden by the 'wait'
+	// query parameter.
+	DefaultQueryTime time.Duration
+
+	// MaxQueryTime is the maximum amount of time a blocking query can wait
+	// before Consul will force a response. Consul applies jitter to the wait
+	// time. The jittered time will be capped to MaxQueryTime.
+	MaxQueryTime time.Duration
+
 	// DevMode is used to enable a development server mode.
 	DevMode bool
 
@@ -546,6 +556,8 @@ func DefaultConfig() *Config {
 
 		ServerHealthInterval: 2 * time.Second,
 		AutopilotInterval:    10 * time.Second,
+		DefaultQueryTime:     300 * time.Second,
+		MaxQueryTime:         600 * time.Second,
 		EnterpriseConfig:     DefaultEnterpriseConfig(),
 	}
 

agent/consul/leader_connect.go

@@ -34,10 +34,6 @@ var (
 	// maxRetryBackoff is the maximum number of seconds to wait between failed blocking
 	// queries when backing off.
 	maxRetryBackoff = 256
-
-	// maxRootsQueryTime is the maximum time the primary roots watch query can block before
-	// returning.
-	maxRootsQueryTime = maxQueryTime
 )
 
 // initializeCAConfig is used to initialize the CA config if necessary
@@ -602,7 +598,8 @@ func (s *Server) secondaryCARootWatch(ctx context.Context) error {
 	args := structs.DCSpecificRequest{
 		Datacenter: s.config.PrimaryDatacenter,
 		QueryOptions: structs.QueryOptions{
-			MaxQueryTime: maxRootsQueryTime,
+			// the maximum time the primary roots watch query can block before returning
+			MaxQueryTime: s.config.MaxQueryTime,
 		},
 	}
 

agent/consul/leader_connect_test.go

@@ -514,12 +514,11 @@ func TestLeader_SecondaryCA_TransitionFromPrimary(t *testing.T) {
 func TestLeader_SecondaryCA_UpgradeBeforePrimary(t *testing.T) {
 	t.Parallel()
 
-	maxRootsQueryTime = 500 * time.Millisecond
-
 	// Initialize dc1 as the primary DC
 	dir1, s1 := testServerWithConfig(t, func(c *Config) {
 		c.PrimaryDatacenter = "dc1"
 		c.Build = "1.3.0"
+		c.MaxQueryTime = 500 * time.Millisecond
 	})
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
@@ -531,6 +530,7 @@ func TestLeader_SecondaryCA_UpgradeBeforePrimary(t *testing.T) {
 		c.Datacenter = "dc2"
 		c.PrimaryDatacenter = "dc1"
 		c.Build = "1.6.0"
+		c.MaxQueryTime = 500 * time.Millisecond
 	})
 	defer os.RemoveAll(dir2)
 	defer s2.Shutdown()

agent/consul/rpc.go

@@ -24,13 +24,6 @@ import (
 )
 
 const (
-	// maxQueryTime is used to bound the limit of a blocking query
-	maxQueryTime = 600 * time.Second
-
-	// defaultQueryTime is the amount of time we block waiting for a change
-	// if no time is specified. Previously we would wait the maxQueryTime.
-	defaultQueryTime = 300 * time.Second
-
 	// jitterFraction is a the limit to the amount of jitter we apply
 	// to a user specified MaxQueryTime. We divide the specified time by
 	// the fraction. So 16 == 6.25% limit of jitter. This same fraction
@@ -466,10 +459,10 @@ func (s *Server) blockingQuery(queryOpts structs.QueryOptionsCompat, queryMeta s
 
 	queryTimeout = queryOpts.GetMaxQueryTime()
 	// Restrict the max query time, and ensure there is always one.
-	if queryTimeout > maxQueryTime {
+	if queryTimeout > s.config.MaxQueryTime {
-		queryTimeout = maxQueryTime
+		queryTimeout = s.config.MaxQueryTime
 	} else if queryTimeout <= 0 {
-		queryTimeout = defaultQueryTime
+		queryTimeout = s.config.DefaultQueryTime
 	}
 
 	// Apply a small amount of jitter to the request.

website/source/docs/agent/options.html.md

@@ -285,6 +285,16 @@ The options below are all specified on the command-line.
 
 * <a name="_log_rotate_max_files"></a><a href="#_log_rotate_max_files">`-log-rotate-max-files`</a> - to specify the maximum number of older log file archives to keep. Defaults to 0 (no files are ever deleted). Set to -1 to discard old log files when a new one is created.
 
+* <a name="_default_query_time"></a><a href="#_default_query_time">`-default-query-time`</a> - This flag controls the
+  amount of time a blocking query will wait before Consul will force a response.
+  This value can be overridden by the `wait` query parameter. Note that Consul
+  applies some jitter on top of this time. Defaults to 300s.
+
+* <a name="_max_query_time"></a><a href="#_max_query_time">`-max-query-time`</a> -
+  this flag controls the maximum amount of time a blocking query can wait before
+  Consul will force a response. Consul applies jitter to the wait time. The jittered
+  time will be capped to this time. Defaults to 600s.
+
 * <a name="_join"></a><a href="#_join">`-join`</a> - Address of another agent
   to join upon starting up. This can be
   specified multiple times to specify multiple agents to join. If Consul is
@@ -1374,6 +1384,12 @@ default will automatically work with some tooling.
 * <a name="log_level"></a><a href="#log_level">`log_level`</a> Equivalent to the
   [`-log-level` command-line flag](#_log_level).
 
+* <a name="default_query_time"></a><a href="#default_query_time">`default_query_time`</a>
+  Equivalent to the [`-default-query-time` command-line flag](#_default_query_time).
+
+* <a name="max_query_time"></a><a href="#max_query_time">`max_query_time`</a>
+  Equivalent to the [`-max-query-time` command-line flag](#_max_query_time).
+
 * <a name="node_id"></a><a href="#node_id">`node_id`</a> Equivalent to the
   [`-node-id` command-line flag](#_node_id).