diff --git a/agent/agent.go b/agent/agent.go index 2a1acaa1d0..0bff3863e4 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1263,6 +1263,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) { base.TLSMinVersion = a.config.TLSMinVersion base.TLSCipherSuites = a.config.TLSCipherSuites base.TLSPreferServerCipherSuites = a.config.TLSPreferServerCipherSuites + base.DefaultQueryTime = a.config.DefaultQueryTime + base.MaxQueryTime = a.config.MaxQueryTime base.AutoEncryptAllowTLS = a.config.AutoEncryptAllowTLS diff --git a/agent/config/builder.go b/agent/config/builder.go index a674d8b013..6ab5bde613 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -819,6 +819,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { ExposeMaxPort: exposeMaxPort, DataDir: b.stringVal(c.DataDir), Datacenter: datacenter, + DefaultQueryTime: b.durationVal("default_query_time", c.DefaultQueryTime), DevMode: b.boolVal(b.Flags.DevMode), DisableAnonymousSignature: b.boolVal(c.DisableAnonymousSignature), DisableCoordinates: b.boolVal(c.DisableCoordinates), @@ -850,6 +851,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) { LogRotateBytes: b.intVal(c.LogRotateBytes), LogRotateDuration: b.durationVal("log_rotate_duration", c.LogRotateDuration), LogRotateMaxFiles: b.intVal(c.LogRotateMaxFiles), + MaxQueryTime: b.durationVal("max_query_time", c.MaxQueryTime), NodeID: types.NodeID(b.stringVal(c.NodeID)), NodeMeta: c.NodeMeta, NodeName: b.nodeName(c.NodeName), diff --git a/agent/config/config.go b/agent/config/config.go index 6af8439e05..ab2284158a 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -204,6 +204,7 @@ type Config struct { DNSRecursors []string `json:"recursors,omitempty" hcl:"recursors" mapstructure:"recursors"` DataDir *string `json:"data_dir,omitempty" hcl:"data_dir" mapstructure:"data_dir"` Datacenter *string `json:"datacenter,omitempty" hcl:"datacenter" mapstructure:"datacenter"` + DefaultQueryTime *string `json:"default_query_time,omitempty" hcl:"default_query_time" mapstructure:"default_query_time"` DisableAnonymousSignature *bool `json:"disable_anonymous_signature,omitempty" hcl:"disable_anonymous_signature" mapstructure:"disable_anonymous_signature"` DisableCoordinates *bool `json:"disable_coordinates,omitempty" hcl:"disable_coordinates" mapstructure:"disable_coordinates"` DisableHostNodeID *bool `json:"disable_host_node_id,omitempty" hcl:"disable_host_node_id" mapstructure:"disable_host_node_id"` @@ -234,6 +235,7 @@ type Config struct { LogRotateDuration *string `json:"log_rotate_duration,omitempty" hcl:"log_rotate_duration" mapstructure:"log_rotate_duration"` LogRotateBytes *int `json:"log_rotate_bytes,omitempty" hcl:"log_rotate_bytes" mapstructure:"log_rotate_bytes"` LogRotateMaxFiles *int `json:"log_rotate_max_files,omitempty" hcl:"log_rotate_max_files" mapstructure:"log_rotate_max_files"` + MaxQueryTime *string `json:"max_query_time,omitempty" hcl:"max_query_time" mapstructure:"max_query_time"` NodeID *string `json:"node_id,omitempty" hcl:"node_id" mapstructure:"node_id"` NodeMeta map[string]string `json:"node_meta,omitempty" hcl:"node_meta" mapstructure:"node_meta"` NodeName *string `json:"node_name,omitempty" hcl:"node_name" mapstructure:"node_name"` diff --git a/agent/config/default.go b/agent/config/default.go index 1ceeb94ad6..6c6c0d06d8 100644 --- a/agent/config/default.go +++ b/agent/config/default.go @@ -55,6 +55,7 @@ func DefaultSource() Source { check_update_interval = "5m" client_addr = "127.0.0.1" datacenter = "` + consul.DefaultDC + `" + default_query_time = "300s" disable_coordinates = false disable_host_node_id = true disable_remote_exec = true @@ -62,6 +63,7 @@ func DefaultSource() Source { encrypt_verify_incoming = true encrypt_verify_outgoing = true log_level = "INFO" + max_query_time = "600s" protocol = 2 retry_interval = "30s" retry_interval_wan = "30s" diff --git a/agent/config/flags.go b/agent/config/flags.go index ef39ddc576..4fd8611fa1 100644 --- a/agent/config/flags.go +++ b/agent/config/flags.go @@ -66,6 +66,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) { add(&f.ConfigFormat, "config-format", "Config files are in this format irrespective of their extension. Must be 'hcl' or 'json'") add(&f.Config.DataDir, "data-dir", "Path to a data directory to store agent state.") add(&f.Config.Datacenter, "datacenter", "Datacenter of the agent.") + add(&f.Config.DefaultQueryTime, "default-query-time", "the amount of time a blocking query will wait before Consul will force a response. This value can be overridden by the 'wait' query parameter.") add(&f.DevMode, "dev", "Starts the agent in development mode.") add(&f.Config.DisableHostNodeID, "disable-host-node-id", "Setting this to true will prevent Consul from using information from the host to generate a node ID, and will cause Consul to generate a random node ID instead.") add(&f.Config.DisableKeyringFile, "disable-keyring-file", "Disables the backing up of the keyring to a file.") @@ -85,6 +86,7 @@ func AddFlags(fs *flag.FlagSet, f *Flags) { add(&f.Config.LogRotateBytes, "log-rotate-bytes", "Maximum number of bytes that should be written to a log file") add(&f.Config.LogRotateDuration, "log-rotate-duration", "Time after which log rotation needs to be performed") add(&f.Config.LogRotateMaxFiles, "log-rotate-max-files", "Maximum number of log file archives to keep") + add(&f.Config.MaxQueryTime, "max-query-time", "the maximum amount of time a blocking query can wait before Consul will force a response. Consul applies jitter to the wait time. The jittered time will be capped to MaxQueryTime.") add(&f.Config.NodeName, "node", "Name of this node. Must be unique in the cluster.") add(&f.Config.NodeID, "node-id", "A unique ID for this node across space and time. Defaults to a randomly-generated ID that persists in the data-dir.") add(&f.Config.NodeMeta, "node-meta", "An arbitrary metadata key/value pair for this node, of the format `key:value`. Can be specified multiple times.") diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 02429ab582..f23226446e 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -600,6 +600,14 @@ type RuntimeConfig struct { // flag: -data-dir string DataDir string + // DefaultQueryTime is the amount of time a blocking query will wait before + // Consul will force a response. This value can be overridden by the 'wait' + // query parameter. + // + // hcl: default_query_time = "duration" + // flag: -default-query-time string + DefaultQueryTime time.Duration + // DevMode enables a fast-path mode of operation to bring up an in-memory // server with minimal configuration. Useful for developing Consul. // @@ -846,6 +854,14 @@ type RuntimeConfig struct { // flags: -log-rotate-max-files int LogRotateMaxFiles int + // MaxQueryTime is the maximum amount of time a blocking query can wait + // before Consul will force a response. Consul applies jitter to the wait + // time. The jittered time will be capped to MaxQueryTime. + // + // hcl: max_query_time = "duration" + // flags: -max-query-time string + MaxQueryTime time.Duration + // Node ID is a unique ID for this node across space and time. Defaults // to a randomly-generated ID that persists in the data-dir. // diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 3d23076705..2d506fb8ad 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -3757,6 +3757,7 @@ func TestFullConfig(t *testing.T) { }, "data_dir": "` + dataDir + `", "datacenter": "rzo029wg", + "default_query_time": "16743s", "disable_anonymous_signature": true, "disable_coordinates": true, "disable_host_node_id": true, @@ -3810,6 +3811,7 @@ func TestFullConfig(t *testing.T) { "kv_max_value_size": 1234567800000000 }, "log_level": "k1zo9Spt", + "max_query_time": "18237s", "node_id": "AsUIlw99", "node_meta": { "5mgGQMBk": "mJLtVMSG", @@ -4356,6 +4358,7 @@ func TestFullConfig(t *testing.T) { } data_dir = "` + dataDir + `" datacenter = "rzo029wg" + default_query_time = "16743s" disable_anonymous_signature = true disable_coordinates = true disable_host_node_id = true @@ -4410,6 +4413,7 @@ func TestFullConfig(t *testing.T) { kv_max_value_size = 1234567800000000 } log_level = "k1zo9Spt" + max_query_time = "18237s" node_id = "AsUIlw99" node_meta { "5mgGQMBk" = "mJLtVMSG" @@ -5064,6 +5068,7 @@ func TestFullConfig(t *testing.T) { DNSCacheMaxAge: 5 * time.Minute, DataDir: dataDir, Datacenter: "rzo029wg", + DefaultQueryTime: 16743 * time.Second, DevMode: true, DisableAnonymousSignature: true, DisableCoordinates: true, @@ -5098,6 +5103,7 @@ func TestFullConfig(t *testing.T) { LeaveDrainTime: 8265 * time.Second, LeaveOnTerm: true, LogLevel: "k1zo9Spt", + MaxQueryTime: 18237 * time.Second, NodeID: types.NodeID("AsUIlw99"), NodeMeta: map[string]string{"5mgGQMBk": "mJLtVMSG", "A7ynFMJB": "0Nx6RGab"}, NodeName: "otlLxGaI", @@ -5924,6 +5930,7 @@ func TestSanitize(t *testing.T) { "DNSCacheMaxAge": "0s", "DataDir": "", "Datacenter": "", + "DefaultQueryTime": "0s", "DevMode": false, "DisableAnonymousSignature": false, "DisableCoordinates": false, @@ -5967,6 +5974,7 @@ func TestSanitize(t *testing.T) { "LogRotateBytes": 0, "LogRotateDuration": "0s", "LogRotateMaxFiles": 0, + "MaxQueryTime": "0s", "NodeID": "", "NodeMeta": {}, "NodeName": "", diff --git a/agent/consul/config.go b/agent/consul/config.go index 2e4fc62c3f..6be6d3c66a 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -86,6 +86,16 @@ type Config struct { // DataDir is the directory to store our state in. DataDir string + // DefaultQueryTime is the amount of time a blocking query will wait before + // Consul will force a response. This value can be overridden by the 'wait' + // query parameter. + DefaultQueryTime time.Duration + + // MaxQueryTime is the maximum amount of time a blocking query can wait + // before Consul will force a response. Consul applies jitter to the wait + // time. The jittered time will be capped to MaxQueryTime. + MaxQueryTime time.Duration + // DevMode is used to enable a development server mode. DevMode bool @@ -546,6 +556,8 @@ func DefaultConfig() *Config { ServerHealthInterval: 2 * time.Second, AutopilotInterval: 10 * time.Second, + DefaultQueryTime: 300 * time.Second, + MaxQueryTime: 600 * time.Second, EnterpriseConfig: DefaultEnterpriseConfig(), } diff --git a/agent/consul/leader_connect.go b/agent/consul/leader_connect.go index 8196c9f101..8455885923 100644 --- a/agent/consul/leader_connect.go +++ b/agent/consul/leader_connect.go @@ -34,10 +34,6 @@ var ( // maxRetryBackoff is the maximum number of seconds to wait between failed blocking // queries when backing off. maxRetryBackoff = 256 - - // maxRootsQueryTime is the maximum time the primary roots watch query can block before - // returning. - maxRootsQueryTime = maxQueryTime ) // initializeCAConfig is used to initialize the CA config if necessary @@ -602,7 +598,8 @@ func (s *Server) secondaryCARootWatch(ctx context.Context) error { args := structs.DCSpecificRequest{ Datacenter: s.config.PrimaryDatacenter, QueryOptions: structs.QueryOptions{ - MaxQueryTime: maxRootsQueryTime, + // the maximum time the primary roots watch query can block before returning + MaxQueryTime: s.config.MaxQueryTime, }, } diff --git a/agent/consul/leader_connect_test.go b/agent/consul/leader_connect_test.go index 38c7be0293..51bf09cdd1 100644 --- a/agent/consul/leader_connect_test.go +++ b/agent/consul/leader_connect_test.go @@ -514,12 +514,11 @@ func TestLeader_SecondaryCA_TransitionFromPrimary(t *testing.T) { func TestLeader_SecondaryCA_UpgradeBeforePrimary(t *testing.T) { t.Parallel() - maxRootsQueryTime = 500 * time.Millisecond - // Initialize dc1 as the primary DC dir1, s1 := testServerWithConfig(t, func(c *Config) { c.PrimaryDatacenter = "dc1" c.Build = "1.3.0" + c.MaxQueryTime = 500 * time.Millisecond }) defer os.RemoveAll(dir1) defer s1.Shutdown() @@ -531,6 +530,7 @@ func TestLeader_SecondaryCA_UpgradeBeforePrimary(t *testing.T) { c.Datacenter = "dc2" c.PrimaryDatacenter = "dc1" c.Build = "1.6.0" + c.MaxQueryTime = 500 * time.Millisecond }) defer os.RemoveAll(dir2) defer s2.Shutdown() diff --git a/agent/consul/rpc.go b/agent/consul/rpc.go index 295d1e2adf..5d380bc270 100644 --- a/agent/consul/rpc.go +++ b/agent/consul/rpc.go @@ -24,13 +24,6 @@ import ( ) const ( - // maxQueryTime is used to bound the limit of a blocking query - maxQueryTime = 600 * time.Second - - // defaultQueryTime is the amount of time we block waiting for a change - // if no time is specified. Previously we would wait the maxQueryTime. - defaultQueryTime = 300 * time.Second - // jitterFraction is a the limit to the amount of jitter we apply // to a user specified MaxQueryTime. We divide the specified time by // the fraction. So 16 == 6.25% limit of jitter. This same fraction @@ -466,10 +459,10 @@ func (s *Server) blockingQuery(queryOpts structs.QueryOptionsCompat, queryMeta s queryTimeout = queryOpts.GetMaxQueryTime() // Restrict the max query time, and ensure there is always one. - if queryTimeout > maxQueryTime { - queryTimeout = maxQueryTime + if queryTimeout > s.config.MaxQueryTime { + queryTimeout = s.config.MaxQueryTime } else if queryTimeout <= 0 { - queryTimeout = defaultQueryTime + queryTimeout = s.config.DefaultQueryTime } // Apply a small amount of jitter to the request. diff --git a/website/source/docs/agent/options.html.md b/website/source/docs/agent/options.html.md index 064f946135..7b3e6097df 100644 --- a/website/source/docs/agent/options.html.md +++ b/website/source/docs/agent/options.html.md @@ -285,6 +285,16 @@ The options below are all specified on the command-line. * `-log-rotate-max-files` - to specify the maximum number of older log file archives to keep. Defaults to 0 (no files are ever deleted). Set to -1 to discard old log files when a new one is created. +* `-default-query-time` - This flag controls the + amount of time a blocking query will wait before Consul will force a response. + This value can be overridden by the `wait` query parameter. Note that Consul + applies some jitter on top of this time. Defaults to 300s. + +* `-max-query-time` - + this flag controls the maximum amount of time a blocking query can wait before + Consul will force a response. Consul applies jitter to the wait time. The jittered + time will be capped to this time. Defaults to 600s. + * `-join` - Address of another agent to join upon starting up. This can be specified multiple times to specify multiple agents to join. If Consul is @@ -1374,6 +1384,12 @@ default will automatically work with some tooling. * `log_level` Equivalent to the [`-log-level` command-line flag](#_log_level). +* `default_query_time` + Equivalent to the [`-default-query-time` command-line flag](#_default_query_time). + +* `max_query_time` + Equivalent to the [`-max-query-time` command-line flag](#_max_query_time). + * `node_id` Equivalent to the [`-node-id` command-line flag](#_node_id).