Merge pull request #2303 from hashicorp/f-tunes

Adds performance tuning capability for Raft, detuned defaults, and supplemental docs.

command/agent/agent.go

@@ -261,6 +261,11 @@ func (a *Agent) consulConfig() *consul.Config {
 	// Apply dev mode
 	base.DevMode = a.config.DevMode
 
+	// Apply performance factors
+	if a.config.Performance.RaftMultiplier > 0 {
+		base.ScaleRaft(a.config.Performance.RaftMultiplier)
+	}
+
 	// Override with our config
 	if a.config.Datacenter != "" {
 		base.Datacenter = a.config.Datacenter

command/agent/agent_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/consul/consul"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/raft"
 )
 
 const (
@@ -191,6 +192,44 @@ func TestAgent_CheckAdvertiseAddrsSettings(t *testing.T) {
 	}
 }
 
+func TestAgent_CheckPerformanceSettings(t *testing.T) {
+	// Try a default config.
+	{
+		c := nextConfig()
+		c.ConsulConfig = nil
+		dir, agent := makeAgent(t, c)
+		defer os.RemoveAll(dir)
+		defer agent.Shutdown()
+
+		raftMult := time.Duration(consul.DefaultRaftMultiplier)
+		r := agent.consulConfig().RaftConfig
+		def := raft.DefaultConfig()
+		if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout ||
+			r.ElectionTimeout != raftMult*def.ElectionTimeout ||
+			r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout {
+			t.Fatalf("bad: %#v", *r)
+		}
+	}
+
+	// Try a multiplier.
+	{
+		c := nextConfig()
+		c.Performance.RaftMultiplier = 99
+		dir, agent := makeAgent(t, c)
+		defer os.RemoveAll(dir)
+		defer agent.Shutdown()
+
+		const raftMult time.Duration = 99
+		r := agent.consulConfig().RaftConfig
+		def := raft.DefaultConfig()
+		if r.HeartbeatTimeout != raftMult*def.HeartbeatTimeout ||
+			r.ElectionTimeout != raftMult*def.ElectionTimeout ||
+			r.LeaderLeaseTimeout != raftMult*def.LeaderLeaseTimeout {
+			t.Fatalf("bad: %#v", *r)
+		}
+	}
+}
+
 func TestAgent_ReconnectConfigSettings(t *testing.T) {
 	c := nextConfig()
 	func() {

command/agent/config.go

@@ -111,6 +111,13 @@ type DNSConfig struct {
 	DisableCompression bool `mapstructure:"disable_compression"`
 }
 
+// Performance is used to tune the performance of Consul's subsystems.
+type Performance struct {
+	// RaftMultiplier is an integer multiplier used to scale Raft timing
+	// parameters: HeartbeatTimeout, ElectionTimeout, and LeaderLeaseTimeout.
+	RaftMultiplier uint `mapstructure:"raft_multiplier"`
+}
+
 // Telemetry is the telemetry configuration for the server
 type Telemetry struct {
 	// StatsiteAddr is the address of a statsite instance. If provided,
@@ -205,10 +212,13 @@ type Telemetry struct {
 // Some of this is configurable as CLI flags, but most must
 // be set using a configuration file.
 type Config struct {
-	// DevMode enables a fast-path mode of opertaion to bring up an in-memory
+	// DevMode enables a fast-path mode of operation to bring up an in-memory
 	// server with minimal configuration. Useful for developing Consul.
 	DevMode bool `mapstructure:"-"`
 
+	// Performance is used to tune the performance of Consul's subsystems.
+	Performance Performance `mapstructure:"performance"`
+
 	// Bootstrap is used to bring up the first Consul server, and
 	// permits that node to elect itself leader
 	Bootstrap bool `mapstructure:"bootstrap"`
@@ -932,6 +942,11 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 		result.AdvertiseAddrs.RPC = addr
 	}
 
+	// Enforce the max Raft multiplier.
+	if result.Performance.RaftMultiplier > consul.MaxRaftMultiplier {
+		return nil, fmt.Errorf("Performance.RaftMultiplier must be <= %d", consul.MaxRaftMultiplier)
+	}
+
 	return &result, nil
 }
 
@@ -1085,6 +1100,11 @@ func DecodeCheckDefinition(raw interface{}) (*CheckDefinition, error) {
 func MergeConfig(a, b *Config) *Config {
 	var result Config = *a
 
+	// Propagate non-default performance settings
+	if b.Performance.RaftMultiplier > 0 {
+		result.Performance.RaftMultiplier = b.Performance.RaftMultiplier
+	}
+
 	// Copy the strings if they're set
 	if b.Bootstrap {
 		result.Bootstrap = true

command/agent/config_test.go

@@ -957,6 +957,23 @@ func TestDecodeConfig_invalidKeys(t *testing.T) {
 	}
 }
 
+func TestDecodeConfig_Performance(t *testing.T) {
+	input := `{"performance": { "raft_multiplier": 3 }}`
+	config, err := DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if config.Performance.RaftMultiplier != 3 {
+		t.Fatalf("bad: multiplier isn't set: %#v", config)
+	}
+
+	input = `{"performance": { "raft_multiplier": 11 }}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err == nil || !strings.Contains(err.Error(), "Performance.RaftMultiplier must be <=") {
+		t.Fatalf("bad: %v", err)
+	}
+}
+
 func TestDecodeConfig_Services(t *testing.T) {
 	input := `{
 		"services": [
@@ -1382,6 +1399,9 @@ func TestMergeConfig(t *testing.T) {
 	}
 
 	b := &Config{
+		Performance: Performance{
+			RaftMultiplier: 99,
+		},
 		Bootstrap:       true,
 		BootstrapExpect: 3,
 		Datacenter:      "dc2",

consul/config.go

@@ -17,6 +17,15 @@ const (
 	DefaultDC          = "dc1"
 	DefaultLANSerfPort = 8301
 	DefaultWANSerfPort = 8302
+
+	// DefaultRaftMultiplier is used as a baseline Raft configuration that
+	// will be reliable on a very basic server. See docs/guides/performance.html
+	// for information on how this value was obtained.
+	DefaultRaftMultiplier uint = 5
+
+	// MaxRaftMultiplier is a fairly arbitrary upper bound that limits the
+	// amount of performance detuning that's possible.
+	MaxRaftMultiplier uint = 10
 )
 
 var (
@@ -314,8 +323,11 @@ func DefaultConfig() *Config {
 		CoordinateUpdateBatchSize:  128,
 		CoordinateUpdateMaxBatches: 5,
 
-		// Hold an RPC for up to 5 seconds by default
-		RPCHoldTimeout: 5 * time.Second,
+		// This holds RPCs during leader elections. For the default Raft
+		// config the election timeout is 5 seconds, so we set this a
+		// bit longer to try to cover that period. This should be more
+		// than enough when running in the high performance mode.
+		RPCHoldTimeout: 7 * time.Second,
 	}
 
 	// Increase our reap interval to 3 days instead of 24h.
@@ -333,6 +345,7 @@ func DefaultConfig() *Config {
 	// Enable interoperability with unversioned Raft library, and don't
 	// start using new ID-based features yet.
 	conf.RaftConfig.ProtocolVersion = 1
+	conf.ScaleRaft(DefaultRaftMultiplier)
 
 	// Disable shutdown on removal
 	conf.RaftConfig.ShutdownOnRemove = false
@@ -340,6 +353,19 @@ func DefaultConfig() *Config {
 	return conf
 }
 
+// ScaleRaft sets the config to have Raft timing parameters scaled by the given
+// performance multiplier. This is done in an idempotent way so it's not tricky
+// to call this when composing configurations and potentially calling this
+// multiple times on the same structure.
+func (c *Config) ScaleRaft(raftMultRaw uint) {
+	raftMult := time.Duration(raftMultRaw)
+
+	def := raft.DefaultConfig()
+	c.RaftConfig.HeartbeatTimeout = raftMult * def.HeartbeatTimeout
+	c.RaftConfig.ElectionTimeout = raftMult * def.ElectionTimeout
+	c.RaftConfig.LeaderLeaseTimeout = raftMult * def.LeaderLeaseTimeout
+}
+
 func (c *Config) tlsConfig() *tlsutil.Config {
 	tlsConf := &tlsutil.Config{
 		VerifyIncoming:       c.VerifyIncoming,

testutil/server.go

@@ -32,6 +32,11 @@ import (
 // offset is used to atomically increment the port numbers.
 var offset uint64
 
+// TestPerformanceConfig configures the performance parameters.
+type TestPerformanceConfig struct {
+	RaftMultiplier uint `json:"raft_multiplier,omitempty"`
+}
+
 // TestPortConfig configures the various ports used for services
 // provided by the Consul server.
 type TestPortConfig struct {
@@ -51,20 +56,21 @@ type TestAddressConfig struct {
 
 // TestServerConfig is the main server configuration struct.
 type TestServerConfig struct {
-	NodeName          string             `json:"node_name"`
-	Bootstrap         bool               `json:"bootstrap,omitempty"`
-	Server            bool               `json:"server,omitempty"`
-	DataDir           string             `json:"data_dir,omitempty"`
-	Datacenter        string             `json:"datacenter,omitempty"`
-	DisableCheckpoint bool               `json:"disable_update_check"`
-	LogLevel          string             `json:"log_level,omitempty"`
-	Bind              string             `json:"bind_addr,omitempty"`
-	Addresses         *TestAddressConfig `json:"addresses,omitempty"`
-	Ports             *TestPortConfig    `json:"ports,omitempty"`
-	ACLMasterToken    string             `json:"acl_master_token,omitempty"`
-	ACLDatacenter     string             `json:"acl_datacenter,omitempty"`
-	ACLDefaultPolicy  string             `json:"acl_default_policy,omitempty"`
-	Stdout, Stderr    io.Writer          `json:"-"`
+	NodeName          string                 `json:"node_name"`
+	Performance       *TestPerformanceConfig `json:"performance,omitempty"`
+	Bootstrap         bool                   `json:"bootstrap,omitempty"`
+	Server            bool                   `json:"server,omitempty"`
+	DataDir           string                 `json:"data_dir,omitempty"`
+	Datacenter        string                 `json:"datacenter,omitempty"`
+	DisableCheckpoint bool                   `json:"disable_update_check"`
+	LogLevel          string                 `json:"log_level,omitempty"`
+	Bind              string                 `json:"bind_addr,omitempty"`
+	Addresses         *TestAddressConfig     `json:"addresses,omitempty"`
+	Ports             *TestPortConfig        `json:"ports,omitempty"`
+	ACLMasterToken    string                 `json:"acl_master_token,omitempty"`
+	ACLDatacenter     string                 `json:"acl_datacenter,omitempty"`
+	ACLDefaultPolicy  string                 `json:"acl_default_policy,omitempty"`
+	Stdout, Stderr    io.Writer              `json:"-"`
 }
 
 // ServerConfigCallback is a function interface which can be
@@ -79,11 +85,14 @@ func defaultServerConfig() *TestServerConfig {
 	return &TestServerConfig{
 		NodeName:          fmt.Sprintf("node%d", idx),
 		DisableCheckpoint: true,
-		Bootstrap:         true,
-		Server:            true,
-		LogLevel:          "debug",
-		Bind:              "127.0.0.1",
-		Addresses:         &TestAddressConfig{},
+		Performance: &TestPerformanceConfig{
+			RaftMultiplier: 1,
+		},
+		Bootstrap: true,
+		Server:    true,
+		LogLevel:  "debug",
+		Bind:      "127.0.0.1",
+		Addresses: &TestAddressConfig{},
 		Ports: &TestPortConfig{
 			DNS:     20000 + idx,
 			HTTP:    21000 + idx,

website/source/docs/agent/options.html.markdown

@@ -576,6 +576,25 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
 * <a name="node_name"></a><a href="#node_name">`node_name`</a> Equivalent to the
   [`-node` command-line flag](#_node).
 
+* <a name="performance"></a><a href="#performance">`performance`</a> Available in Consul 0.7 and
+  later, this is a nested object that allows tuning the performance of different subsystems in
+  Consul. See the [Server Performance](/docs/guides/performance.html) guide for more details. The
+  following parameters are available:
+    * <a name="raft_multiplier"></a><a href="#raft_multiplier">`raft_multiplier`</a> - An integer
+      multiplier used by Consul servers to scale key Raft timing parameters. Omitting this value
+      or setting it to 0 uses default timing described below. Lower values are used to tighten
+      timing and increase sensitivity while higher values relax timings and reduce sensitivity.
+      Tuning this affects the time it takes Consul to detect leader failures and to perform
+      leader elections, at the expense of requiring more network and CPU resources for better
+      performance.<br><br>By default, Consul will use a lower-performance timing that's suitable
+      for [minimal Consul servers](/docs/guides/performance.html#minumum), currently equivalent
+      to setting this to a value of 5 (this default may be changed in future versions of Consul,
+      depending if the target minimum server profile changes). Setting this to a value of 1 will
+      configure Raft to its highest-performance mode, equivalent to the default timing of Consul
+      prior to 0.7, and is recommended for [production Consul servers](/docs/guides/performance.html#production).
+      See the note on [last contact](/docs/guides/performance.html#last-contact) timing for more
+      details on tuning this parameter. The maximum allowed value is 10.
+
 * <a name="ports"></a><a href="#ports">`ports`</a> This is a nested object that allows setting
   the bind ports for the following keys:
     * <a name="dns_port"></a><a href="#dns_port">`dns`</a> - The DNS server, -1 to disable. Default 8600.

website/source/docs/agent/telemetry.html.markdown

@@ -129,8 +129,8 @@ These metrics are used to monitor the health of the Consul servers.
     <td>timer</td>
   </tr>
   <tr>
-    <td>`consul.raft.leader.lastContact`</td>
-    <td>This measures the time that a Consul server was last contacted by the leader (will be zero on the leader itself). This is a general indicator of latency in the Raft subsystem, and gives a general indicator of how far behind [stale](/docs/agent/http.html#consistency) queries will be.</td>
+    <td><a name="last-contact"></a>`consul.raft.leader.lastContact`</td>
+    <td>This will only be emitted by the Raft leader and measures the time since the leader was last able to contact the follower nodes when checking its leader lease. It can be used as a measure for how stable the Raft timing is and how close the leader is to timing out its lease.<br><br>The lease timeout is 500 ms times the [`raft_multiplier` configuration](/docs/agent/options.html#raft_multiplier), so this telemetry value should not be getting close to that configured value, otherwise the Raft timing is marginal and might need to be tuned, or more powerful servers might be needed. See the [Server Performance](/docs/guides/performance.html) guide for more details.</td>
     <td>ms</td>
     <td>timer</td>
   </tr>

website/source/docs/guides/dns-cache.html.markdown

@@ -20,6 +20,7 @@ for each lookup and can potentially exhaust the query throughput of a cluster.
 For this reason, Consul provides a number of tuning parameters that can
 customize how DNS queries are handled.
 
+<a name="stale"></a>
 ## Stale Reads
 
 Stale reads can be used to reduce latency and increase the throughput
@@ -60,6 +61,7 @@ client and Consul and set the cache values appropriately. In many cases
 "appropriately" simply is turning negative response caching off to get the best
 recovery time when a service becomes available again.
 
+<a name="ttl"></a>
 ## TTL Values
 
 TTL values can be set to allow DNS results to be cached downstream of Consul. Higher

website/source/docs/guides/performance.html.markdown

@@ -0,0 +1,86 @@
+---
+layout: "docs"
+page_title: "Server Performance"
+sidebar_current: "docs-guides-performance"
+description: |-
+  Consul requires different amounts of compute resources, depending on cluster size and expected workload. This guide provides guidance on choosing compute resources.
+---
+
+# Server Performance
+
+Since Consul servers run a [consensus protocol](/docs/internals/consensus.html) to
+process all write operations and are contacted on nearly all read operations, server
+performance is critical for overall throughput and health of a Consul cluster. Servers
+are generally I/O bound for writes because the underlying Raft log store performs a sync
+to disk every time an entry is appended. Servers are generally CPU bound for reads since
+reads work from a fully in-memory data store that is optimized for concurrent access.
+
+<a name="minimum"></a>
+## Minimum Server Requirements
+
+In Consul 0.7, the default server [performance parameters](/docs/agent/options.html#performance)
+were tuned to allow Consul to run reliably (but relatively slowly) on a server cluster of three
+[AWS t2.micro](https://aws.amazon.com/ec2/instance-types/) instances. These thresholds
+were determined empirically using a leader instance that was under sufficient read, write,
+and network load to cause it to permanently be at zero CPU credits, forcing it to the baseline
+performance mode for that instance type. Real-world workloads typically have more bursts of
+activity, so this is a conservative and pessimistic tuning strategy.
+
+This default was chosen based on feedback from users, many of whom wanted a low cost way
+to run small production or development clusters with low cost compute resources, at the
+expense of some performance in leader failure detection and leader election times.
+
+The default performance configuration is equivalent to this:
+
+```javascript
+{
+  "performance": {
+    "raft_multiplier": 5
+  }
+}
+```
+
+<a name="production"></a>
+## Production Server Requirements
+
+When running Consul 0.7 and later in production, it is recommended to configure the server
+[performance parameters](/docs/agent/options.html#performance) back to Consul's original
+high-performance settings. This will let Consul servers detect a failed leader and complete
+leader elections much more quickly than the default configuration which extends key Raft
+timeouts by a factor of 5, so it quite slow during these events.
+
+The high performance configuration is simple and looks like this:
+
+```javascript
+{
+  "performance": {
+    "raft_multiplier": 1
+  }
+}
+```
+
+It's best to benchmark with a realistic workload when choosing a production server for Consul.
+Here are some general recommendations:
+
+* For write-heavy workloads, disk speed on the servers is key for performance. Use SSDs or
+another fast disk technology for the best write throughput.
+
+* <a name="last-contact"></a>Spurious leader elections can be caused by networking issues between
+the servers or insufficient CPU resources. Users in cloud environments often bump their servers
+up to the next instance class with improved networking and CPU until leader elections stabilize,
+and in Consul 0.7 or later the [performance parameters](/docs/agent/options.html#performance)
+configuration now gives you tools to trade off performance instead of upsizing servers. You can
+use the [`consul.raft.leader.lastContact` telemetry](/docs/agent/telemetry.html#last-contact)
+to observe how the Raft timing is performing and guide the decision to de-tune Raft performance
+or add more powerful servers.
+
+* For DNS-heavy workloads, configuring all Consul agents in a cluster with the
+[`allow_stale`](/docs/agent/options.html#allow_stale) configuration option will allow reads to
+scale across all Consul servers, not just the leader. See [Stale Reads](/docs/guides/dns-cache.html#stale)
+in the [DNS Caching](/docs/guides/dns-cache.html) guide for more details. It's also good to set
+reasonable, non-zero [DNS TTL values](/docs/guides/dns-cache.html#ttl) if your clients will
+respect them.
+
+* In other applications that perform high volumes of reads against Consul, consider using the
+[stale consistency mode](/docs/agent/http.html#consistency) available to allow reads to scale
+across all the servers and not just be forwarded to the leader.

website/source/docs/upgrade-specific.html.markdown

@@ -19,6 +19,27 @@ standard upgrade flow.
 Consul version 0.7 is a very large release with many important changes. Changes
 to be aware of during an upgrade are categorized below.
 
+#### Performance Tuning and New Defaults
+
+Consul 0.7 introduced support for tuning Raft performance using a new
+[performance configuration block](/docs/agent/options.html#performance). Also,
+the default Raft timing is set to a lower-performance mode suitable for
+[minimal Consul servers](/docs/guides/performance.html#minumum).
+
+To continue to use the high-performance settings that were the default prior to
+Consul 0.7 (recommended for production servers), add the following configuration
+to all Consul servers when upgrading:
+
+```javascript
+{
+  "performance": {
+    "raft_multiplier": 1
+  }
+}
+```
+
+See the [Server Performance](/docs/guides/performance.html) guide for more details.
+
 #### Default Configuration Changes
 
 The default behavior of [`skip_leave_on_interrupt`](/docs/agent/options.html#skip_leave_on_interrupt)

website/source/layouts/docs.erb

@@ -228,6 +228,10 @@
 					<a href="/docs/guides/atlas.html">Atlas Integration</a>
 					</li>
 
+					<li<%= sidebar_current("docs-guides-performance") %>>
+					<a href="/docs/guides/performance.html">Server Performance</a>
+					</li>
+
 					<li<%= sidebar_current("docs-guides-servers") %>>
 					<a href="/docs/guides/servers.html">Adding/Removing Servers</a>
 					</li>