agent: add allowStale option for HTTP API (#3142)

This patch adds an "allowStale" option to the HTTP API
configuration which allows stale reads to provide linear
read scalability.

Fixes #3142

agent/config.go

@@ -131,6 +131,12 @@ type DNSConfig struct {
 
 // HTTPConfig is used to fine tune the Http sub-system.
 type HTTPConfig struct {
+	// AllowStale is used to enable lookups with stale
+	// data. This gives horizontal read scalability since
+	// any Consul server can service the query instead of
+	// only the leader.
+	AllowStale *bool `mapstructure:"allow_stale"`
+
 	// ResponseHeaders are used to add HTTP header response fields to the HTTP API responses.
 	ResponseHeaders map[string]string `mapstructure:"response_headers"`
 }
@@ -918,6 +924,9 @@ func DefaultConfig() *Config {
 			MaxStale:        10 * 365 * 24 * time.Hour,
 			RecursorTimeout: 2 * time.Second,
 		},
+		HTTPConfig: HTTPConfig{
+			AllowStale: Bool(true),
+		},
 		Telemetry: Telemetry{
 			StatsitePrefix: "consul",
 		},
@@ -2002,6 +2011,9 @@ func MergeConfig(a, b *Config) *Config {
 			result.HTTPConfig.ResponseHeaders[field] = value
 		}
 	}
+	if b.HTTPConfig.AllowStale != nil {
+		result.HTTPConfig.AllowStale = b.HTTPConfig.AllowStale
+	}
 	if len(b.Meta) != 0 {
 		if result.Meta == nil {
 			result.Meta = make(map[string]string)

agent/http.go

@@ -358,15 +358,17 @@ func parseWait(resp http.ResponseWriter, req *http.Request, b *structs.QueryOpti
 }
 
 // parseConsistency is used to parse the ?stale and ?consistent query params.
+// allowStale forces stale consistency instead of default one if none was provided in the query.
 // Returns true on error
-func parseConsistency(resp http.ResponseWriter, req *http.Request, b *structs.QueryOptions) bool {
+func parseConsistency(resp http.ResponseWriter, req *http.Request, allowStale bool, b *structs.QueryOptions) bool {
 	query := req.URL.Query()
-	if _, ok := query["stale"]; ok {
-		b.AllowStale = true
-	}
 	if _, ok := query["consistent"]; ok {
 		b.RequireConsistent = true
 	}
+	b.AllowStale = !b.RequireConsistent && allowStale
+	if _, ok := query["stale"]; ok {
+		b.AllowStale = true
+	}
 	if b.AllowStale && b.RequireConsistent {
 		resp.WriteHeader(http.StatusBadRequest) // 400
 		fmt.Fprint(resp, "Cannot specify ?stale with ?consistent, conflicting semantics.")
@@ -433,7 +435,8 @@ func (s *HTTPServer) parseMetaFilter(req *http.Request) map[string]string {
 func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, dc *string, b *structs.QueryOptions) bool {
 	s.parseDC(req, dc)
 	s.parseToken(req, &b.Token)
-	if parseConsistency(resp, req, b) {
+	allowStale := s.agent.config.HTTPConfig.AllowStale
+	if parseConsistency(resp, req, *allowStale, b) {
 		return true
 	}
 	return parseWait(resp, req, b)

agent/http_test.go

@@ -436,31 +436,36 @@ func TestParseWait_InvalidIndex(t *testing.T) {
 func TestParseConsistency(t *testing.T) {
 	t.Parallel()
 	resp := httptest.NewRecorder()
-	var b structs.QueryOptions
 
-	req, _ := http.NewRequest("GET", "/v1/catalog/nodes?stale", nil)
-	if d := parseConsistency(resp, req, &b); d {
-		t.Fatalf("unexpected done")
+	tests := []struct {
+		url                   string
+		allowStale            bool
+		wantAllowStale        bool
+		wantRequireConsistent bool
+	}{
+		{"/v1/catalog/nodes?stale", false, true, false},
+		{"/v1/catalog/nodes?stale", true, true, false},
+		{"/v1/catalog/nodes?consistent", false, false, true},
+		{"/v1/catalog/nodes?consistent", true, false, true},
+		{"/v1/catalog/nodes", false, false, false},
+		{"/v1/catalog/nodes", true, true, false},
 	}
 
-	if !b.AllowStale {
-		t.Fatalf("Bad: %v", b)
-	}
-	if b.RequireConsistent {
-		t.Fatalf("Bad: %v", b)
-	}
-
-	b = structs.QueryOptions{}
-	req, _ = http.NewRequest("GET", "/v1/catalog/nodes?consistent", nil)
-	if d := parseConsistency(resp, req, &b); d {
-		t.Fatalf("unexpected done")
-	}
-
-	if b.AllowStale {
-		t.Fatalf("Bad: %v", b)
-	}
-	if !b.RequireConsistent {
-		t.Fatalf("Bad: %v", b)
+	for _, tt := range tests {
+		name := fmt.Sprintf("url=%v, HTTP.AllowStale=%v", tt.url, tt.allowStale)
+		t.Run(name, func(t *testing.T) {
+			var q structs.QueryOptions
+			req, _ := http.NewRequest("GET", tt.url, nil)
+			if d := parseConsistency(resp, req, tt.allowStale, &q); d {
+				t.Fatalf("Failed to parse consistency.")
+			}
+			if got, want := q.AllowStale, tt.wantAllowStale; got != want {
+				t.Fatalf("got allowStale %v want %v", got, want)
+			}
+			if got, want := q.RequireConsistent, tt.wantRequireConsistent; got != want {
+				t.Fatalf("got requireConsistent %v want %v", got, want)
+			}
+		})
 	}
 }
 
@@ -470,7 +475,7 @@ func TestParseConsistency_Invalid(t *testing.T) {
 	var b structs.QueryOptions
 
 	req, _ := http.NewRequest("GET", "/v1/catalog/nodes?stale&consistent", nil)
-	if d := parseConsistency(resp, req, &b); !d {
+	if d := parseConsistency(resp, req, false, &b); !d {
 		t.Fatalf("expected done")
 	}
 

website/source/docs/agent/options.html.md

@@ -753,6 +753,13 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
   <br><br>
   The following sub-keys are available:
 
+  * <a name="allow_stale"></a><a href="#allow_stale">`allow_stale`</a> - Enables a stale query
+  for DNS information. This allows any Consul server, rather than only the leader, to service
+  the request. The advantage of this is you get linear read scalability with Consul servers.
+  In versions of Consul prior to 0.7, this defaulted to false, meaning all requests are serviced
+  by the leader, providing stronger consistency but less throughput and higher latency. In Consul
+  0.7 and later, this defaults to true for better utilization of available servers.
+
   * <a name="response_headers"></a><a href="#response_headers">`response_headers`</a>
     This object allows adding headers to the HTTP API responses.
     For example, the following config can be used to enable