adding check_update_stagger

command/agent/config.go

@@ -263,6 +263,12 @@ type Config struct {
 	CheckUpdateInterval    time.Duration `mapstructure:"-"`
 	CheckUpdateIntervalRaw string        `mapstructure:"check_update_interval" json:"-"`
 
+	// CheckUpdateStagger enables a randomization of the CheckUpdateInterval between
+	// .5 and 1.5 of that interval. This is useful if checks happen often to stagger
+	// writes and pervent them from all executing at the end of the same CheckUpdateInterval.
+	// Off by default.
+	CheckUpdateStagger bool `mapstructure:"check_update_stagger"`
+
 	// ACLToken is the default token used to make requests if a per-request
 	// token is not provided. If not configured the 'anonymous' token is used.
 	ACLToken string `mapstructure:"acl_token" json:"-"`

command/agent/config_test.go

@@ -494,6 +494,17 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	// CheckUpdateStagger
+	input = `{"check_update_stagger": true}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !config.CheckUpdateStagger {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	// ACLs
 	input = `{"acl_token": "1234", "acl_datacenter": "dc2",
 	"acl_ttl": "60s", "acl_down_policy": "deny",

command/agent/local.go

@@ -3,6 +3,7 @@ package agent
 import (
 	"fmt"
 	"log"
+	"math/rand"
 	"reflect"
 	"strings"
 	"sync"
@@ -208,7 +209,13 @@ func (l *localState) UpdateCheck(checkID, status, output string) {
 	if l.config.CheckUpdateInterval > 0 && check.Status == status {
 		check.Output = output
 		if _, ok := l.deferCheck[checkID]; !ok {
-			deferSync := time.AfterFunc(l.config.CheckUpdateInterval, func() {
+			var intv time.Duration
+			if l.config.CheckUpdateStagger {
+				intv = time.Duration(uint64(l.config.CheckUpdateInterval)/2 + uint64(rand.Int63())%uint64(l.config.CheckUpdateInterval))
+			} else {
+				intv = l.config.CheckUpdateInterval
+			}
+			deferSync := time.AfterFunc(intv, func() {
 				l.Lock()
 				if _, ok := l.checkStatus[checkID]; ok {
 					l.checkStatus[checkID] = syncStatus{inSync: false}

command/agent/local_test.go

@@ -601,7 +601,7 @@ func TestAgentAntiEntropy_Check_DeferSync(t *testing.T) {
 	})
 }
 
-func TestAgentAntiEntory_deleteService_fails(t *testing.T) {
+func TestAgentAntiEntropy_deleteService_fails(t *testing.T) {
 	l := new(localState)
 	if err := l.deleteService(""); err == nil {
 		t.Fatalf("should have failed")

website/source/docs/agent/options.html.markdown

@@ -353,6 +353,12 @@ definitions support being updated during a reload.
   reduce write pressure. If a check ever changes state, the new state and associated
   output is synchronized immediately. To disable this behavior, set the value to "0s".
 
+* <a name="check_update_stagger"></a><a href="#check_update_stagger">`check_update_stagger`</a>
+  CheckUpdateStagger enables a randomization of the CheckUpdateInterval between
+  .5 and 1.5 of that interval. This is useful if checks happen often to stagger
+  writes and pervent them from all executing at the end of the same CheckUpdateInterval.
+  Off by default.
+
 * <a name="client_addr"></a><a href="#client_addr">`client_addr`</a> Equivalent to the
   [`-client` command-line flag](#_client).