Retains the last output when a TTL check times out.

command/agent/check.go

@@ -232,6 +232,9 @@ type CheckTTL struct {
 
 	timer *time.Timer
 
+	lastOutput     string
+	lastOutputLock sync.RWMutex
+
 	stop     bool
 	stopCh   chan struct{}
 	stopLock sync.Mutex
@@ -265,7 +268,7 @@ func (c *CheckTTL) run() {
 		case <-c.timer.C:
 			c.Logger.Printf("[WARN] agent: Check '%v' missed TTL, is now critical",
 				c.CheckID)
-			c.Notify.UpdateCheck(c.CheckID, structs.HealthCritical, "TTL expired")
+			c.Notify.UpdateCheck(c.CheckID, structs.HealthCritical, c.getExpiredOutput())
 
 		case <-c.stopCh:
 			return
@@ -273,12 +276,31 @@ func (c *CheckTTL) run() {
 	}
 }
 
+// getExpiredOutput formats the output for the case when the TTL is expired.
+func (c *CheckTTL) getExpiredOutput() string {
+	c.lastOutputLock.RLock()
+	defer c.lastOutputLock.RUnlock()
+
+	const prefix = "TTL expired"
+	if c.lastOutput == "" {
+		return fmt.Sprintf("%s (no output was available before timeout)", prefix)
+	}
+
+	return fmt.Sprintf("%s (last output before timeout follows)\n\n%s", prefix, c.lastOutput)
+}
+
 // SetStatus is used to update the status of the check,
 // and to renew the TTL. If expired, TTL is restarted.
 func (c *CheckTTL) SetStatus(status, output string) {
 	c.Logger.Printf("[DEBUG] agent: Check '%v' status is now %v",
 		c.CheckID, status)
 	c.Notify.UpdateCheck(c.CheckID, status, output)
+
+	// Store the last output so we can retain it if the TTL expires.
+	c.lastOutputLock.Lock()
+	c.lastOutput = output
+	c.lastOutputLock.Unlock()
+
 	c.timer.Reset(c.TTL)
 }
 

command/agent/check_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -150,7 +151,7 @@ func TestCheckTTL(t *testing.T) {
 	defer check.Stop()
 
 	time.Sleep(50 * time.Millisecond)
-	check.SetStatus(structs.HealthPassing, "")
+	check.SetStatus(structs.HealthPassing, "test-output")
 
 	if mock.updates["foo"] != 1 {
 		t.Fatalf("should have 1 updates %v", mock.updates)
@@ -176,6 +177,10 @@ func TestCheckTTL(t *testing.T) {
 	if mock.state["foo"] != structs.HealthCritical {
 		t.Fatalf("should be critical %v", mock.state)
 	}
+
+	if !strings.Contains(mock.output["foo"], "test-output") {
+		t.Fatalf("should have retained output %v", mock.output)
+	}
 }
 
 func mockHTTPServer(responseCode int) *httptest.Server {