From 1f9479603cbcc53ea616c5326822fa02dfcfa983 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 14 Sep 2021 12:47:52 -0400 Subject: [PATCH] Add failures_before_warning to checks (#10969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Sokołowski * agent: add failures_before_warning setting The new setting allows users to specify the number of check failures that have to happen before a service status us updated to be `warning`. This allows for more visibility for detected issues without creating alerts and pinging administrators. Unlike the previous behavior, which caused the service status to not update until it reached the configured `failures_before_critical` setting, now Consul updates the Web UI view with the `warning` state and the output of the service check when `failures_before_warning` is breached. The default value of `FailuresBeforeWarning` is the same as the value of `FailuresBeforeCritical`, which allows for retaining the previous default behavior of not triggering a warning. When `FailuresBeforeWarning` is set to a value higher than that of `FailuresBeforeCritical it has no effect as `FailuresBeforeCritical` takes precedence. Resolves: https://github.com/hashicorp/consul/issues/10680 Signed-off-by: Jakub Sokołowski Co-authored-by: Jakub Sokołowski --- .changelog/10969.txt | 4 + agent/agent.go | 7 +- agent/checks/check.go | 16 +- agent/checks/check_test.go | 191 +++++++++++++++--- agent/checks/grpc_test.go | 4 +- agent/config/builder.go | 7 + agent/config/config.go | 1 + agent/config/runtime.go | 3 + agent/config/runtime_test.go | 56 ++--- .../TestRuntimeConfig_Sanitize.golden | 4 +- agent/config/testdata/full-config.hcl | 23 --- agent/config/testdata/full-config.json | 23 --- agent/structs/check_definition.go | 2 + agent/structs/check_type.go | 5 + agent/structs/service_definition.go | 11 +- api/agent.go | 1 + build-support/scripts/proto-gen.sh | 2 +- proto/pbservice/healthcheck.gen.go | 2 + proto/pbservice/healthcheck.pb.go | 168 ++++++++------- proto/pbservice/healthcheck.proto | 2 + website/content/api-docs/agent/check.mdx | 9 +- website/content/docs/discovery/checks.mdx | 18 +- 22 files changed, 364 insertions(+), 195 deletions(-) create mode 100644 .changelog/10969.txt diff --git a/.changelog/10969.txt b/.changelog/10969.txt new file mode 100644 index 0000000000..c3c9ac33fb --- /dev/null +++ b/.changelog/10969.txt @@ -0,0 +1,4 @@ +```release-note:improvement +checks: add failures_before_warning setting for interval checks. +``` + diff --git a/agent/agent.go b/agent/agent.go index 5b9e4a8b89..1fdea4bc6e 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2459,6 +2459,11 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, maxOutputSize = chkType.OutputMaxSize } + // FailuresBeforeWarning has to default to same value as FailuresBeforeCritical + if chkType.FailuresBeforeWarning == 0 { + chkType.FailuresBeforeWarning = chkType.FailuresBeforeCritical + } + // Get the address of the proxy for this service if it exists // Need its config to know whether we should reroute checks to it var proxy *structs.NodeService @@ -2473,7 +2478,7 @@ func (a *Agent) addCheck(check *structs.HealthCheck, chkType *structs.CheckType, } } - statusHandler := checks.NewStatusHandler(a.State, a.logger, chkType.SuccessBeforePassing, chkType.FailuresBeforeCritical) + statusHandler := checks.NewStatusHandler(a.State, a.logger, chkType.SuccessBeforePassing, chkType.FailuresBeforeWarning, chkType.FailuresBeforeCritical) sid := check.CompoundServiceID() cid := check.CompoundCheckID() diff --git a/agent/checks/check.go b/agent/checks/check.go index 78a98b61e6..687de9e51a 100644 --- a/agent/checks/check.go +++ b/agent/checks/check.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "fmt" - http2 "golang.org/x/net/http2" "io" "io/ioutil" "net" @@ -16,6 +15,8 @@ import ( "syscall" "time" + http2 "golang.org/x/net/http2" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/go-hclog" @@ -907,17 +908,19 @@ type StatusHandler struct { logger hclog.Logger successBeforePassing int successCounter int + failuresBeforeWarning int failuresBeforeCritical int failuresCounter int } // NewStatusHandler set counters values to threshold in order to immediatly update status after first check. -func NewStatusHandler(inner CheckNotifier, logger hclog.Logger, successBeforePassing, failuresBeforeCritical int) *StatusHandler { +func NewStatusHandler(inner CheckNotifier, logger hclog.Logger, successBeforePassing, failuresBeforeWarning, failuresBeforeCritical int) *StatusHandler { return &StatusHandler{ logger: logger, inner: inner, successBeforePassing: successBeforePassing, successCounter: successBeforePassing, + failuresBeforeWarning: failuresBeforeWarning, failuresBeforeCritical: failuresBeforeCritical, failuresCounter: failuresBeforeCritical, } @@ -950,10 +953,17 @@ func (s *StatusHandler) updateCheck(checkID structs.CheckID, status, output stri s.inner.UpdateCheck(checkID, status, output) return } - s.logger.Warn("Check failed but has not reached failure threshold", + // Defaults to same value as failuresBeforeCritical if not set. + if s.failuresCounter >= s.failuresBeforeWarning { + s.logger.Warn("Check is now warning", "check", checkID.String()) + s.inner.UpdateCheck(checkID, api.HealthWarning, output) + return + } + s.logger.Warn("Check failed but has not reached warning/failure threshold", "check", checkID.String(), "status", status, "failure_count", s.failuresCounter, + "warning_threshold", s.failuresBeforeWarning, "failure_threshold", s.failuresBeforeCritical, ) } diff --git a/agent/checks/check_test.go b/agent/checks/check_test.go index 580e2eac16..af97d02389 100644 --- a/agent/checks/check_test.go +++ b/agent/checks/check_test.go @@ -49,7 +49,7 @@ func TestCheckMonitor_Script(t *testing.T) { t.Run(tt.status, func(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckMonitor{ @@ -94,7 +94,7 @@ func TestCheckMonitor_Args(t *testing.T) { t.Run(tt.status, func(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckMonitor{ @@ -128,7 +128,7 @@ func TestCheckMonitor_Timeout(t *testing.T) { // t.Parallel() // timing test. no parallel notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckMonitor{ @@ -163,7 +163,7 @@ func TestCheckMonitor_RandomStagger(t *testing.T) { // t.Parallel() // timing test. no parallel notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) @@ -195,7 +195,7 @@ func TestCheckMonitor_LimitOutput(t *testing.T) { t.Parallel() notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckMonitor{ @@ -354,7 +354,7 @@ func TestCheckHTTP(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) @@ -397,7 +397,7 @@ func TestCheckHTTP_Proxied(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckHTTP{ @@ -433,7 +433,7 @@ func TestCheckHTTP_NotProxied(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckHTTP{ @@ -558,7 +558,7 @@ func TestCheckMaxOutputSize(t *testing.T) { Interval: 2 * time.Millisecond, Logger: logger, OutputMaxSize: maxOutputSize, - StatusHandler: NewStatusHandler(notif, logger, 0, 0), + StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0), } check.Start() @@ -586,7 +586,7 @@ func TestCheckHTTPTimeout(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("bar", nil) @@ -659,7 +659,7 @@ func TestCheckHTTPBody(t *testing.T) { Timeout: timeout, Interval: 2 * time.Millisecond, Logger: logger, - StatusHandler: NewStatusHandler(notif, logger, 0, 0), + StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0), } check.Start() defer check.Stop() @@ -690,7 +690,7 @@ func TestCheckHTTP_disablesKeepAlives(t *testing.T) { HTTP: "http://foo.bar/baz", Interval: 10 * time.Second, Logger: logger, - StatusHandler: NewStatusHandler(notif, logger, 0, 0), + StatusHandler: NewStatusHandler(notif, logger, 0, 0, 0), } check.Start() @@ -725,7 +725,7 @@ func TestCheckHTTP_TLS_SkipVerify(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("skipverify_true", nil) check := &CheckHTTP{ @@ -767,7 +767,7 @@ func TestCheckHTTP_TLS_BadVerify(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("skipverify_false", nil) check := &CheckHTTP{ @@ -819,7 +819,7 @@ func mockTCPServer(network string) net.Listener { func expectTCPStatus(t *testing.T, tcp string, status string) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckTCP{ @@ -846,13 +846,12 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te cid := structs.NewCheckID("foo", nil) notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 2, 3) + statusHandler := NewStatusHandler(notif, logger, 2, 2, 3) // Set the initial status to passing after a single success statusHandler.updateCheck(cid, api.HealthPassing, "bar") - // Status should become critical after 3 failed checks only - statusHandler.updateCheck(cid, api.HealthCritical, "bar") + // Status should still be passing after 1 failed check only statusHandler.updateCheck(cid, api.HealthCritical, "bar") retry.Run(t, func(r *retry.R) { @@ -860,10 +859,19 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te require.Equal(r, api.HealthPassing, notif.State(cid)) }) + // Status should become warning after 2 failed checks only statusHandler.updateCheck(cid, api.HealthCritical, "bar") retry.Run(t, func(r *retry.R) { require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + // Status should become critical after 4 failed checks only + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 3, notif.Updates(cid)) require.Equal(r, api.HealthCritical, notif.State(cid)) }) @@ -871,14 +879,14 @@ func TestStatusHandlerUpdateStatusAfterConsecutiveChecksThresholdIsReached(t *te statusHandler.updateCheck(cid, api.HealthPassing, "bar") retry.Run(t, func(r *retry.R) { - require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, 3, notif.Updates(cid)) require.Equal(r, api.HealthCritical, notif.State(cid)) }) statusHandler.updateCheck(cid, api.HealthPassing, "bar") retry.Run(t, func(r *retry.R) { - require.Equal(r, 3, notif.Updates(cid)) + require.Equal(r, 4, notif.Updates(cid)) require.Equal(r, api.HealthPassing, notif.State(cid)) }) } @@ -888,17 +896,18 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T cid := structs.NewCheckID("foo", nil) notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 2, 3) + statusHandler := NewStatusHandler(notif, logger, 2, 2, 3) // Set the initial status to passing after a single success statusHandler.updateCheck(cid, api.HealthPassing, "bar") - // Status should remain passing after FAIL PASS FAIL FAIL sequence + // Status should remain passing after FAIL PASS FAIL PASS FAIL sequence // Although we have 3 FAILS, they are not consecutive statusHandler.updateCheck(cid, api.HealthCritical, "bar") statusHandler.updateCheck(cid, api.HealthPassing, "bar") statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthPassing, "bar") statusHandler.updateCheck(cid, api.HealthCritical, "bar") retry.Run(t, func(r *retry.R) { @@ -906,11 +915,19 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T require.Equal(r, api.HealthPassing, notif.State(cid)) }) - // Critical after a 3rd consecutive FAIL + // Warning after a 2rd consecutive FAIL statusHandler.updateCheck(cid, api.HealthCritical, "bar") retry.Run(t, func(r *retry.R) { require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + // Critical after a 3rd consecutive FAIL + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 3, notif.Updates(cid)) require.Equal(r, api.HealthCritical, notif.State(cid)) }) @@ -920,19 +937,137 @@ func TestStatusHandlerResetCountersOnNonIdenticalsConsecutiveChecks(t *testing.T statusHandler.updateCheck(cid, api.HealthPassing, "bar") retry.Run(t, func(r *retry.R) { - require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, 3, notif.Updates(cid)) require.Equal(r, api.HealthCritical, notif.State(cid)) }) // Passing after a 2nd consecutive PASS statusHandler.updateCheck(cid, api.HealthPassing, "bar") + retry.Run(t, func(r *retry.R) { + require.Equal(r, 4, notif.Updates(cid)) + require.Equal(r, api.HealthPassing, notif.State(cid)) + }) +} + +func TestStatusHandlerWarningAndCriticalThresholdsTheSameSetsCritical(t *testing.T) { + t.Parallel() + cid := structs.NewCheckID("foo", nil) + notif := mock.NewNotify() + logger := testutil.Logger(t) + statusHandler := NewStatusHandler(notif, logger, 2, 3, 3) + + // Set the initial status to passing after a single success + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + + // Status should remain passing after FAIL FAIL sequence + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 1, notif.Updates(cid)) + require.Equal(r, api.HealthPassing, notif.State(cid)) + }) + + // Critical and not Warning after a 3rd consecutive FAIL + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, api.HealthCritical, notif.State(cid)) + }) + + // Passing after consecutive PASS PASS sequence + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + retry.Run(t, func(r *retry.R) { require.Equal(r, 3, notif.Updates(cid)) require.Equal(r, api.HealthPassing, notif.State(cid)) }) } +func TestStatusHandlerMaintainWarningStatusWhenCheckIsFlapping(t *testing.T) { + t.Parallel() + cid := structs.NewCheckID("foo", nil) + notif := mock.NewNotify() + logger := testutil.Logger(t) + statusHandler := NewStatusHandler(notif, logger, 3, 3, 5) + + // Set the initial status to passing after a single success. + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + + // Status should remain passing after a FAIL FAIL sequence. + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 1, notif.Updates(cid)) + require.Equal(r, api.HealthPassing, notif.State(cid)) + }) + + // Warning after a 3rd consecutive FAIL. + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + retry.Run(t, func(r *retry.R) { + require.Equal(r, 2, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + // Status should remain passing after PASS FAIL FAIL FAIL PASS FAIL FAIL FAIL PASS sequence. + // Although we have 6 FAILS, they are not consecutive. + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + // The status gets updated due to failuresCounter being reset + // but the status itself remains as Warning. + retry.Run(t, func(r *retry.R) { + require.Equal(r, 3, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + // Status doesn'tn change, but the state update is triggered. + retry.Run(t, func(r *retry.R) { + require.Equal(r, 4, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + // Status should change only after 5 consecutive FAIL updates. + statusHandler.updateCheck(cid, api.HealthPassing, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + // The status doesn't change, but a status update is triggered. + retry.Run(t, func(r *retry.R) { + require.Equal(r, 5, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + // The status doesn't change, but a status update is triggered. + retry.Run(t, func(r *retry.R) { + require.Equal(r, 6, notif.Updates(cid)) + require.Equal(r, api.HealthWarning, notif.State(cid)) + }) + + statusHandler.updateCheck(cid, api.HealthCritical, "bar") + + // The FailuresBeforeCritical threshold is finally breached. + retry.Run(t, func(r *retry.R) { + require.Equal(r, 7, notif.Updates(cid)) + require.Equal(r, api.HealthCritical, notif.State(cid)) + }) +} + func TestCheckTCPCritical(t *testing.T) { t.Parallel() var ( @@ -992,7 +1127,7 @@ func TestCheckH2PING(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) tlsCfg := &api.TLSConfig{ InsecureSkipVerify: true, @@ -1044,7 +1179,7 @@ func TestCheckH2PING_TLS_BadVerify(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) tlsCfg := &api.TLSConfig{} tlsClientCfg, err := api.SetupTLSConfig(tlsCfg) @@ -1085,7 +1220,7 @@ func TestCheckH2PINGInvalidListener(t *testing.T) { notif := mock.NewNotify() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) tlsCfg := &api.TLSConfig{ InsecureSkipVerify: true, @@ -1388,7 +1523,7 @@ func TestCheck_Docker(t *testing.T) { notif, upd := mock.NewNotifyChan() logger := testutil.Logger(t) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) id := structs.NewCheckID("chk", nil) check := &CheckDocker{ diff --git a/agent/checks/grpc_test.go b/agent/checks/grpc_test.go index ccdadba9fe..63f5405a64 100644 --- a/agent/checks/grpc_test.go +++ b/agent/checks/grpc_test.go @@ -113,7 +113,7 @@ func TestGRPC_Proxied(t *testing.T) { Output: ioutil.Discard, }) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckGRPC{ @@ -147,7 +147,7 @@ func TestGRPC_NotProxied(t *testing.T) { Output: ioutil.Discard, }) - statusHandler := NewStatusHandler(notif, logger, 0, 0) + statusHandler := NewStatusHandler(notif, logger, 0, 0, 0) cid := structs.NewCheckID("foo", nil) check := &CheckGRPC{ diff --git a/agent/config/builder.go b/agent/config/builder.go index 4b9fabad59..7269da4cd4 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1415,6 +1415,12 @@ func (b *builder) validate(rt RuntimeConfig) error { return fmt.Errorf("service %q: %s", s.Name, err) } } + // Check for errors in the node check definitions + for _, c := range rt.Checks { + if err := c.CheckType().Validate(); err != nil { + return fmt.Errorf("check %q: %w", c.Name, err) + } + } // Validate the given Connect CA provider config validCAProviders := map[string]bool{ @@ -1584,6 +1590,7 @@ func (b *builder) checkVal(v *CheckDefinition) *structs.CheckDefinition { TTL: b.durationVal(fmt.Sprintf("check[%s].ttl", id), v.TTL), SuccessBeforePassing: intVal(v.SuccessBeforePassing), FailuresBeforeCritical: intVal(v.FailuresBeforeCritical), + FailuresBeforeWarning: intValWithDefault(v.FailuresBeforeWarning, intVal(v.FailuresBeforeCritical)), H2PING: stringVal(v.H2PING), DeregisterCriticalServiceAfter: b.durationVal(fmt.Sprintf("check[%s].deregister_critical_service_after", id), v.DeregisterCriticalServiceAfter), OutputMaxSize: intValWithDefault(v.OutputMaxSize, checks.DefaultBufSize), diff --git a/agent/config/config.go b/agent/config/config.go index e20bf1580f..acb54d7a88 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -424,6 +424,7 @@ type CheckDefinition struct { TTL *string `mapstructure:"ttl"` H2PING *string `mapstructure:"h2ping"` SuccessBeforePassing *int `mapstructure:"success_before_passing"` + FailuresBeforeWarning *int `mapstructure:"failures_before_warning"` FailuresBeforeCritical *int `mapstructure:"failures_before_critical"` DeregisterCriticalServiceAfter *string `mapstructure:"deregister_critical_service_after" alias:"deregistercriticalserviceafter"` diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 1e9ebf1e72..78aca0241b 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -434,6 +434,9 @@ type RuntimeConfig struct { // tls_skip_verify = (true|false) // timeout = "duration" // ttl = "duration" + // success_before_passing = int + // failures_before_warning = int + // failures_before_critical = int // deregister_critical_service_after = "duration" // }, // ... diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index cbe1469957..2db98314b5 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -2330,17 +2330,17 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { `-data-dir=` + dataDir, }, json: []string{ - `{ "check": { "name": "a", "args": ["/bin/true"] } }`, - `{ "check": { "name": "b", "args": ["/bin/false"] } }`, + `{ "check": { "name": "a", "args": ["/bin/true"], "interval": "1s" } }`, + `{ "check": { "name": "b", "args": ["/bin/false"], "interval": "1s" } }`, }, hcl: []string{ - `check = { name = "a" args = ["/bin/true"] }`, - `check = { name = "b" args = ["/bin/false"] }`, + `check = { name = "a" args = ["/bin/true"] interval = "1s"}`, + `check = { name = "b" args = ["/bin/false"] interval = "1s" }`, }, expected: func(rt *RuntimeConfig) { rt.Checks = []*structs.CheckDefinition{ - {Name: "a", ScriptArgs: []string{"/bin/true"}, OutputMaxSize: checks.DefaultBufSize}, - {Name: "b", ScriptArgs: []string{"/bin/false"}, OutputMaxSize: checks.DefaultBufSize}, + {Name: "a", ScriptArgs: []string{"/bin/true"}, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second}, + {Name: "b", ScriptArgs: []string{"/bin/false"}, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second}, } rt.DataDir = dataDir }, @@ -2351,14 +2351,14 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { `-data-dir=` + dataDir, }, json: []string{ - `{ "check": { "name": "a", "grpc": "localhost:12345/foo", "grpc_use_tls": true } }`, + `{ "check": { "name": "a", "grpc": "localhost:12345/foo", "grpc_use_tls": true, "interval": "1s" } }`, }, hcl: []string{ - `check = { name = "a" grpc = "localhost:12345/foo", grpc_use_tls = true }`, + `check = { name = "a" grpc = "localhost:12345/foo", grpc_use_tls = true interval = "1s" }`, }, expected: func(rt *RuntimeConfig) { rt.Checks = []*structs.CheckDefinition{ - {Name: "a", GRPC: "localhost:12345/foo", GRPCUseTLS: true, OutputMaxSize: checks.DefaultBufSize}, + {Name: "a", GRPC: "localhost:12345/foo", GRPCUseTLS: true, OutputMaxSize: checks.DefaultBufSize, Interval: time.Second}, } rt.DataDir = dataDir }, @@ -2478,7 +2478,8 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { "name": "y", "DockerContainerID": "z", "DeregisterCriticalServiceAfter": "10s", - "ScriptArgs": ["a", "b"] + "ScriptArgs": ["a", "b"], + "Interval": "2s" } } }`, @@ -2500,6 +2501,7 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { DockerContainerID = "z" DeregisterCriticalServiceAfter = "10s" ScriptArgs = ["a", "b"] + Interval = "2s" } }`, }, @@ -2517,12 +2519,13 @@ func TestLoad_IntegrationWithFlags(t *testing.T) { EnableTagOverride: true, Checks: []*structs.CheckType{ { - CheckID: types.CheckID("x"), + CheckID: "x", Name: "y", DockerContainerID: "z", DeregisterCriticalServiceAfter: 10 * time.Second, ScriptArgs: []string{"a", "b"}, OutputMaxSize: checks.DefaultBufSize, + Interval: 2 * time.Second, }, }, Weights: &structs.Weights{ @@ -5299,7 +5302,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "bdeb5f6a", TLSSkipVerify: true, Timeout: 1813 * time.Second, - TTL: 21743 * time.Second, DeregisterCriticalServiceAfter: 14232 * time.Second, }, { @@ -5326,7 +5328,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "6adc3bfb", TLSSkipVerify: true, Timeout: 18506 * time.Second, - TTL: 31006 * time.Second, DeregisterCriticalServiceAfter: 2366 * time.Second, }, { @@ -5353,7 +5354,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "7BdnzBYk", TLSSkipVerify: true, Timeout: 5954 * time.Second, - TTL: 30044 * time.Second, DeregisterCriticalServiceAfter: 13209 * time.Second, }, }, @@ -5559,7 +5559,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "4f191d4F", TLSSkipVerify: true, Timeout: 38333 * time.Second, - TTL: 57201 * time.Second, DeregisterCriticalServiceAfter: 44214 * time.Second, }, }, @@ -5611,30 +5610,14 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "f43ouY7a", TLSSkipVerify: true, Timeout: 34738 * time.Second, - TTL: 22773 * time.Second, DeregisterCriticalServiceAfter: 84282 * time.Second, }, &structs.CheckType{ - CheckID: "UHsDeLxG", - Name: "PQSaPWlT", - Notes: "jKChDOdl", - Status: "5qFz6OZn", - ScriptArgs: []string{"NMtYWlT9", "vj74JXsm"}, - HTTP: "1LBDJhw4", - Header: map[string][]string{ - "cXPmnv1M": {"imDqfaBx", "NFxZ1bQe"}, - "vr7wY7CS": {"EtCoNPPL", "9vAarJ5s"}, - }, - Method: "wzByP903", - Body: "4I8ucZgZ", + CheckID: "UHsDeLxG", + Name: "PQSaPWlT", + Notes: "jKChDOdl", + Status: "5qFz6OZn", OutputMaxSize: checks.DefaultBufSize, - TCP: "2exjZIGE", - H2PING: "jTDuR1DC", - Interval: 5656 * time.Second, - DockerContainerID: "5tDBWpfA", - Shell: "rlTpLM8s", - TLSServerName: "sOv5WTtp", - TLSSkipVerify: true, Timeout: 4868 * time.Second, TTL: 11222 * time.Second, DeregisterCriticalServiceAfter: 68482 * time.Second, @@ -5770,7 +5753,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "axw5QPL5", TLSSkipVerify: true, Timeout: 18913 * time.Second, - TTL: 44743 * time.Second, DeregisterCriticalServiceAfter: 8482 * time.Second, }, &structs.CheckType{ @@ -5795,7 +5777,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "7uwWOnUS", TLSSkipVerify: true, Timeout: 38282 * time.Second, - TTL: 1181 * time.Second, DeregisterCriticalServiceAfter: 4992 * time.Second, }, &structs.CheckType{ @@ -5820,7 +5801,6 @@ func TestLoad_FullConfig(t *testing.T) { TLSServerName: "ECSHk8WF", TLSSkipVerify: true, Timeout: 38483 * time.Second, - TTL: 10943 * time.Second, DeregisterCriticalServiceAfter: 68787 * time.Second, }, }, diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index cab09932ab..020d12c067 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -94,6 +94,7 @@ "DeregisterCriticalServiceAfter": "0s", "DockerContainerID": "", "EnterpriseMeta": {}, + "FailuresBeforeWarning": 0, "FailuresBeforeCritical": 0, "GRPC": "", "GRPCUseTLS": false, @@ -295,6 +296,7 @@ "CheckID": "", "DeregisterCriticalServiceAfter": "0s", "DockerContainerID": "", + "FailuresBeforeWarning": 0, "FailuresBeforeCritical": 0, "GRPC": "", "GRPCUseTLS": false, @@ -416,4 +418,4 @@ "Watches": [], "XDSAddrs": [], "XDSPort": 0 -} \ No newline at end of file +} diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index 2f387ee499..1c840d192d 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -117,7 +117,6 @@ check = { tls_server_name = "7BdnzBYk" tls_skip_verify = true timeout = "5954s" - ttl = "30044s" deregister_critical_service_after = "13209s" }, checks = [ @@ -145,7 +144,6 @@ checks = [ tls_server_name = "bdeb5f6a" tls_skip_verify = true timeout = "1813s" - ttl = "21743s" deregister_critical_service_after = "14232s" }, { @@ -172,7 +170,6 @@ checks = [ tls_server_name = "6adc3bfb" tls_skip_verify = true timeout = "18506s" - ttl = "31006s" deregister_critical_service_after = "2366s" } ] @@ -389,7 +386,6 @@ service = { tls_server_name = "ECSHk8WF" tls_skip_verify = true timeout = "38483s" - ttl = "10943s" deregister_critical_service_after = "68787s" } checks = [ @@ -415,7 +411,6 @@ service = { tls_server_name = "axw5QPL5" tls_skip_verify = true timeout = "18913s" - ttl = "44743s" deregister_critical_service_after = "8482s" }, { @@ -440,7 +435,6 @@ service = { tls_server_name = "7uwWOnUS" tls_skip_verify = true timeout = "38282s" - ttl = "1181s" deregister_critical_service_after = "4992s" } ] @@ -479,7 +473,6 @@ services = [ tls_server_name = "4f191d4F" tls_skip_verify = true timeout = "38333s" - ttl = "57201s" deregister_critical_service_after = "44214s" } connect { @@ -521,7 +514,6 @@ services = [ tls_server_name = "f43ouY7a" tls_skip_verify = true timeout = "34738s" - ttl = "22773s" deregister_critical_service_after = "84282s" }, { @@ -529,22 +521,7 @@ services = [ name = "PQSaPWlT" notes = "jKChDOdl" status = "5qFz6OZn" - args = ["NMtYWlT9", "vj74JXsm"] - http = "1LBDJhw4" - header = { - "cXPmnv1M" = [ "imDqfaBx", "NFxZ1bQe" ], - "vr7wY7CS" = [ "EtCoNPPL", "9vAarJ5s" ] - } - method = "wzByP903" - body = "4I8ucZgZ" - tcp = "2exjZIGE" - h2ping = "jTDuR1DC" - interval = "5656s" output_max_size = 4096 - docker_container_id = "5tDBWpfA" - shell = "rlTpLM8s" - tls_server_name = "sOv5WTtp" - tls_skip_verify = true timeout = "4868s" ttl = "11222s" deregister_critical_service_after = "68482s" diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 70369860fe..7e28cc0c4f 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -118,7 +118,6 @@ "tls_server_name": "7BdnzBYk", "tls_skip_verify": true, "timeout": "5954s", - "ttl": "30044s", "deregister_critical_service_after": "13209s" }, "checks": [ @@ -146,7 +145,6 @@ "tls_server_name": "bdeb5f6a", "tls_skip_verify": true, "timeout": "1813s", - "ttl": "21743s", "deregister_critical_service_after": "14232s" }, { @@ -173,7 +171,6 @@ "tls_server_name": "6adc3bfb", "tls_skip_verify": true, "timeout": "18506s", - "ttl": "31006s", "deregister_critical_service_after": "2366s" } ], @@ -386,7 +383,6 @@ "tls_server_name": "ECSHk8WF", "tls_skip_verify": true, "timeout": "38483s", - "ttl": "10943s", "deregister_critical_service_after": "68787s" }, "checks": [ @@ -412,7 +408,6 @@ "tls_server_name": "axw5QPL5", "tls_skip_verify": true, "timeout": "18913s", - "ttl": "44743s", "deregister_critical_service_after": "8482s" }, { @@ -437,7 +432,6 @@ "tls_server_name": "7uwWOnUS", "tls_skip_verify": true, "timeout": "38282s", - "ttl": "1181s", "deregister_critical_service_after": "4992s" } ], @@ -476,7 +470,6 @@ "tls_server_name": "4f191d4F", "tls_skip_verify": true, "timeout": "38333s", - "ttl": "57201s", "deregister_critical_service_after": "44214s" }, "connect": { @@ -518,7 +511,6 @@ "tls_server_name": "f43ouY7a", "tls_skip_verify": true, "timeout": "34738s", - "ttl": "22773s", "deregister_critical_service_after": "84282s" }, { @@ -526,22 +518,7 @@ "name": "PQSaPWlT", "notes": "jKChDOdl", "status": "5qFz6OZn", - "args": ["NMtYWlT9", "vj74JXsm"], - "http": "1LBDJhw4", - "header": { - "cXPmnv1M": [ "imDqfaBx", "NFxZ1bQe" ], - "vr7wY7CS": [ "EtCoNPPL", "9vAarJ5s" ] - }, - "method": "wzByP903", - "body": "4I8ucZgZ", - "tcp": "2exjZIGE", - "h2ping": "jTDuR1DC", - "interval": "5656s", "output_max_size": 4096, - "docker_container_id": "5tDBWpfA", - "shell": "rlTpLM8s", - "tls_server_name": "sOv5WTtp", - "tls_skip_verify": true, "timeout": "4868s", "ttl": "11222s", "deregister_critical_service_after": "68482s" diff --git a/agent/structs/check_definition.go b/agent/structs/check_definition.go index cfac67d13c..5ac0d60444 100644 --- a/agent/structs/check_definition.go +++ b/agent/structs/check_definition.go @@ -41,6 +41,7 @@ type CheckDefinition struct { Timeout time.Duration TTL time.Duration SuccessBeforePassing int + FailuresBeforeWarning int FailuresBeforeCritical int DeregisterCriticalServiceAfter time.Duration OutputMaxSize int @@ -196,6 +197,7 @@ func (c *CheckDefinition) CheckType() *CheckType { Timeout: c.Timeout, TTL: c.TTL, SuccessBeforePassing: c.SuccessBeforePassing, + FailuresBeforeWarning: c.FailuresBeforeWarning, FailuresBeforeCritical: c.FailuresBeforeCritical, DeregisterCriticalServiceAfter: c.DeregisterCriticalServiceAfter, } diff --git a/agent/structs/check_type.go b/agent/structs/check_type.go index 372ab57491..50c9714b39 100644 --- a/agent/structs/check_type.go +++ b/agent/structs/check_type.go @@ -49,6 +49,7 @@ type CheckType struct { Timeout time.Duration TTL time.Duration SuccessBeforePassing int + FailuresBeforeWarning int FailuresBeforeCritical int // Definition fields used when exposing checks through a proxy @@ -182,6 +183,10 @@ func (c *CheckType) Validate() error { if c.OutputMaxSize < 0 { return fmt.Errorf("MaxOutputMaxSize must be positive") } + if c.FailuresBeforeWarning > c.FailuresBeforeCritical { + return fmt.Errorf("FailuresBeforeWarning can't be higher than FailuresBeforeCritical") + } + return nil } diff --git a/agent/structs/service_definition.go b/agent/structs/service_definition.go index d6e2aaa75c..5141cbd96b 100644 --- a/agent/structs/service_definition.go +++ b/agent/structs/service_definition.go @@ -1,8 +1,11 @@ package structs import ( - "github.com/hashicorp/consul/lib" + "fmt" + "github.com/hashicorp/go-multierror" + + "github.com/hashicorp/consul/lib" ) // ServiceDefinition is used to JSON decode the Service definitions. For @@ -123,7 +126,11 @@ func (s *ServiceDefinition) Validate() error { if err := s.NodeService().Validate(); err != nil { result = multierror.Append(result, err) } - + for _, c := range s.Checks { + if err := c.Validate(); err != nil { + return fmt.Errorf("check %q: %s", c.Name, err) + } + } return result } diff --git a/api/agent.go b/api/agent.go index 6c6343048a..6b81367818 100644 --- a/api/agent.go +++ b/api/agent.go @@ -330,6 +330,7 @@ type AgentServiceCheck struct { AliasNode string `json:",omitempty"` AliasService string `json:",omitempty"` SuccessBeforePassing int `json:",omitempty"` + FailuresBeforeWarning int `json:",omitempty"` FailuresBeforeCritical int `json:",omitempty"` // In Consul 0.7 and later, checks that are associated with a service diff --git a/build-support/scripts/proto-gen.sh b/build-support/scripts/proto-gen.sh index 28c5c67f87..82230bb745 100755 --- a/build-support/scripts/proto-gen.sh +++ b/build-support/scripts/proto-gen.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})" pushd $(dirname ${BASH_SOURCE[0]}) > /dev/null SCRIPT_DIR=$(pwd) diff --git a/proto/pbservice/healthcheck.gen.go b/proto/pbservice/healthcheck.gen.go index ee7f142e7c..d195c84d7c 100644 --- a/proto/pbservice/healthcheck.gen.go +++ b/proto/pbservice/healthcheck.gen.go @@ -30,6 +30,7 @@ func CheckTypeToStructs(s CheckType) structs.CheckType { t.TTL = s.TTL t.SuccessBeforePassing = int(s.SuccessBeforePassing) t.FailuresBeforeCritical = int(s.FailuresBeforeCritical) + t.FailuresBeforeWarning = int(s.FailuresBeforeWarning) t.ProxyHTTP = s.ProxyHTTP t.ProxyGRPC = s.ProxyGRPC t.DeregisterCriticalServiceAfter = s.DeregisterCriticalServiceAfter @@ -62,6 +63,7 @@ func NewCheckTypeFromStructs(t structs.CheckType) CheckType { s.TTL = t.TTL s.SuccessBeforePassing = int32(t.SuccessBeforePassing) s.FailuresBeforeCritical = int32(t.FailuresBeforeCritical) + s.FailuresBeforeWarning = int32(t.FailuresBeforeWarning) s.ProxyHTTP = t.ProxyHTTP s.ProxyGRPC = t.ProxyGRPC s.DeregisterCriticalServiceAfter = t.DeregisterCriticalServiceAfter diff --git a/proto/pbservice/healthcheck.pb.go b/proto/pbservice/healthcheck.pb.go index a56d3c4a32..f1e53fb4fa 100644 --- a/proto/pbservice/healthcheck.pb.go +++ b/proto/pbservice/healthcheck.pb.go @@ -232,6 +232,8 @@ type CheckType struct { // mog: func-to=int func-from=int32 SuccessBeforePassing int32 `protobuf:"varint,21,opt,name=SuccessBeforePassing,proto3" json:"SuccessBeforePassing,omitempty"` // mog: func-to=int func-from=int32 + FailuresBeforeWarning int32 `protobuf:"varint,29,opt,name=FailuresBeforeWarning,proto3" json:"FailuresBeforeWarning,omitempty"` + // mog: func-to=int func-from=int32 FailuresBeforeCritical int32 `protobuf:"varint,22,opt,name=FailuresBeforeCritical,proto3" json:"FailuresBeforeCritical,omitempty"` // Definition fields used when exposing checks through a proxy ProxyHTTP string `protobuf:"bytes,23,opt,name=ProxyHTTP,proto3" json:"ProxyHTTP,omitempty"` @@ -289,74 +291,75 @@ func init() { func init() { proto.RegisterFile("proto/pbservice/healthcheck.proto", fileDescriptor_8a6f7448747c9fbe) } var fileDescriptor_8a6f7448747c9fbe = []byte{ - // 1062 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0x41, 0x4f, 0xe3, 0x46, - 0x14, 0x8e, 0x09, 0x24, 0xf1, 0x64, 0x61, 0x61, 0x96, 0xa5, 0xb3, 0xec, 0xca, 0xa4, 0x74, 0x0f, - 0x54, 0xa5, 0x8e, 0x44, 0xd5, 0xaa, 0xad, 0xd4, 0x56, 0x84, 0xb0, 0x90, 0x0a, 0x68, 0xea, 0xb8, - 0x7b, 0xe8, 0xcd, 0x38, 0x93, 0xc4, 0xc2, 0xf1, 0x58, 0xe3, 0x31, 0x22, 0xfd, 0x15, 0x7b, 0xdc, - 0xff, 0xd0, 0x3f, 0xc2, 0x91, 0x63, 0xa5, 0x4a, 0xb4, 0x85, 0x7f, 0xd1, 0x53, 0x35, 0x6f, 0xec, - 0xe0, 0x6c, 0xbc, 0x25, 0x5d, 0x6d, 0x4f, 0xcc, 0xfb, 0xde, 0x7b, 0x33, 0x9e, 0x79, 0xdf, 0xf7, - 0x05, 0xf4, 0x61, 0xc8, 0x99, 0x60, 0xf5, 0xf0, 0x34, 0xa2, 0xfc, 0xdc, 0x73, 0x69, 0x7d, 0x40, - 0x1d, 0x5f, 0x0c, 0xdc, 0x01, 0x75, 0xcf, 0x4c, 0xc8, 0x61, 0x7d, 0x9c, 0x5c, 0x37, 0xfa, 0x8c, - 0xf5, 0x7d, 0x5a, 0x87, 0xc4, 0x69, 0xdc, 0xab, 0x77, 0x63, 0xee, 0x08, 0x8f, 0x05, 0xaa, 0x74, - 0xfd, 0x69, 0xba, 0x9b, 0xcb, 0x86, 0x43, 0x16, 0xd4, 0xd5, 0x9f, 0x24, 0xb9, 0xda, 0x67, 0x7d, - 0xa6, 0x0a, 0xe4, 0x4a, 0xa1, 0x9b, 0xbf, 0xcf, 0xa3, 0xea, 0x21, 0x9c, 0xb9, 0x27, 0xcf, 0xc4, - 0x18, 0xcd, 0x9f, 0xb0, 0x2e, 0x25, 0x5a, 0x4d, 0xdb, 0xd2, 0x2d, 0x58, 0xe3, 0x03, 0x54, 0x86, - 0x64, 0xab, 0x49, 0xe6, 0x24, 0xdc, 0xf8, 0xf4, 0xef, 0xeb, 0x8d, 0x8f, 0xfb, 0x9e, 0x18, 0xc4, - 0xa7, 0xa6, 0xcb, 0x86, 0xf5, 0x81, 0x13, 0x0d, 0x3c, 0x97, 0xf1, 0xb0, 0xee, 0xb2, 0x20, 0x8a, - 0xfd, 0xba, 0x18, 0x85, 0x34, 0x32, 0x93, 0x26, 0x2b, 0xed, 0x86, 0xcd, 0x9d, 0x21, 0x25, 0xc5, - 0x64, 0x73, 0x67, 0x48, 0xf1, 0x1a, 0x2a, 0x75, 0x84, 0x23, 0xe2, 0x88, 0xcc, 0x03, 0x9a, 0x44, - 0x78, 0x15, 0x2d, 0x9c, 0x30, 0x41, 0x23, 0xb2, 0x00, 0xb0, 0x0a, 0x64, 0xf5, 0x0f, 0xb1, 0x08, - 0x63, 0x41, 0x4a, 0xaa, 0x5a, 0x45, 0xf8, 0x19, 0xd2, 0x3b, 0xea, 0x91, 0x5a, 0x4d, 0x52, 0x86, - 0xd4, 0x1d, 0x80, 0x6b, 0xa8, 0x9a, 0x04, 0x70, 0x7c, 0x05, 0xf2, 0x59, 0x28, 0x53, 0x61, 0x3b, - 0xfd, 0x88, 0xe8, 0xb5, 0x62, 0xa6, 0x42, 0x42, 0xf2, 0xdb, 0xed, 0x51, 0x48, 0xc9, 0x03, 0xf5, - 0xed, 0x72, 0x8d, 0x5f, 0x20, 0xd4, 0xa4, 0x3d, 0x2f, 0xf0, 0xe4, 0x0c, 0x08, 0xaa, 0x69, 0x5b, - 0xd5, 0x9d, 0x9a, 0x39, 0x9e, 0x97, 0x99, 0x79, 0xd8, 0xbb, 0xba, 0xc6, 0xfc, 0xe5, 0xf5, 0x46, - 0xc1, 0xca, 0x74, 0xe2, 0xaf, 0x90, 0x6e, 0x39, 0x3d, 0xd1, 0x0a, 0xba, 0xf4, 0x82, 0x54, 0x61, - 0x9b, 0x15, 0x33, 0x19, 0xde, 0x38, 0xd1, 0xa8, 0xc8, 0xbe, 0xab, 0xeb, 0x0d, 0xcd, 0xba, 0xab, - 0xc6, 0x4d, 0xb4, 0xb4, 0x1f, 0x08, 0xca, 0x43, 0xee, 0x45, 0xf4, 0x98, 0x0a, 0x87, 0x2c, 0x42, - 0xff, 0x5a, 0xda, 0x3f, 0x99, 0x4d, 0x0e, 0x7f, 0xa3, 0x47, 0x5e, 0x7f, 0xff, 0x22, 0x64, 0x11, - 0xed, 0xb6, 0x19, 0x17, 0x64, 0xa9, 0xa6, 0x6d, 0x2d, 0x58, 0x59, 0x08, 0xaf, 0xa3, 0x4a, 0x4b, - 0xf6, 0x9c, 0x3b, 0x3e, 0x79, 0x08, 0x4f, 0x30, 0x8e, 0x31, 0x41, 0x65, 0xdb, 0x1b, 0x52, 0x16, - 0x0b, 0xb2, 0x0c, 0xa9, 0x34, 0xdc, 0xfc, 0x08, 0xc8, 0xd5, 0xa5, 0xfc, 0xa5, 0xe3, 0xc7, 0x54, - 0xce, 0x14, 0x16, 0x44, 0x83, 0xf7, 0x55, 0xc1, 0xe6, 0xab, 0x32, 0x7a, 0x9c, 0xfb, 0x52, 0xf2, - 0xcd, 0x0f, 0x6d, 0xbb, 0x9d, 0x92, 0x51, 0xae, 0xf1, 0x73, 0xb4, 0x68, 0x1f, 0x75, 0xe4, 0x64, - 0x28, 0x87, 0x69, 0x3e, 0x82, 0xe4, 0x24, 0x98, 0x56, 0x9d, 0x79, 0xe1, 0x4b, 0xca, 0xbd, 0xde, - 0x08, 0x88, 0x5b, 0xb1, 0x26, 0x41, 0xfc, 0x3d, 0x2a, 0xa9, 0xcf, 0x23, 0xc5, 0x5a, 0x71, 0xab, - 0xba, 0xb3, 0x7d, 0xdf, 0xec, 0x4c, 0x55, 0xbe, 0x1f, 0x08, 0x3e, 0x4a, 0x9e, 0x32, 0xd9, 0x41, - 0x32, 0xf3, 0x98, 0x8a, 0x01, 0xeb, 0xa6, 0x3c, 0x56, 0x91, 0xbc, 0x43, 0x83, 0x75, 0x47, 0x04, - 0xab, 0x3b, 0xc8, 0x35, 0x5e, 0x46, 0x45, 0x7b, 0xaf, 0x9d, 0x30, 0x5b, 0x2e, 0xf1, 0x77, 0x99, - 0xe7, 0x2d, 0xc1, 0x00, 0x9f, 0x98, 0x4a, 0xec, 0x66, 0x2a, 0x76, 0xb3, 0x99, 0x88, 0x5d, 0x11, - 0xe1, 0xf5, 0x1f, 0x1b, 0x5a, 0x66, 0x06, 0xcf, 0xd1, 0xa2, 0x92, 0xc2, 0xb1, 0x73, 0xd1, 0xf1, - 0x7e, 0xa1, 0x44, 0xaf, 0x69, 0x5b, 0x8b, 0xd6, 0x24, 0x88, 0xbf, 0xb9, 0x9b, 0x54, 0x79, 0xf6, - 0x53, 0xd2, 0x1e, 0x7c, 0x86, 0x8c, 0x26, 0xe5, 0xb4, 0xef, 0x45, 0x82, 0xf2, 0x3d, 0xee, 0x09, - 0xcf, 0x75, 0xfc, 0x44, 0x24, 0xbb, 0x3d, 0x41, 0x39, 0x48, 0x6b, 0xc6, 0x5d, 0xef, 0xd9, 0x0a, - 0x1b, 0x08, 0x75, 0x5c, 0xee, 0x85, 0x62, 0x97, 0xf7, 0x23, 0x82, 0x80, 0x31, 0x19, 0x04, 0x6f, - 0xa3, 0x95, 0x26, 0x73, 0xcf, 0x28, 0xdf, 0x63, 0x81, 0x70, 0xbc, 0x80, 0xf2, 0x56, 0x13, 0xc4, - 0xa3, 0x5b, 0xd3, 0x09, 0x49, 0xbd, 0xce, 0x80, 0xfa, 0x7e, 0xa2, 0x5f, 0x15, 0xc8, 0xa1, 0x1d, - 0xee, 0xb4, 0x5b, 0x27, 0x07, 0x64, 0x55, 0x0d, 0x4d, 0x45, 0x72, 0x68, 0x07, 0x56, 0x7b, 0x0f, - 0xb4, 0xa4, 0x5b, 0xb0, 0x96, 0xdf, 0x23, 0xff, 0xfe, 0x14, 0x51, 0xfb, 0xa8, 0x03, 0x12, 0xa9, - 0x58, 0x19, 0x44, 0x5a, 0xd0, 0xae, 0xef, 0x39, 0x11, 0xd8, 0xa7, 0x92, 0xc8, 0x1d, 0x80, 0x37, - 0xd1, 0x03, 0x08, 0x92, 0x2b, 0x26, 0x42, 0x99, 0xc0, 0xf0, 0xe7, 0xa8, 0x68, 0xdb, 0x47, 0x64, - 0x65, 0xf6, 0x37, 0x94, 0xf5, 0xeb, 0x3f, 0xa6, 0x22, 0x03, 0x5a, 0x4a, 0x72, 0x9d, 0xd1, 0x51, - 0xa2, 0x19, 0xb9, 0xc4, 0xdb, 0x68, 0xe1, 0x1c, 0x64, 0x37, 0x97, 0x58, 0xc3, 0x04, 0xcb, 0x53, - 0x75, 0x5a, 0xaa, 0xe8, 0xeb, 0xb9, 0x2f, 0xb5, 0xcd, 0x5f, 0x75, 0xa4, 0x03, 0xf5, 0xc1, 0xe6, - 0x32, 0xfe, 0xaf, 0xbd, 0x17, 0xff, 0x9f, 0xcb, 0xf5, 0xff, 0x62, 0xbe, 0xff, 0xcf, 0x67, 0xfd, - 0x7f, 0x92, 0x14, 0x0b, 0x53, 0xa4, 0x48, 0x1d, 0xa3, 0x94, 0x71, 0x8c, 0x6f, 0xc7, 0x2a, 0x5f, - 0x05, 0x95, 0x67, 0x1d, 0x7a, 0x7c, 0xc9, 0x99, 0x94, 0x5d, 0xce, 0x55, 0xf6, 0xfa, 0xb4, 0xb2, - 0x2b, 0xf9, 0xca, 0xd6, 0xdf, 0x45, 0xd9, 0x13, 0xbc, 0x42, 0xf7, 0xf1, 0xaa, 0x9a, 0xc3, 0xab, - 0x5c, 0xa5, 0x3c, 0xb8, 0x57, 0x29, 0x8b, 0xf9, 0x4a, 0x79, 0x96, 0xab, 0x94, 0xa5, 0xb7, 0x2a, - 0xe5, 0xe1, 0x94, 0x52, 0xa6, 0x2c, 0xfc, 0xe9, 0x4c, 0x16, 0xbe, 0x9c, 0x67, 0xe1, 0x19, 0x47, - 0x5b, 0x79, 0x07, 0x47, 0x4b, 0x24, 0x87, 0xff, 0x9b, 0xe4, 0xf0, 0x0e, 0x5a, 0xed, 0xc4, 0xae, - 0x4b, 0xa3, 0xa8, 0x41, 0x7b, 0x8c, 0xd3, 0xb6, 0x13, 0x45, 0x5e, 0xd0, 0x27, 0x8f, 0xe1, 0x87, - 0x33, 0x37, 0x87, 0xbf, 0x40, 0x6b, 0x2f, 0x1c, 0xcf, 0x8f, 0x39, 0x4d, 0x12, 0xa9, 0xeb, 0x91, - 0x35, 0xe8, 0x7a, 0x4b, 0x56, 0xce, 0xbf, 0xcd, 0xd9, 0xc5, 0x08, 0x78, 0xfd, 0x81, 0x9a, 0xff, - 0x18, 0x18, 0x67, 0x61, 0x08, 0x24, 0x93, 0x85, 0x49, 0xdc, 0x6f, 0xd8, 0x8f, 0xde, 0x9f, 0x61, - 0x4f, 0xfd, 0x04, 0x3d, 0x81, 0x7b, 0x4d, 0x82, 0xff, 0x83, 0x5b, 0x35, 0x8e, 0x2f, 0xff, 0x32, - 0x0a, 0x97, 0x37, 0x86, 0x76, 0x75, 0x63, 0x68, 0x7f, 0xde, 0x18, 0xda, 0xab, 0x5b, 0xa3, 0xf0, - 0xfa, 0xd6, 0x28, 0x5c, 0xdd, 0x1a, 0x85, 0xdf, 0x6e, 0x8d, 0xc2, 0xcf, 0x9f, 0xfc, 0x9b, 0x59, - 0xbd, 0xf1, 0x2f, 0xf8, 0x69, 0x09, 0x80, 0xcf, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0xce, 0x2e, - 0x2d, 0x41, 0x9c, 0x0b, 0x00, 0x00, + // 1076 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x4f, 0xe3, 0x46, + 0x14, 0x8d, 0x09, 0x24, 0xf1, 0x04, 0x58, 0x98, 0x05, 0x3a, 0xcb, 0x6e, 0x4d, 0x4a, 0xf7, 0x81, + 0xaa, 0x34, 0x91, 0xe8, 0x87, 0xda, 0x4a, 0x6d, 0x45, 0x08, 0x0b, 0xa9, 0x80, 0xa6, 0x4e, 0xba, + 0x95, 0xfa, 0x66, 0x9c, 0x49, 0x62, 0xe1, 0x78, 0xac, 0xf1, 0x18, 0x91, 0xfe, 0x8a, 0x7d, 0xdc, + 0x9f, 0xc4, 0x23, 0x8f, 0x95, 0x2a, 0xd1, 0x2e, 0xfc, 0x8b, 0x3e, 0x55, 0x73, 0xc7, 0x0e, 0xf6, + 0xc6, 0x5b, 0xd2, 0xd5, 0xee, 0x13, 0x73, 0xef, 0xb9, 0x77, 0xc6, 0x33, 0xf7, 0x9c, 0x13, 0xd0, + 0x47, 0x3e, 0x67, 0x82, 0xd5, 0xfc, 0xd3, 0x80, 0xf2, 0x73, 0xc7, 0xa6, 0xb5, 0x01, 0xb5, 0x5c, + 0x31, 0xb0, 0x07, 0xd4, 0x3e, 0xab, 0x02, 0x86, 0xf5, 0x31, 0xb8, 0x6e, 0xf4, 0x19, 0xeb, 0xbb, + 0xb4, 0x06, 0xc0, 0x69, 0xd8, 0xab, 0x75, 0x43, 0x6e, 0x09, 0x87, 0x79, 0xaa, 0x74, 0xfd, 0x71, + 0xbc, 0x9b, 0xcd, 0x86, 0x43, 0xe6, 0xd5, 0xd4, 0x9f, 0x08, 0x5c, 0xe9, 0xb3, 0x3e, 0x53, 0x05, + 0x72, 0xa5, 0xb2, 0x9b, 0x7f, 0xce, 0xa2, 0xf2, 0x21, 0x9c, 0xb9, 0x27, 0xcf, 0xc4, 0x18, 0xcd, + 0x9e, 0xb0, 0x2e, 0x25, 0x5a, 0x45, 0xdb, 0xd2, 0x4d, 0x58, 0xe3, 0x03, 0x54, 0x04, 0xb0, 0xd9, + 0x20, 0x33, 0x32, 0x5d, 0xff, 0xec, 0x9f, 0xeb, 0x8d, 0x4f, 0xfa, 0x8e, 0x18, 0x84, 0xa7, 0x55, + 0x9b, 0x0d, 0x6b, 0x03, 0x2b, 0x18, 0x38, 0x36, 0xe3, 0x7e, 0xcd, 0x66, 0x5e, 0x10, 0xba, 0x35, + 0x31, 0xf2, 0x69, 0x50, 0x8d, 0x9a, 0xcc, 0xb8, 0x1b, 0x36, 0xb7, 0x86, 0x94, 0xe4, 0xa3, 0xcd, + 0xad, 0x21, 0xc5, 0x6b, 0xa8, 0xd0, 0x16, 0x96, 0x08, 0x03, 0x32, 0x0b, 0xd9, 0x28, 0xc2, 0x2b, + 0x68, 0xee, 0x84, 0x09, 0x1a, 0x90, 0x39, 0x48, 0xab, 0x40, 0x56, 0xff, 0x14, 0x0a, 0x3f, 0x14, + 0xa4, 0xa0, 0xaa, 0x55, 0x84, 0x9f, 0x20, 0xbd, 0xad, 0x1e, 0xa9, 0xd9, 0x20, 0x45, 0x80, 0xee, + 0x12, 0xb8, 0x82, 0xca, 0x51, 0x00, 0xc7, 0x97, 0x00, 0x4f, 0xa6, 0x12, 0x15, 0x1d, 0xab, 0x1f, + 0x10, 0xbd, 0x92, 0x4f, 0x54, 0xc8, 0x94, 0xfc, 0xf6, 0xce, 0xc8, 0xa7, 0x64, 0x5e, 0x7d, 0xbb, + 0x5c, 0xe3, 0x67, 0x08, 0x35, 0x68, 0xcf, 0xf1, 0x1c, 0x39, 0x03, 0x82, 0x2a, 0xda, 0x56, 0x79, + 0xa7, 0x52, 0x1d, 0xcf, 0xab, 0x9a, 0x78, 0xd8, 0xbb, 0xba, 0xfa, 0xec, 0xe5, 0xf5, 0x46, 0xce, + 0x4c, 0x74, 0xe2, 0x6f, 0x90, 0x6e, 0x5a, 0x3d, 0xd1, 0xf4, 0xba, 0xf4, 0x82, 0x94, 0x61, 0x9b, + 0xe5, 0x6a, 0x34, 0xbc, 0x31, 0x50, 0x2f, 0xc9, 0xbe, 0xab, 0xeb, 0x0d, 0xcd, 0xbc, 0xab, 0xc6, + 0x0d, 0xb4, 0xb8, 0xef, 0x09, 0xca, 0x7d, 0xee, 0x04, 0xf4, 0x98, 0x0a, 0x8b, 0x2c, 0x40, 0xff, + 0x5a, 0xdc, 0x9f, 0x46, 0xa3, 0xc3, 0x5f, 0xeb, 0x91, 0xd7, 0xdf, 0xbf, 0xf0, 0x59, 0x40, 0xbb, + 0x2d, 0xc6, 0x05, 0x59, 0xac, 0x68, 0x5b, 0x73, 0x66, 0x32, 0x85, 0xd7, 0x51, 0xa9, 0x29, 0x7b, + 0xce, 0x2d, 0x97, 0x3c, 0x80, 0x27, 0x18, 0xc7, 0x98, 0xa0, 0x62, 0xc7, 0x19, 0x52, 0x16, 0x0a, + 0xb2, 0x04, 0x50, 0x1c, 0x6e, 0x7e, 0x0c, 0xe4, 0xea, 0x52, 0xfe, 0xdc, 0x72, 0x43, 0x2a, 0x67, + 0x0a, 0x0b, 0xa2, 0xc1, 0xfb, 0xaa, 0x60, 0xf3, 0x45, 0x11, 0xad, 0x66, 0xbe, 0x94, 0x7c, 0xf3, + 0xc3, 0x4e, 0xa7, 0x15, 0x93, 0x51, 0xae, 0xf1, 0x53, 0xb4, 0xd0, 0x39, 0x6a, 0xcb, 0xc9, 0x50, + 0x0e, 0xd3, 0x7c, 0x08, 0x60, 0x3a, 0x19, 0x57, 0x9d, 0x39, 0xfe, 0x73, 0xca, 0x9d, 0xde, 0x08, + 0x88, 0x5b, 0x32, 0xd3, 0x49, 0xfc, 0x23, 0x2a, 0xa8, 0xcf, 0x23, 0xf9, 0x4a, 0x7e, 0xab, 0xbc, + 0xb3, 0x7d, 0xdf, 0xec, 0xaa, 0xaa, 0x7c, 0xdf, 0x13, 0x7c, 0x14, 0x3d, 0x65, 0xb4, 0x83, 0x64, + 0xe6, 0x31, 0x15, 0x03, 0xd6, 0x8d, 0x79, 0xac, 0x22, 0x79, 0x87, 0x3a, 0xeb, 0x8e, 0x08, 0x56, + 0x77, 0x90, 0x6b, 0xbc, 0x84, 0xf2, 0x9d, 0xbd, 0x56, 0xc4, 0x6c, 0xb9, 0xc4, 0x3f, 0x24, 0x9e, + 0xb7, 0x00, 0x03, 0x7c, 0x54, 0x55, 0x62, 0xaf, 0xc6, 0x62, 0xaf, 0x36, 0x22, 0xb1, 0x2b, 0x22, + 0xbc, 0xfc, 0x6b, 0x43, 0x4b, 0xcc, 0xe0, 0x29, 0x5a, 0x50, 0x52, 0x38, 0xb6, 0x2e, 0xda, 0xce, + 0xef, 0x94, 0xe8, 0x15, 0x6d, 0x6b, 0xc1, 0x4c, 0x27, 0xf1, 0x77, 0x77, 0x93, 0x2a, 0x4e, 0x7f, + 0x4a, 0xdc, 0x83, 0xcf, 0x90, 0xd1, 0xa0, 0x9c, 0xf6, 0x9d, 0x40, 0x50, 0xbe, 0xc7, 0x1d, 0xe1, + 0xd8, 0x96, 0x1b, 0x89, 0x64, 0xb7, 0x27, 0x28, 0x07, 0x69, 0x4d, 0xb9, 0xeb, 0x3d, 0x5b, 0x61, + 0x03, 0xa1, 0xb6, 0xcd, 0x1d, 0x5f, 0xec, 0xf2, 0x7e, 0x40, 0x10, 0x30, 0x26, 0x91, 0xc1, 0xdb, + 0x68, 0xb9, 0xc1, 0xec, 0x33, 0xca, 0xf7, 0x98, 0x27, 0x2c, 0xc7, 0xa3, 0xbc, 0xd9, 0x00, 0xf1, + 0xe8, 0xe6, 0x24, 0x20, 0xa9, 0xd7, 0x1e, 0x50, 0xd7, 0x8d, 0xf4, 0xab, 0x02, 0x39, 0xb4, 0xc3, + 0x9d, 0x56, 0xf3, 0xe4, 0x80, 0xac, 0xa8, 0xa1, 0xa9, 0x48, 0x0e, 0xed, 0xc0, 0x6c, 0xed, 0x81, + 0x96, 0x74, 0x13, 0xd6, 0xf2, 0x7b, 0xe4, 0xdf, 0x5f, 0x02, 0xda, 0x39, 0x6a, 0x83, 0x44, 0x4a, + 0x66, 0x22, 0x23, 0x2d, 0x68, 0xd7, 0x75, 0xac, 0x00, 0xec, 0x53, 0x49, 0xe4, 0x2e, 0x81, 0x37, + 0xd1, 0x3c, 0x04, 0xd1, 0x15, 0x23, 0xa1, 0xa4, 0x72, 0xf8, 0x4b, 0x94, 0xef, 0x74, 0x8e, 0xc8, + 0xf2, 0xf4, 0x6f, 0x28, 0xeb, 0xd7, 0x7f, 0x8e, 0x45, 0x06, 0xb4, 0x94, 0xe4, 0x3a, 0xa3, 0xa3, + 0x48, 0x33, 0x72, 0x89, 0xb7, 0xd1, 0xdc, 0x39, 0xc8, 0x6e, 0x26, 0xb2, 0x86, 0x14, 0xcb, 0x63, + 0x75, 0x9a, 0xaa, 0xe8, 0xdb, 0x99, 0xaf, 0xb5, 0xcd, 0x57, 0x3a, 0xd2, 0x81, 0xfa, 0x60, 0x73, + 0x09, 0xff, 0xd7, 0xde, 0x89, 0xff, 0xcf, 0x64, 0xfa, 0x7f, 0x3e, 0xdb, 0xff, 0x67, 0x93, 0xfe, + 0x9f, 0x26, 0xc5, 0xdc, 0x04, 0x29, 0x62, 0xc7, 0x28, 0x24, 0x1c, 0xe3, 0xfb, 0xb1, 0xca, 0x57, + 0x40, 0xe5, 0x49, 0x87, 0x1e, 0x5f, 0x72, 0x2a, 0x65, 0x17, 0x33, 0x95, 0xbd, 0x3e, 0xa9, 0xec, + 0x52, 0xb6, 0xb2, 0xf5, 0xb7, 0x51, 0x76, 0x8a, 0x57, 0xe8, 0x3e, 0x5e, 0x95, 0x33, 0x78, 0x95, + 0xa9, 0x94, 0xf9, 0x7b, 0x95, 0xb2, 0x90, 0xad, 0x94, 0x27, 0x99, 0x4a, 0x59, 0x7c, 0xa3, 0x52, + 0x1e, 0x4c, 0x28, 0x65, 0xc2, 0xc2, 0x1f, 0x4f, 0x65, 0xe1, 0x4b, 0x59, 0x16, 0x9e, 0x70, 0xb4, + 0xe5, 0xb7, 0x70, 0xb4, 0x48, 0x72, 0xf8, 0xff, 0x49, 0x0e, 0xef, 0xa0, 0x95, 0x76, 0x68, 0xdb, + 0x34, 0x08, 0xea, 0xb4, 0xc7, 0x38, 0x6d, 0x59, 0x41, 0xe0, 0x78, 0x7d, 0xb2, 0x0a, 0x3f, 0x9c, + 0x99, 0x18, 0xfe, 0x02, 0xad, 0x3e, 0xb3, 0x1c, 0x37, 0xe4, 0x34, 0x02, 0x7e, 0xb5, 0xb8, 0x27, + 0x9b, 0x3e, 0x84, 0xa6, 0x6c, 0x10, 0x7f, 0x85, 0xd6, 0xd2, 0x40, 0xec, 0x95, 0x64, 0x0d, 0xda, + 0xde, 0x80, 0x4a, 0xd6, 0xb4, 0x38, 0xbb, 0x18, 0x81, 0x1a, 0x3e, 0x50, 0xac, 0x19, 0x27, 0xc6, + 0x28, 0x8c, 0x8e, 0x24, 0x50, 0x98, 0xdf, 0xfd, 0x36, 0xff, 0xf0, 0xdd, 0xd9, 0xfc, 0xc4, 0x0f, + 0xd7, 0x23, 0xb8, 0x57, 0x3a, 0xf9, 0x1e, 0x3c, 0xae, 0x7e, 0x7c, 0xf9, 0xca, 0xc8, 0x5d, 0xde, + 0x18, 0xda, 0xd5, 0x8d, 0xa1, 0xfd, 0x7d, 0x63, 0x68, 0x2f, 0x6e, 0x8d, 0xdc, 0xcb, 0x5b, 0x23, + 0x77, 0x75, 0x6b, 0xe4, 0xfe, 0xb8, 0x35, 0x72, 0xbf, 0x7d, 0xfa, 0x5f, 0x16, 0xf7, 0xda, 0x3f, + 0xee, 0xa7, 0x05, 0x48, 0x7c, 0xfe, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8b, 0x36, 0xc6, 0x76, + 0xd2, 0x0b, 0x00, 0x00, } func (m *HealthCheck) Marshal() (dAtA []byte, err error) { @@ -757,6 +760,13 @@ func (m *CheckType) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.FailuresBeforeWarning != 0 { + i = encodeVarintHealthcheck(dAtA, i, uint64(m.FailuresBeforeWarning)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xe8 + } if len(m.H2PING) > 0 { i -= len(m.H2PING) copy(dAtA[i:], m.H2PING) @@ -1296,6 +1306,9 @@ func (m *CheckType) Size() (n int) { if l > 0 { n += 2 + l + sovHealthcheck(uint64(l)) } + if m.FailuresBeforeWarning != 0 { + n += 2 + sovHealthcheck(uint64(m.FailuresBeforeWarning)) + } return n } @@ -3656,6 +3669,25 @@ func (m *CheckType) Unmarshal(dAtA []byte) error { } m.H2PING = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 29: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailuresBeforeWarning", wireType) + } + m.FailuresBeforeWarning = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowHealthcheck + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FailuresBeforeWarning |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipHealthcheck(dAtA[iNdEx:]) diff --git a/proto/pbservice/healthcheck.proto b/proto/pbservice/healthcheck.proto index 217373cce2..5bc893f4ae 100644 --- a/proto/pbservice/healthcheck.proto +++ b/proto/pbservice/healthcheck.proto @@ -136,6 +136,8 @@ message CheckType { // mog: func-to=int func-from=int32 int32 SuccessBeforePassing = 21; // mog: func-to=int func-from=int32 + int32 FailuresBeforeWarning = 29; + // mog: func-to=int func-from=int32 int32 FailuresBeforeCritical = 22; // Definition fields used when exposing checks through a proxy diff --git a/website/content/api-docs/agent/check.mdx b/website/content/api-docs/agent/check.mdx index 3a2638b139..ba832ff76e 100644 --- a/website/content/api-docs/agent/check.mdx +++ b/website/content/api-docs/agent/check.mdx @@ -240,11 +240,16 @@ The table below shows this endpoint's support for - `SuccessBeforePassing` `(int: 0)` - Specifies the number of consecutive successful results required before check status transitions to passing. Available for HTTP, - TCP, gRPC, Docker & Monitor checks. + TCP, gRPC, Docker & Monitor checks. Added in Consul 1.7.0. + +- `FailuresBeforeWarning` `(int: 0)` - Specifies the number of consecutive unsuccessful + results required before check status transitions to warning. Defaults to the same value + as `FailuresBeforeCritical`. Values higher than `FailuresBeforeCritical` are invalid. + Available for HTTP, TCP, gRPC, Docker & Monitor checks. Added in Consul 1.11.0. - `FailuresBeforeCritical` `(int: 0)` - Specifies the number of consecutive unsuccessful results required before check status transitions to critical. Available for HTTP, - TCP, gRPC, Docker & Monitor checks. + TCP, gRPC, Docker & Monitor checks. Added in Consul 1.7.0. ### Sample Payload diff --git a/website/content/docs/discovery/checks.mdx b/website/content/docs/discovery/checks.mdx index e4967e2ef5..2a6ad28617 100644 --- a/website/content/docs/discovery/checks.mdx +++ b/website/content/docs/discovery/checks.mdx @@ -413,12 +413,23 @@ key in your configuration file. } ``` -## Success/Failures before passing/critical +## Success/Failures before passing/warning/critical -In Consul 1.7.0 and later, a check may be configured to become passing/critical -only after a specified number of consecutive checks return passing/critical. +To prevent flapping health checks, and limit the load they cause on the cluster, +a health check may be configured to become passing/warning/critical only after a +specified number of consecutive checks return passing/critical. The status will not transition states until the configured threshold is reached. +* `success_before_passing` - Number of consecutive successful results required + before check status transitions to passing. Defaults to `0`. Added in Consul 1.7.0. +* `failures_before_warning` - Number of consecutive unsuccessful results required + before check status transitions to warning. Defaults to the same value as that of + `failures_before_critical` to maintain the expected behavior of not changing the + status of serivce checks to `warning` before `critical` unless configured to do so. + Values higher than `failures_before_critical` are invalid. Added in Consul 1.11.0. +* `failures_before_critical` - Number of consecutive unsuccessful results required + before check status transitions to critical. Defaults to `0`. Added in Consul 1.7.0. + This feature is available for HTTP, TCP, gRPC, Docker & Monitor checks. By default, both passing and critical thresholds will be set to 0 so the check status will always reflect the last check result. @@ -432,6 +443,7 @@ status will always reflect the last check result. "interval": "10s", "timeout": "1s", "success_before_passing": 3, + "failures_before_warning": 1, "failures_before_critical": 3 } ]