Support Envoy's MaxEjectionPercent and BaseEjectionTime config entries for passive health checks (#15979)

* Add MaxEjectionPercent to config entry

* Add BaseEjectionTime to config entry

* Add MaxEjectionPercent and BaseEjectionTime to protobufs

* Add MaxEjectionPercent and BaseEjectionTime to api

* Fix integration test breakage

* Verify MaxEjectionPercent and BaseEjectionTime in integration test upstream confings

* Website docs for MaxEjectionPercent and BaseEjection time

* Add `make docs` to browse docs at http://localhost:3000

* Changelog entry

* so that is the difference between consul-docker and dev-docker

* blah

* update proto funcs

* update proto

---------

Co-authored-by: Maliz <maliheh.monshizadeh@hashicorp.com>

.changelog/15979.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs.
+```

GNUmakefile

@@ -405,6 +405,7 @@ ui-build-image:
 	@echo "Building UI build container"
 	@docker build $(NOCACHE) $(QUIET) -t $(UI_BUILD_TAG) - < build-support/docker/Build-UI.dockerfile
 
+# Builds consul in a docker container and then dumps executable into ./pkg/bin/...
 consul-docker: go-build-image
 	@$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh consul
 
@@ -538,6 +539,11 @@ envoy-regen:
 	@find "command/connect/envoy/testdata" -name '*.golden' -delete
 	@go test -tags '$(GOTAGS)' ./command/connect/envoy -update
 
+# Point your web browser to http://localhost:3000/consul to live render docs from ./website/
+.PHONY: docs
+docs:
+	make -C website
+
 .PHONY: help
 help:
 	$(info available make targets)

agent/consul/config_endpoint_test.go

@@ -1484,6 +1484,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								Interval:                10,
 								MaxFailures:             2,
 								EnforcingConsecutive5xx: uintPointer(60),
+								MaxEjectionPercent:      uintPointer(61),
+								BaseEjectionTime:        durationPointer(62 * time.Second),
 							},
 						},
 						Overrides: []*structs.UpstreamConfig{
@@ -1518,6 +1520,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								"Interval":                int64(10),
 								"MaxFailures":             int64(2),
 								"EnforcingConsecutive5xx": int64(60),
+								"MaxEjectionPercent":      int64(61),
+								"BaseEjectionTime":        uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "none",
@@ -1532,6 +1536,8 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) {
 								"Interval":                int64(10),
 								"MaxFailures":             int64(2),
 								"EnforcingConsecutive5xx": int64(60),
+								"MaxEjectionPercent":      int64(61),
+								"BaseEjectionTime":        uint64(62 * time.Second),
 							},
 							"mesh_gateway": map[string]interface{}{
 								"Mode": "local",
@@ -2639,3 +2645,7 @@ func Test_gateWriteToSecondary_AllowedKinds(t *testing.T) {
 func uintPointer(v uint32) *uint32 {
 	return &v
 }
+
+func durationPointer(d time.Duration) *time.Duration {
+	return &d
+}

agent/proxycfg/proxycfg.deepcopy.go

@@ -7,6 +7,7 @@ import (
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/types"
+	"time"
 )
 
 // DeepCopy generates a deep copy of *ConfigSnapshot
@@ -452,6 +453,14 @@ func (o *configSnapshotIngressGateway) DeepCopy() *configSnapshotIngressGateway
 			cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 			*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
 		}
+		if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
+			cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+			*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
+		}
+		if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
+			cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+			*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
+		}
 	}
 	return &cp
 }

agent/structs/config_entry.go

@@ -1101,6 +1101,16 @@ type PassiveHealthCheck struct {
 	// when an outlier status is detected through consecutive 5xx.
 	// This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100.
 	EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
+
+	// The maximum % of an upstream cluster that can be ejected due to outlier detection.
+	// Defaults to 10% but will eject at least one host regardless of the value.
+	// TODO: remove me
+	MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`
+
+	// The base time that a host is ejected for. The real time is equal to the base time
+	// multiplied by the number of times the host has been ejected and is capped by
+	// max_ejection_time (Default 300s). Defaults to 30000ms or 30s.
+	BaseEjectionTime *time.Duration `json:",omitempty" alias:"base_ejection_time"`
 }
 
 func (chk *PassiveHealthCheck) Clone() *PassiveHealthCheck {
@@ -1120,6 +1130,15 @@ func (chk PassiveHealthCheck) Validate() error {
 	if chk.Interval < 0*time.Second {
 		return fmt.Errorf("passive health check interval cannot be negative")
 	}
+	if chk.EnforcingConsecutive5xx != nil && *chk.EnforcingConsecutive5xx > 100 {
+		return fmt.Errorf("passive health check enforcing_consecutive_5xx must be a percentage between 0 and 100")
+	}
+	if chk.MaxEjectionPercent != nil && *chk.MaxEjectionPercent > 100 {
+		return fmt.Errorf("passive health check max_ejection_percent must be a percentage between 0 and 100")
+	}
+	if chk.BaseEjectionTime != nil && *chk.BaseEjectionTime < 0*time.Second {
+		return fmt.Errorf("passive health check base_ejection_time cannot be negative")
+	}
 	return nil
 }
 

agent/structs/config_entry_test.go

@@ -416,6 +416,9 @@ func TestDecodeConfigEntry(t *testing.T) {
 							passive_health_check {
 								interval = "2s"
 								max_failures = 3
+								enforcing_consecutive_5xx = 4
+								max_ejection_percent = 5
+								base_ejection_time = "6s"
 							}
 						},
 						{
@@ -460,6 +463,9 @@ func TestDecodeConfigEntry(t *testing.T) {
 							PassiveHealthCheck {
 								MaxFailures = 3
 								Interval = "2s"
+								EnforcingConsecutive5xx = 4
+								MaxEjectionPercent = 5
+								BaseEjectionTime = "6s"
 							}
 						},
 						{
@@ -502,8 +508,11 @@ func TestDecodeConfigEntry(t *testing.T) {
 						{
 							Name: "redis",
 							PassiveHealthCheck: &PassiveHealthCheck{
-								MaxFailures: 3,
+								MaxFailures:             3,
-								Interval:    2 * time.Second,
+								Interval:                2 * time.Second,
+								EnforcingConsecutive5xx: uintPointer(4),
+								MaxEjectionPercent:      uintPointer(5),
+								BaseEjectionTime:        durationPointer(6 * time.Second),
 							},
 						},
 						{
@@ -2351,6 +2360,39 @@ func TestPassiveHealthCheck_Validate(t *testing.T) {
 			wantErr: true,
 			wantMsg: "cannot be negative",
 		},
+		{
+			name:    "valid enforcing_consecutive_5xx",
+			input:   PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(100)},
+			wantErr: false,
+		},
+		{
+			name:    "invalid enforcing_consecutive_5xx",
+			input:   PassiveHealthCheck{EnforcingConsecutive5xx: uintPointer(101)},
+			wantErr: true,
+			wantMsg: "must be a percentage",
+		},
+		{
+			name:    "valid max_ejection_percent",
+			input:   PassiveHealthCheck{MaxEjectionPercent: uintPointer(100)},
+			wantErr: false,
+		},
+		{
+			name:    "invalid max_ejection_percent",
+			input:   PassiveHealthCheck{MaxEjectionPercent: uintPointer(101)},
+			wantErr: true,
+			wantMsg: "must be a percentage",
+		},
+		{
+			name:    "valid base_ejection_time",
+			input:   PassiveHealthCheck{BaseEjectionTime: durationPointer(0 * time.Second)},
+			wantErr: false,
+		},
+		{
+			name:    "negative base_ejection_time",
+			input:   PassiveHealthCheck{BaseEjectionTime: durationPointer(-1 * time.Second)},
+			wantErr: true,
+			wantMsg: "cannot be negative",
+		},
 	}
 
 	for _, tc := range tt {
@@ -2890,8 +2932,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				PassiveHealthCheck: &PassiveHealthCheck{
-					MaxFailures: 3,
+					Interval:                2 * time.Second,
-					Interval:    2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2908,8 +2953,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 3,
+					Interval:                2 * time.Second,
-					Interval:    2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2928,8 +2976,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				PassiveHealthCheck: &PassiveHealthCheck{
-					MaxFailures: 3,
+					Interval:                2 * time.Second,
-					Interval:    2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				MeshGateway: MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2945,8 +2996,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(12),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 13,
+					MaxFailures:             13,
-					Interval:    14 * time.Second,
+					Interval:                14 * time.Second,
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -2962,8 +3016,11 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 					MaxConcurrentRequests: intPointer(5),
 				},
 				"passive_health_check": &PassiveHealthCheck{
-					MaxFailures: 3,
+					Interval:                2 * time.Second,
-					Interval:    2 * time.Second,
+					MaxFailures:             3,
+					EnforcingConsecutive5xx: uintPointer(4),
+					MaxEjectionPercent:      uintPointer(5),
+					BaseEjectionTime:        durationPointer(6 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeRemote},
 			},
@@ -2985,7 +3042,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 				"passive_health_check": &PassiveHealthCheck{
 					MaxFailures:             13,
 					Interval:                14 * time.Second,
-					EnforcingConsecutive5xx: uintPointer(80),
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -3003,7 +3062,9 @@ func TestUpstreamConfig_MergeInto(t *testing.T) {
 				"passive_health_check": &PassiveHealthCheck{
 					MaxFailures:             13,
 					Interval:                14 * time.Second,
-					EnforcingConsecutive5xx: uintPointer(80),
+					EnforcingConsecutive5xx: uintPointer(15),
+					MaxEjectionPercent:      uintPointer(16),
+					BaseEjectionTime:        durationPointer(17 * time.Second),
 				},
 				"mesh_gateway": MeshGatewayConfig{Mode: MeshGatewayModeLocal},
 			},
@@ -3138,15 +3199,21 @@ func TestParseUpstreamConfig(t *testing.T) {
 			name: "passive health check map",
 			input: map[string]interface{}{
 				"passive_health_check": map[string]interface{}{
-					"interval":     "22s",
+					"interval":                  "22s",
-					"max_failures": 7,
+					"max_failures":              7,
+					"enforcing_consecutive_5xx": 8,
+					"max_ejection_percent":      9,
+					"base_ejection_time":        "10s",
 				},
 			},
 			want: UpstreamConfig{
 				ConnectTimeoutMs: 5000,
 				PassiveHealthCheck: &PassiveHealthCheck{
-					Interval:    22 * time.Second,
+					Interval:                22 * time.Second,
-					MaxFailures: 7,
+					MaxFailures:             7,
+					EnforcingConsecutive5xx: uintPointer(8),
+					MaxEjectionPercent:      uintPointer(9),
+					BaseEjectionTime:        durationPointer(10 * time.Second),
 				},
 				Protocol: "tcp",
 			},
@@ -3273,10 +3340,6 @@ func requireContainsLower(t *testing.T, haystack, needle string) {
 	require.Contains(t, strings.ToLower(haystack), strings.ToLower(needle))
 }
 
-func intPointer(i int) *int {
-	return &i
-}
-
 func TestConfigEntryQuery_CacheInfoKey(t *testing.T) {
 	assertCacheInfoKeyIsComplete(t, &ConfigEntryQuery{})
 }
@@ -3371,6 +3434,14 @@ func testConfigEntryNormalizeAndValidate(t *testing.T, cases map[string]configEn
 	}
 }
 
+func intPointer(i int) *int {
+	return &i
+}
+
 func uintPointer(v uint32) *uint32 {
 	return &v
 }
+
+func durationPointer(d time.Duration) *time.Duration {
+	return &d
+}

agent/structs/structs.deepcopy.go

@@ -529,6 +529,14 @@ func (o *IngressListener) DeepCopy() *IngressListener {
 					cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 					*cp.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Services[i2].PassiveHealthCheck.EnforcingConsecutive5xx
 				}
+				if o.Services[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
+					cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+					*cp.Services[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Services[i2].PassiveHealthCheck.MaxEjectionPercent
+				}
+				if o.Services[i2].PassiveHealthCheck.BaseEjectionTime != nil {
+					cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+					*cp.Services[i2].PassiveHealthCheck.BaseEjectionTime = *o.Services[i2].PassiveHealthCheck.BaseEjectionTime
+				}
 			}
 			if o.Services[i2].Meta != nil {
 				cp.Services[i2].Meta = make(map[string]string, len(o.Services[i2].Meta))
@@ -1134,6 +1142,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
 						cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 						*cp.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx = *o.Overrides[i2].PassiveHealthCheck.EnforcingConsecutive5xx
 					}
+					if o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent != nil {
+						cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+						*cp.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent = *o.Overrides[i2].PassiveHealthCheck.MaxEjectionPercent
+					}
+					if o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime != nil {
+						cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+						*cp.Overrides[i2].PassiveHealthCheck.BaseEjectionTime = *o.Overrides[i2].PassiveHealthCheck.BaseEjectionTime
+					}
 				}
 			}
 		}
@@ -1164,6 +1180,14 @@ func (o *UpstreamConfiguration) DeepCopy() *UpstreamConfiguration {
 				cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = new(uint32)
 				*cp.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx = *o.Defaults.PassiveHealthCheck.EnforcingConsecutive5xx
 			}
+			if o.Defaults.PassiveHealthCheck.MaxEjectionPercent != nil {
+				cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = new(uint32)
+				*cp.Defaults.PassiveHealthCheck.MaxEjectionPercent = *o.Defaults.PassiveHealthCheck.MaxEjectionPercent
+			}
+			if o.Defaults.PassiveHealthCheck.BaseEjectionTime != nil {
+				cp.Defaults.PassiveHealthCheck.BaseEjectionTime = new(time.Duration)
+				*cp.Defaults.PassiveHealthCheck.BaseEjectionTime = *o.Defaults.PassiveHealthCheck.BaseEjectionTime
+			}
 		}
 	}
 	return &cp

agent/structs/structs.go

@@ -3026,6 +3026,26 @@ func DurationFromProto(d *durationpb.Duration) time.Duration {
 	return d.AsDuration()
 }
 
+// This should only be used for conversions generated by MOG
+func DurationPointerToProto(d *time.Duration) *durationpb.Duration {
+	if d == nil {
+		return nil
+	}
+	return durationpb.New(*d)
+}
+
+// This should only be used for conversions generated by MOG
+func DurationPointerFromProto(d *durationpb.Duration) *time.Duration {
+	if d == nil {
+		return nil
+	}
+	return DurationPointer(d.AsDuration())
+}
+
+func DurationPointer(d time.Duration) *time.Duration {
+	return &d
+}
+
 // This should only be used for conversions generated by MOG
 func TimeFromProto(s *timestamppb.Timestamp) time.Time {
 	return s.AsTime()

agent/xds/clusters_test.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 	"testing"
 	"text/template"
+	"time"
 
 	envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	"github.com/hashicorp/consul/agent/xds/testcommon"
@@ -270,7 +271,9 @@ func TestClustersFromSnapshot(t *testing.T) {
 					ns.Proxy.Upstreams[0].Config["passive_health_check"] = map[string]interface{}{
 						"enforcing_consecutive_5xx": float64(80),
 						"max_failures":              float64(5),
-						"interval":                  float64(10),
+						"interval":                  float64(10 * time.Second),
+						"max_ejection_percent":      float64(100),
+						"base_ejection_time":        float64(10 * time.Second),
 					}
 				}, nil)
 			},

agent/xds/config.go

@@ -178,7 +178,7 @@ func ParseGatewayConfig(m map[string]interface{}) (GatewayConfig, error) {
 	return cfg, err
 }
 
-// Return an envoy.OutlierDetection populated by the values from structs.PassiveHealthChec.
+// Return an envoy.OutlierDetection populated by the values from structs.PassiveHealthCheck.
 // If all values are zero a default empty OutlierDetection will be returned to
 // enable outlier detection with default values.
 //   - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults
@@ -197,13 +197,20 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
 		}
 
 		if p.EnforcingConsecutive5xx != nil {
-			// NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway
+			// NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway
 			if *p.EnforcingConsecutive5xx != 0 {
 				od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
 			} else if allowZero {
 				od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx}
 			}
 		}
+
+		if p.MaxEjectionPercent != nil {
+			od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *p.MaxEjectionPercent}
+		}
+		if p.BaseEjectionTime != nil {
+			od.BaseEjectionTime = durationpb.New(*p.BaseEjectionTime)
+		}
 	}
 
 	if override == nil {
@@ -227,5 +234,12 @@ func ToOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passive
 		}
 	}
 
+	if override.MaxEjectionPercent != nil {
+		od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *override.MaxEjectionPercent}
+	}
+	if override.BaseEjectionTime != nil {
+		od.BaseEjectionTime = durationpb.New(*override.BaseEjectionTime)
+	}
+
 	return od
 }

agent/xds/testdata/clusters/custom-passive-healthcheck.latest.golden

@@ -20,8 +20,10 @@
       },
       "outlierDetection": {
         "consecutive5xx": 5,
-        "interval": "0.000000010s",
+        "interval": "10s",
-        "enforcingConsecutive5xx": 80
+        "enforcingConsecutive5xx": 80,
+        "maxEjectionPercent": 100,
+        "baseEjectionTime": "10s"
       },
       "commonLbConfig": {
         "healthyPanicThreshold": {

api/config_entry.go

@@ -277,6 +277,15 @@ type PassiveHealthCheck struct {
 	// when an outlier status is detected through consecutive 5xx.
 	// This setting can be used to disable ejection or to ramp it up slowly.
 	EnforcingConsecutive5xx *uint32 `json:",omitempty" alias:"enforcing_consecutive_5xx"`
+
+	// The maximum % of an upstream cluster that can be ejected due to outlier detection.
+	// Defaults to 10% but will eject at least one host regardless of the value.
+	MaxEjectionPercent *uint32 `json:",omitempty" alias:"max_ejection_percent"`
+
+	// The base time that a host is ejected for. The real time is equal to the base time
+	// multiplied by the number of times the host has been ejected and is capped by
+	// max_ejection_time (Default 300s). Defaults to 30000ms or 30s.
+	BaseEjectionTime *time.Duration `json:",omitempty" alias:"base_ejection_time"`
 }
 
 // UpstreamLimits describes the limits that are associated with a specific

api/config_entry_test.go

@@ -484,7 +484,9 @@ func TestDecodeConfigEntry(t *testing.T) {
 							"PassiveHealthCheck": {
 								"MaxFailures": 3,
 								"Interval": "2s",
-								"EnforcingConsecutive5xx": 60
+								"EnforcingConsecutive5xx": 60,
+								"MaxEjectionPercent": 4,
+								"BaseEjectionTime": "5s"
 							},
 							"BalanceOutboundConnections": "exact_balance"
 						},
@@ -507,7 +509,11 @@ func TestDecodeConfigEntry(t *testing.T) {
 						},
 						"PassiveHealthCheck": {
 								"MaxFailures": 5,
-								"Interval": "4s"
+								"Interval": "4s",
+								"EnforcingConsecutive5xx": 61,
+								"MaxEjectionPercent": 5,
+								"BaseEjectionTime": "6s"
+
 						}
 					}
 				}
@@ -539,6 +545,8 @@ func TestDecodeConfigEntry(t *testing.T) {
 								MaxFailures:             3,
 								Interval:                2 * time.Second,
 								EnforcingConsecutive5xx: uint32Pointer(60),
+								MaxEjectionPercent:      uint32Pointer(4),
+								BaseEjectionTime:        durationPointer(5 * time.Second),
 							},
 							BalanceOutboundConnections: "exact_balance",
 						},
@@ -558,8 +566,11 @@ func TestDecodeConfigEntry(t *testing.T) {
 							MaxConcurrentRequests: intPointer(5),
 						},
 						PassiveHealthCheck: &PassiveHealthCheck{
-							MaxFailures: 5,
+							MaxFailures:             5,
-							Interval:    4 * time.Second,
+							Interval:                4 * time.Second,
+							EnforcingConsecutive5xx: uint32Pointer(61),
+							MaxEjectionPercent:      uint32Pointer(5),
+							BaseEjectionTime:        durationPointer(6 * time.Second),
 						},
 					},
 				},
@@ -1429,3 +1440,7 @@ func intPointer(v int) *int {
 func uint32Pointer(v uint32) *uint32 {
 	return &v
 }
+
+func durationPointer(d time.Duration) *time.Duration {
+	return &d
+}

proto/private/pbconfigentry/config_entry.gen.go

@@ -1243,6 +1243,8 @@ func PassiveHealthCheckToStructs(s *PassiveHealthCheck, t *structs.PassiveHealth
 	t.Interval = structs.DurationFromProto(s.Interval)
 	t.MaxFailures = s.MaxFailures
 	t.EnforcingConsecutive5xx = pointerToUint32FromUint32(s.EnforcingConsecutive5Xx)
+	t.MaxEjectionPercent = pointerToUint32FromUint32(s.MaxEjectionPercent)
+	t.BaseEjectionTime = structs.DurationPointerFromProto(s.BaseEjectionTime)
 }
 func PassiveHealthCheckFromStructs(t *structs.PassiveHealthCheck, s *PassiveHealthCheck) {
 	if s == nil {
@@ -1251,6 +1253,8 @@ func PassiveHealthCheckFromStructs(t *structs.PassiveHealthCheck, s *PassiveHeal
 	s.Interval = structs.DurationToProto(t.Interval)
 	s.MaxFailures = t.MaxFailures
 	s.EnforcingConsecutive5Xx = uint32FromPointerToUint32(t.EnforcingConsecutive5xx)
+	s.MaxEjectionPercent = uint32FromPointerToUint32(t.MaxEjectionPercent)
+	s.BaseEjectionTime = structs.DurationPointerToProto(t.BaseEjectionTime)
 }
 func PeeringMeshConfigToStructs(s *PeeringMeshConfig, t *structs.PeeringMeshConfig) {
 	if s == nil {

proto/private/pbconfigentry/config_entry.proto

@@ -633,6 +633,10 @@ message PassiveHealthCheck {
   uint32 MaxFailures = 2;
   // mog: target=EnforcingConsecutive5xx func-to=pointerToUint32FromUint32 func-from=uint32FromPointerToUint32
   uint32 EnforcingConsecutive5xx = 3;
+  // mog: func-to=pointerToUint32FromUint32 func-from=uint32FromPointerToUint32
+  uint32 MaxEjectionPercent = 4;
+  // mog: func-to=structs.DurationPointerFromProto func-from=structs.DurationPointerToProto
+  google.protobuf.Duration BaseEjectionTime = 5;
 }
 
 // mog annotation:

test/integration/connect/envoy/case-http/verify.bats

@@ -76,6 +76,8 @@ load helpers
   [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ]
   [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = null ]
   [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.enforcing_consecutive_5xx')" = null ]
+  [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.max_ejection_percent')" = null ]
+  [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.base_ejection_time')" = null ]
 }
 
 @test "s2 proxy should have been configured with http rbac filters" {

test/integration/connect/envoy/case-upstream-config/service_s1.hcl

@@ -20,6 +20,9 @@ services {
               passive_health_check {
                 interval = "22s"
                 max_failures = 4
+                enforcing_consecutive_5xx = 99
+                max_ejection_percent = 50
+                base_ejection_time = "60s"
               }
             }
           }

test/integration/connect/envoy/case-upstream-config/verify.bats

@@ -49,4 +49,7 @@ load helpers
 
   [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.consecutive_5xx')" = "4" ]
   [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.interval')" = "22s" ]
+  [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.enforcing_consecutive_5xx')" = "99" ]
+  [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.max_ejection_percent')" = "50" ]
+  [ "$(echo $CLUSTER_CONFIG | jq --raw-output '.outlier_detection.base_ejection_time')" = "60s" ]
 }

website/content/docs/connect/config-entries/service-defaults.mdx

@@ -1746,6 +1746,20 @@ represents a location outside the Consul cluster. Services can dial destinations
                        after a passive health check detects an outlier status through consecutive 5xx.`,
                   },
                 },
+                {
+                  name: 'MaxEjectionPercent',
+                  type: 'int: 10',
+                  description: `Measured in percent (%), the maximum percentage of hosts that can be ejected
+                    from a upstream cluster due to passive health check failures. If not specified, inherits 
+                    Envoy's default of 10% or at least one host.`,
+                },
+                {
+                  name: 'BaseEjectionTime',
+                  type: 'duration: 30s',
+                  description: `The base time that a host is ejected for. The real time is equal to the base
+                    time multiplied by the number of times the host has been ejected and is capped by 
+                    max_ejection_time (Default 300s). If not speficied, inherits Envoy's default value of 30s.`,
+                },
               ],
             },
           ],
@@ -1897,6 +1911,20 @@ represents a location outside the Consul cluster. Services can dial destinations
                        after a passive health check detects an outlier status through consecutive 5xx.`,
                   },
                 },
+                {
+                  name: 'MaxEjectionPercent',
+                  type: 'int: 10',
+                  description: `Measured in percent (%), the maximum percentage of hosts that can be ejected
+                    from a upstream cluster due to passive health check failures. If not specified, inherits 
+                    Envoy's default of 10% or at least one host.`,
+                },
+                {
+                  name: 'BaseEjectionTime',
+                  type: 'duration: 30s',
+                  description: `The base time that a host is ejected for. The real time is equal to the base
+                    time multiplied by the number of times the host has been ejected and is capped by 
+                    max_ejection_time (Default 300s). If not speficied, inherits Envoy's default value of 30s.`,
+                },
               ],
             },
           ],

website/content/docs/connect/proxies/envoy.mdx

@@ -412,6 +412,13 @@ definition](/consul/docs/connect/registration/service-registration) or
     host will be actually ejected when the proxy detects an outlier status
     through consecutive errors in the 500 code range. If not specified, Consul
     uses the proxy's default value. For example, `100` for Envoy proxy.
+  - `max_ejection_percent` - The maximum percentage of hosts that can be ejected
+    from a upstream cluster due to passive health check failures. If not specified,
+    inherits Envoy's default of 10% or at least one host.
+  - `base_ejection_time` - The base time that a host is ejected for. The real 
+     time is equal to the base time multiplied by the number of times the host has 
+     been ejected and is capped by max_ejection_time (Default 300s). If not 
+     speficied, inherits Envoy's default value of 30s.
 
 - `balance_outbound_connections` - Specifies the strategy for balancing outbound connections
   across Envoy worker threads. Consul service mesh Envoy integration supports the