From 1b08626358a42815852ecc943d929714a346b75b Mon Sep 17 00:00:00 2001 From: Dan Stough Date: Mon, 10 Jul 2023 17:08:06 -0400 Subject: [PATCH] [OSS] Fix initial_fetch_timeout to wait for all xDS resources (#18024) * fix(connect): set initial_fetch_time to wait indefinitely * changelog * PR feedback 1 --- .changelog/18024.txt | 3 ++ agent/proxycfg/api_gateway.go | 20 +++++++++--- agent/proxycfg/ingress_gateway.go | 6 +++- agent/proxycfg/snapshot.go | 31 ++++++++++++++++--- agent/proxycfg/state.go | 2 ++ agent/proxycfg/testing_api_gateway.go | 11 ++++--- command/connect/envoy/bootstrap_tpl.go | 2 ++ ..._ADDR-with-https-scheme-enables-tls.golden | 2 ++ ...tps-scheme-does-not-affect-grpc-tls.golden | 2 ++ .../envoy/testdata/access-log-path.golden | 2 ++ .../access-logs-enabled-custom.golden | 2 ++ .../envoy/testdata/access-logs-enabled.golden | 2 ++ .../testdata/acl-enabled-and-token.golden | 2 ++ .../testdata/acl-enabled-but-no-token.golden | 2 ++ ...AIN-and-CONSUL_GRPC_ADDR-TLS-is-tls.golden | 2 ++ ...and-CONSUL_GRPC_ADDR-PLAIN-is-plain.golden | 2 ++ .../envoy/testdata/defaults-nodemeta.golden | 2 ++ .../connect/envoy/testdata/defaults.golden | 2 ++ .../deprecated-grpc-addr-config.golden | 2 ++ .../testdata/envoy-readiness-probe.golden | 2 ++ .../envoy/testdata/existing-ca-file.golden | 2 ++ .../envoy/testdata/existing-ca-path.golden | 2 ++ .../envoy/testdata/extra_-multiple.golden | 2 ++ .../envoy/testdata/extra_-single.golden | 2 ++ .../envoy/testdata/grpc-addr-env.golden | 2 ++ .../envoy/testdata/grpc-addr-flag.golden | 2 ++ .../testdata/grpc-addr-unix-with-tls.golden | 2 ++ .../envoy/testdata/grpc-addr-unix.golden | 2 ++ .../testdata/grpc-tls-addr-config.golden | 2 ++ .../ingress-gateway-address-specified.golden | 2 ++ .../ingress-gateway-no-auto-register.golden | 2 ++ .../testdata/ingress-gateway-nodemeta.golden | 2 ++ ...-register-with-service-and-proxy-id.golden | 2 ++ ...ister-with-service-without-proxy-id.golden | 2 ++ .../envoy/testdata/ingress-gateway.golden | 2 ++ .../prometheus-metrics-tls-ca-file.golden | 2 ++ .../prometheus-metrics-tls-ca-path.golden | 2 ++ .../envoy/testdata/prometheus-metrics.golden | 2 ++ .../testdata/stats-config-override.golden | 2 ++ .../envoy/testdata/telemetry-collector.golden | 2 ++ .../connect/envoy/testdata/token-arg.golden | 2 ++ .../connect/envoy/testdata/token-env.golden | 2 ++ .../envoy/testdata/token-file-arg.golden | 2 ++ .../envoy/testdata/token-file-env.golden | 2 ++ .../envoy/testdata/xds-addr-config.golden | 2 ++ .../testdata/zipkin-tracing-config.golden | 2 ++ .../test/gateways/ingress_gateway_test.go | 5 --- 47 files changed, 137 insertions(+), 21 deletions(-) create mode 100644 .changelog/18024.txt diff --git a/.changelog/18024.txt b/.changelog/18024.txt new file mode 100644 index 0000000000..a661e7304c --- /dev/null +++ b/.changelog/18024.txt @@ -0,0 +1,3 @@ +```release-note:bug +connect: fix a bug with Envoy potentially starting with incomplete configuration by not waiting enough for initial xDS configuration. +``` \ No newline at end of file diff --git a/agent/proxycfg/api_gateway.go b/agent/proxycfg/api_gateway.go index 7a4a48d0d2..41eb592125 100644 --- a/agent/proxycfg/api_gateway.go +++ b/agent/proxycfg/api_gateway.go @@ -6,6 +6,7 @@ package proxycfg import ( "context" "fmt" + "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/leafcert" "github.com/hashicorp/consul/agent/proxycfg/internal/watch" @@ -48,13 +49,13 @@ func (h *handlerAPIGateway) initialize(ctx context.Context) (ConfigSnapshot, err } // Watch the api-gateway's config entry - err = h.subscribeToConfigEntry(ctx, structs.APIGateway, h.service, h.proxyID.EnterpriseMeta, gatewayConfigWatchID) + err = h.subscribeToConfigEntry(ctx, structs.APIGateway, h.service, h.proxyID.EnterpriseMeta, apiGatewayConfigWatchID) if err != nil { return snap, err } // Watch the bound-api-gateway's config entry - err = h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, h.proxyID.EnterpriseMeta, gatewayConfigWatchID) + err = h.subscribeToConfigEntry(ctx, structs.BoundAPIGateway, h.service, h.proxyID.EnterpriseMeta, boundGatewayConfigWatchID) if err != nil { return snap, err } @@ -108,9 +109,9 @@ func (h *handlerAPIGateway) handleUpdate(ctx context.Context, u UpdateEvent, sna if err := h.handleRootCAUpdate(u, snap); err != nil { return err } - case u.CorrelationID == gatewayConfigWatchID: + case u.CorrelationID == apiGatewayConfigWatchID || u.CorrelationID == boundGatewayConfigWatchID: // Handle change in the api-gateway or bound-api-gateway config entry - if err := h.handleGatewayConfigUpdate(ctx, u, snap); err != nil { + if err := h.handleGatewayConfigUpdate(ctx, u, snap, u.CorrelationID); err != nil { return err } case u.CorrelationID == inlineCertificateConfigWatchID: @@ -146,11 +147,20 @@ func (h *handlerAPIGateway) handleRootCAUpdate(u UpdateEvent, snap *ConfigSnapsh // In particular, we want to make sure that we're subscribing to any attached resources such // as routes and certificates. These additional subscriptions will enable us to update the // config snapshot appropriately for any route or certificate changes. -func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot) error { +func (h *handlerAPIGateway) handleGatewayConfigUpdate(ctx context.Context, u UpdateEvent, snap *ConfigSnapshot, correlationID string) error { resp, ok := u.Result.(*structs.ConfigEntryResponse) if !ok { return fmt.Errorf("invalid type for response: %T", u.Result) } else if resp.Entry == nil { + // A nil response indicates that we have the watch configured and that we are done with further changes + // until a new response comes in. By setting these earlier we allow a minimal xDS snapshot to configure the + // gateway. + if correlationID == apiGatewayConfigWatchID { + snap.APIGateway.GatewayConfigLoaded = true + } + if correlationID == boundGatewayConfigWatchID { + snap.APIGateway.BoundGatewayConfigLoaded = true + } return nil } diff --git a/agent/proxycfg/ingress_gateway.go b/agent/proxycfg/ingress_gateway.go index 22eb405056..efb774c9b1 100644 --- a/agent/proxycfg/ingress_gateway.go +++ b/agent/proxycfg/ingress_gateway.go @@ -95,6 +95,11 @@ func (s *handlerIngressGateway) handleUpdate(ctx context.Context, u UpdateEvent, if !ok { return fmt.Errorf("invalid type for response: %T", u.Result) } + + // We set this even if the response is empty so that we know the watch is set, + // but we don't block if the ingress config entry is unset for this gateway + snap.IngressGateway.GatewayConfigLoaded = true + if resp.Entry == nil { return nil } @@ -103,7 +108,6 @@ func (s *handlerIngressGateway) handleUpdate(ctx context.Context, u UpdateEvent, return fmt.Errorf("invalid type for config entry: %T", resp.Entry) } - snap.IngressGateway.GatewayConfigLoaded = true snap.IngressGateway.TLSConfig = gatewayConf.TLS if gatewayConf.Defaults != nil { snap.IngressGateway.Defaults = *gatewayConf.Defaults diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index e8f95d9651..1d06e5fd8c 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -824,6 +824,18 @@ DOMAIN_LOOP: return services, upstreams, compiled, err } +// valid tests for two valid api gateway snapshot states: +// 1. waiting: the watch on api and bound gateway entries is set, but none were received +// 2. loaded: both the valid config entries AND the leaf certs are set +func (c *configSnapshotAPIGateway) valid() bool { + waiting := c.GatewayConfigLoaded && len(c.Upstreams) == 0 && c.BoundGatewayConfigLoaded && c.Leaf == nil + + // If we have a leaf, it implies we successfully watched parent resources + loaded := c.GatewayConfigLoaded && c.BoundGatewayConfigLoaded && c.Leaf != nil + + return waiting || loaded +} + type configSnapshotIngressGateway struct { ConfigSnapshotUpstreams @@ -872,6 +884,18 @@ func (c *configSnapshotIngressGateway) isEmpty() bool { !c.MeshConfigSet } +// valid tests for two valid ingress snapshot states: +// 1. waiting: the watch on ingress config entries is set, but none were received +// 2. loaded: both the ingress config entry AND the leaf cert are set +func (c *configSnapshotIngressGateway) valid() bool { + waiting := c.GatewayConfigLoaded && len(c.Upstreams) == 0 && c.Leaf == nil + + // If we have a leaf, it implies we successfully watched parent resources + loaded := c.GatewayConfigLoaded && c.Leaf != nil + + return waiting || loaded +} + type APIGatewayListenerKey = IngressListenerKey func APIGatewayListenerKeyFromListener(l structs.APIGatewayListener) APIGatewayListenerKey { @@ -965,17 +989,14 @@ func (s *ConfigSnapshot) Valid() bool { case structs.ServiceKindIngressGateway: return s.Roots != nil && - s.IngressGateway.Leaf != nil && - s.IngressGateway.GatewayConfigLoaded && + s.IngressGateway.valid() && s.IngressGateway.HostsSet && s.IngressGateway.MeshConfigSet case structs.ServiceKindAPIGateway: // TODO Is this the proper set of things to validate? return s.Roots != nil && - s.APIGateway.Leaf != nil && - s.APIGateway.GatewayConfigLoaded && - s.APIGateway.BoundGatewayConfigLoaded && + s.APIGateway.valid() && s.APIGateway.MeshConfigSet default: return false diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index 028a3fd59d..7bbb7f7b87 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -36,6 +36,8 @@ const ( serviceResolversWatchID = "service-resolvers" gatewayServicesWatchID = "gateway-services" gatewayConfigWatchID = "gateway-config" + apiGatewayConfigWatchID = "api-gateway-config" + boundGatewayConfigWatchID = "bound-gateway-config" inlineCertificateConfigWatchID = "inline-certificate-config" routeConfigWatchID = "route-config" externalServiceIDPrefix = "external-service:" diff --git a/agent/proxycfg/testing_api_gateway.go b/agent/proxycfg/testing_api_gateway.go index 75229ccfdd..87ff58fbf0 100644 --- a/agent/proxycfg/testing_api_gateway.go +++ b/agent/proxycfg/testing_api_gateway.go @@ -6,9 +6,10 @@ package proxycfg import ( "fmt" + "github.com/mitchellh/go-testing-interface" + "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/consul/discoverychain" - "github.com/mitchellh/go-testing-interface" "github.com/hashicorp/consul/agent/configentry" "github.com/hashicorp/consul/agent/structs" @@ -49,13 +50,13 @@ func TestConfigSnapshotAPIGateway( Result: placeholderLeaf, }, { - CorrelationID: gatewayConfigWatchID, + CorrelationID: apiGatewayConfigWatchID, Result: &structs.ConfigEntryResponse{ Entry: entry, }, }, { - CorrelationID: gatewayConfigWatchID, + CorrelationID: boundGatewayConfigWatchID, Result: &structs.ConfigEntryResponse{ Entry: boundEntry, }, @@ -141,13 +142,13 @@ func TestConfigSnapshotAPIGateway_NilConfigEntry( Result: roots, }, { - CorrelationID: gatewayConfigWatchID, + CorrelationID: apiGatewayConfigWatchID, Result: &structs.ConfigEntryResponse{ Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist. }, }, { - CorrelationID: gatewayConfigWatchID, + CorrelationID: boundGatewayConfigWatchID, Result: &structs.ConfigEntryResponse{ Entry: nil, // The first watch on a config entry will return nil if the config entry doesn't exist. }, diff --git a/command/connect/envoy/bootstrap_tpl.go b/command/connect/envoy/bootstrap_tpl.go index 78d88f9b1d..26b8e2118b 100644 --- a/command/connect/envoy/bootstrap_tpl.go +++ b/command/connect/envoy/bootstrap_tpl.go @@ -281,10 +281,12 @@ const bootstrapTemplate = `{ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/CONSUL_GRPC_ADDR-with-https-scheme-enables-tls.golden b/command/connect/envoy/testdata/CONSUL_GRPC_ADDR-with-https-scheme-enables-tls.golden index a8ba4704f5..c8144ac72f 100644 --- a/command/connect/envoy/testdata/CONSUL_GRPC_ADDR-with-https-scheme-enables-tls.golden +++ b/command/connect/envoy/testdata/CONSUL_GRPC_ADDR-with-https-scheme-enables-tls.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/CONSUL_HTTP_ADDR-with-https-scheme-does-not-affect-grpc-tls.golden b/command/connect/envoy/testdata/CONSUL_HTTP_ADDR-with-https-scheme-does-not-affect-grpc-tls.golden index 5485256b4f..a89575d2c1 100644 --- a/command/connect/envoy/testdata/CONSUL_HTTP_ADDR-with-https-scheme-does-not-affect-grpc-tls.golden +++ b/command/connect/envoy/testdata/CONSUL_HTTP_ADDR-with-https-scheme-does-not-affect-grpc-tls.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/access-log-path.golden b/command/connect/envoy/testdata/access-log-path.golden index 18a28a491e..184a290b24 100644 --- a/command/connect/envoy/testdata/access-log-path.golden +++ b/command/connect/envoy/testdata/access-log-path.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/access-logs-enabled-custom.golden b/command/connect/envoy/testdata/access-logs-enabled-custom.golden index 6af720bdac..50531f89c2 100644 --- a/command/connect/envoy/testdata/access-logs-enabled-custom.golden +++ b/command/connect/envoy/testdata/access-logs-enabled-custom.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/access-logs-enabled.golden b/command/connect/envoy/testdata/access-logs-enabled.golden index 892cf46324..ea2e150607 100644 --- a/command/connect/envoy/testdata/access-logs-enabled.golden +++ b/command/connect/envoy/testdata/access-logs-enabled.golden @@ -219,10 +219,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/acl-enabled-and-token.golden b/command/connect/envoy/testdata/acl-enabled-and-token.golden index 3342f03904..a5427c7f24 100644 --- a/command/connect/envoy/testdata/acl-enabled-and-token.golden +++ b/command/connect/envoy/testdata/acl-enabled-and-token.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/acl-enabled-but-no-token.golden b/command/connect/envoy/testdata/acl-enabled-but-no-token.golden index 5485256b4f..a89575d2c1 100644 --- a/command/connect/envoy/testdata/acl-enabled-but-no-token.golden +++ b/command/connect/envoy/testdata/acl-enabled-but-no-token.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-PLAIN-and-CONSUL_GRPC_ADDR-TLS-is-tls.golden b/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-PLAIN-and-CONSUL_GRPC_ADDR-TLS-is-tls.golden index a8ba4704f5..c8144ac72f 100644 --- a/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-PLAIN-and-CONSUL_GRPC_ADDR-TLS-is-tls.golden +++ b/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-PLAIN-and-CONSUL_GRPC_ADDR-TLS-is-tls.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-TLS-and-CONSUL_GRPC_ADDR-PLAIN-is-plain.golden b/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-TLS-and-CONSUL_GRPC_ADDR-PLAIN-is-plain.golden index 5485256b4f..a89575d2c1 100644 --- a/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-TLS-and-CONSUL_GRPC_ADDR-PLAIN-is-plain.golden +++ b/command/connect/envoy/testdata/both-CONSUL_HTTP_ADDR-TLS-and-CONSUL_GRPC_ADDR-PLAIN-is-plain.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/defaults-nodemeta.golden b/command/connect/envoy/testdata/defaults-nodemeta.golden index 41788c3c42..ea65421109 100644 --- a/command/connect/envoy/testdata/defaults-nodemeta.golden +++ b/command/connect/envoy/testdata/defaults-nodemeta.golden @@ -185,10 +185,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/defaults.golden b/command/connect/envoy/testdata/defaults.golden index 5485256b4f..a89575d2c1 100644 --- a/command/connect/envoy/testdata/defaults.golden +++ b/command/connect/envoy/testdata/defaults.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/deprecated-grpc-addr-config.golden b/command/connect/envoy/testdata/deprecated-grpc-addr-config.golden index ab87aa9b43..e604c61d50 100644 --- a/command/connect/envoy/testdata/deprecated-grpc-addr-config.golden +++ b/command/connect/envoy/testdata/deprecated-grpc-addr-config.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/envoy-readiness-probe.golden b/command/connect/envoy/testdata/envoy-readiness-probe.golden index 444528e880..18680a3206 100644 --- a/command/connect/envoy/testdata/envoy-readiness-probe.golden +++ b/command/connect/envoy/testdata/envoy-readiness-probe.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/existing-ca-file.golden b/command/connect/envoy/testdata/existing-ca-file.golden index a8ba4704f5..c8144ac72f 100644 --- a/command/connect/envoy/testdata/existing-ca-file.golden +++ b/command/connect/envoy/testdata/existing-ca-file.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/existing-ca-path.golden b/command/connect/envoy/testdata/existing-ca-path.golden index e246dbb324..c0566ab351 100644 --- a/command/connect/envoy/testdata/existing-ca-path.golden +++ b/command/connect/envoy/testdata/existing-ca-path.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/extra_-multiple.golden b/command/connect/envoy/testdata/extra_-multiple.golden index 038e65d8d9..819994f0c0 100644 --- a/command/connect/envoy/testdata/extra_-multiple.golden +++ b/command/connect/envoy/testdata/extra_-multiple.golden @@ -206,10 +206,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/extra_-single.golden b/command/connect/envoy/testdata/extra_-single.golden index 14b8f08205..b1fb71997b 100644 --- a/command/connect/envoy/testdata/extra_-single.golden +++ b/command/connect/envoy/testdata/extra_-single.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/grpc-addr-env.golden b/command/connect/envoy/testdata/grpc-addr-env.golden index ab87aa9b43..e604c61d50 100644 --- a/command/connect/envoy/testdata/grpc-addr-env.golden +++ b/command/connect/envoy/testdata/grpc-addr-env.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/grpc-addr-flag.golden b/command/connect/envoy/testdata/grpc-addr-flag.golden index ab87aa9b43..e604c61d50 100644 --- a/command/connect/envoy/testdata/grpc-addr-flag.golden +++ b/command/connect/envoy/testdata/grpc-addr-flag.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/grpc-addr-unix-with-tls.golden b/command/connect/envoy/testdata/grpc-addr-unix-with-tls.golden index 89764c310a..2b902d48d4 100644 --- a/command/connect/envoy/testdata/grpc-addr-unix-with-tls.golden +++ b/command/connect/envoy/testdata/grpc-addr-unix-with-tls.golden @@ -196,10 +196,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/grpc-addr-unix.golden b/command/connect/envoy/testdata/grpc-addr-unix.golden index eb8841ae99..88e6e46bfb 100644 --- a/command/connect/envoy/testdata/grpc-addr-unix.golden +++ b/command/connect/envoy/testdata/grpc-addr-unix.golden @@ -183,10 +183,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/grpc-tls-addr-config.golden b/command/connect/envoy/testdata/grpc-tls-addr-config.golden index e79cf0455d..e65fc5f09c 100644 --- a/command/connect/envoy/testdata/grpc-tls-addr-config.golden +++ b/command/connect/envoy/testdata/grpc-tls-addr-config.golden @@ -197,10 +197,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway-address-specified.golden b/command/connect/envoy/testdata/ingress-gateway-address-specified.golden index cc2e504eaf..038c8fcc5d 100644 --- a/command/connect/envoy/testdata/ingress-gateway-address-specified.golden +++ b/command/connect/envoy/testdata/ingress-gateway-address-specified.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway-no-auto-register.golden b/command/connect/envoy/testdata/ingress-gateway-no-auto-register.golden index cddaa490dd..80f1f9a8b7 100644 --- a/command/connect/envoy/testdata/ingress-gateway-no-auto-register.golden +++ b/command/connect/envoy/testdata/ingress-gateway-no-auto-register.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway-nodemeta.golden b/command/connect/envoy/testdata/ingress-gateway-nodemeta.golden index d79739ed72..8a2d5f6276 100644 --- a/command/connect/envoy/testdata/ingress-gateway-nodemeta.golden +++ b/command/connect/envoy/testdata/ingress-gateway-nodemeta.golden @@ -274,10 +274,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway-register-with-service-and-proxy-id.golden b/command/connect/envoy/testdata/ingress-gateway-register-with-service-and-proxy-id.golden index 3149fc4b33..6192f674ac 100644 --- a/command/connect/envoy/testdata/ingress-gateway-register-with-service-and-proxy-id.golden +++ b/command/connect/envoy/testdata/ingress-gateway-register-with-service-and-proxy-id.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway-register-with-service-without-proxy-id.golden b/command/connect/envoy/testdata/ingress-gateway-register-with-service-without-proxy-id.golden index f8eb447541..92d80e033e 100644 --- a/command/connect/envoy/testdata/ingress-gateway-register-with-service-without-proxy-id.golden +++ b/command/connect/envoy/testdata/ingress-gateway-register-with-service-without-proxy-id.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/ingress-gateway.golden b/command/connect/envoy/testdata/ingress-gateway.golden index c4fba520ed..40a5c74d1b 100644 --- a/command/connect/envoy/testdata/ingress-gateway.golden +++ b/command/connect/envoy/testdata/ingress-gateway.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/prometheus-metrics-tls-ca-file.golden b/command/connect/envoy/testdata/prometheus-metrics-tls-ca-file.golden index 71b82250e3..c93286e64e 100644 --- a/command/connect/envoy/testdata/prometheus-metrics-tls-ca-file.golden +++ b/command/connect/envoy/testdata/prometheus-metrics-tls-ca-file.golden @@ -310,10 +310,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/prometheus-metrics-tls-ca-path.golden b/command/connect/envoy/testdata/prometheus-metrics-tls-ca-path.golden index 07e56ae878..272175f707 100644 --- a/command/connect/envoy/testdata/prometheus-metrics-tls-ca-path.golden +++ b/command/connect/envoy/testdata/prometheus-metrics-tls-ca-path.golden @@ -310,10 +310,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/prometheus-metrics.golden b/command/connect/envoy/testdata/prometheus-metrics.golden index 5adc8a9485..5806dd671f 100644 --- a/command/connect/envoy/testdata/prometheus-metrics.golden +++ b/command/connect/envoy/testdata/prometheus-metrics.golden @@ -273,10 +273,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/stats-config-override.golden b/command/connect/envoy/testdata/stats-config-override.golden index d2a2371f20..dd4ff53dd1 100644 --- a/command/connect/envoy/testdata/stats-config-override.golden +++ b/command/connect/envoy/testdata/stats-config-override.golden @@ -62,10 +62,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/telemetry-collector.golden b/command/connect/envoy/testdata/telemetry-collector.golden index 3977ce65bb..81ef486622 100644 --- a/command/connect/envoy/testdata/telemetry-collector.golden +++ b/command/connect/envoy/testdata/telemetry-collector.golden @@ -222,10 +222,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/token-arg.golden b/command/connect/envoy/testdata/token-arg.golden index ac057e9a60..816bf74875 100644 --- a/command/connect/envoy/testdata/token-arg.golden +++ b/command/connect/envoy/testdata/token-arg.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/token-env.golden b/command/connect/envoy/testdata/token-env.golden index ac057e9a60..816bf74875 100644 --- a/command/connect/envoy/testdata/token-env.golden +++ b/command/connect/envoy/testdata/token-env.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/token-file-arg.golden b/command/connect/envoy/testdata/token-file-arg.golden index ac057e9a60..816bf74875 100644 --- a/command/connect/envoy/testdata/token-file-arg.golden +++ b/command/connect/envoy/testdata/token-file-arg.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/token-file-env.golden b/command/connect/envoy/testdata/token-file-env.golden index ac057e9a60..816bf74875 100644 --- a/command/connect/envoy/testdata/token-file-env.golden +++ b/command/connect/envoy/testdata/token-file-env.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/xds-addr-config.golden b/command/connect/envoy/testdata/xds-addr-config.golden index ab87aa9b43..e604c61d50 100644 --- a/command/connect/envoy/testdata/xds-addr-config.golden +++ b/command/connect/envoy/testdata/xds-addr-config.golden @@ -184,10 +184,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/command/connect/envoy/testdata/zipkin-tracing-config.golden b/command/connect/envoy/testdata/zipkin-tracing-config.golden index fd52f5ae3f..dc4779a775 100644 --- a/command/connect/envoy/testdata/zipkin-tracing-config.golden +++ b/command/connect/envoy/testdata/zipkin-tracing-config.golden @@ -217,10 +217,12 @@ "dynamic_resources": { "lds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "cds_config": { "ads": {}, + "initial_fetch_timeout": "0s", "resource_api_version": "V3" }, "ads_config": { diff --git a/test/integration/consul-container/test/gateways/ingress_gateway_test.go b/test/integration/consul-container/test/gateways/ingress_gateway_test.go index 1a8958741c..fcadd3a0fa 100644 --- a/test/integration/consul-container/test/gateways/ingress_gateway_test.go +++ b/test/integration/consul-container/test/gateways/ingress_gateway_test.go @@ -62,11 +62,6 @@ func TestIngressGateway(t *testing.T) { ingressService, err := libservice.NewGatewayService(context.Background(), gwCfg, clientNode) require.NoError(t, err) - // this is deliberate - // internally, ingress gw have a 15s timeout before the /ready endpoint is available, - // then we need to wait for the health check to re-execute and propagate. - time.Sleep(45 * time.Second) - // We check this is healthy here because in the case of bringing up a new kube cluster, // it is not possible to create the config entry in advance. // The health checks must pass so the pod can start up.