connect: update supported envoy versions to 1.18.2, 1.17.2, 1.16.3, and 1.15.4 (#10101)

The only thing that needed fixing up pertained to this section of the 1.18.x release notes:

> grpc_stats: the default value for stats_for_all_methods is switched from true to false, in order to avoid possible memory exhaustion due to an untrusted downstream sending a large number of unique method names. The previous default value was deprecated in version 1.14.0. This only changes the behavior when the value is not set. The previous behavior can be used by setting the value to true. This behavior change by be overridden by setting runtime feature envoy.deprecated_features.grpc_stats_filter_enable_stats_for_all_methods_by_default.

For now to maintain status-quo I'm explicitly setting `stats_for_all_methods=true` in all versions to avoid relying upon the default.

Additionally the naming of the emitted metrics for these gRPC requests changed slightly so the integration test assertions for `case-grpc` needed adjusting.

.changelog/10101.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+connect: update supported envoy versions to 1.18.2, 1.17.2, 1.16.3, 1.15.4
+```

.circleci/config.yml

@@ -767,14 +767,14 @@ jobs:
           command: make test-coverage-ci
       - run: *notify-slack-failure
 
-  envoy-integration-test-1_14_6: &ENVOY_TESTS
+  envoy-integration-test-1_15_4: &ENVOY_TESTS
     docker:
       # We only really need bash and docker-compose which is installed on all
       # Circle images but pick Go since we have to pick one of them.
       - image: *GOLANG_IMAGE
     parallelism: 2
     environment:
-      ENVOY_VERSION: "1.14.6"
+      ENVOY_VERSION: "1.15.4"
     steps: &ENVOY_INTEGRATION_TEST_STEPS
       - checkout
       # Get go binary from workspace
@@ -807,38 +807,32 @@ jobs:
           path: *TEST_RESULTS_DIR
       - run: *notify-slack-failure
 
-  envoy-integration-test-1_14_6-v2compat:
+  envoy-integration-test-1_15_4-v2compat:
     <<: *ENVOY_TESTS
     environment:
-      ENVOY_VERSION: "1.14.6"
+      ENVOY_VERSION: "1.15.4"
       TEST_V2_XDS: "1"
 
-  envoy-integration-test-1_15_3:
+  envoy-integration-test-1_16_3:
     <<: *ENVOY_TESTS
     environment:
-      ENVOY_VERSION: "1.15.3"
+      ENVOY_VERSION: "1.16.3"
 
-  envoy-integration-test-1_15_3-v2compat:
+  envoy-integration-test-1_16_3-v2compat:
     <<: *ENVOY_TESTS
     environment:
-      ENVOY_VERSION: "1.15.3"
+      ENVOY_VERSION: "1.16.3"
       TEST_V2_XDS: "1"
 
-  envoy-integration-test-1_16_2:
+  envoy-integration-test-1_17_2:
     <<: *ENVOY_TESTS
     environment:
-      ENVOY_VERSION: "1.16.2"
+      ENVOY_VERSION: "1.17.2"
 
-  envoy-integration-test-1_16_2-v2compat:
+  envoy-integration-test-1_18_2:
     <<: *ENVOY_TESTS
     environment:
-      ENVOY_VERSION: "1.16.2"
-      TEST_V2_XDS: "1"
-
-  envoy-integration-test-1_17_0:
-    <<: *ENVOY_TESTS
-    environment:
-      ENVOY_VERSION: "1.17.0"
+      ENVOY_VERSION: "1.18.2"
 
   # run integration tests for the connect ca providers
   test-connect-ca-providers:
@@ -1047,25 +1041,22 @@ workflows:
       - nomad-integration-0_8:
           requires:
             - dev-build
-      - envoy-integration-test-1_14_6:
+      - envoy-integration-test-1_15_4:
           requires:
             - dev-build
-      - envoy-integration-test-1_14_6-v2compat:
+      - envoy-integration-test-1_15_4-v2compat:
           requires:
             - dev-build
-      - envoy-integration-test-1_15_3:
+      - envoy-integration-test-1_16_3:
           requires:
             - dev-build
-      - envoy-integration-test-1_15_3-v2compat:
+      - envoy-integration-test-1_16_3-v2compat:
           requires:
             - dev-build
-      - envoy-integration-test-1_16_2:
+      - envoy-integration-test-1_17_2:
           requires:
             - dev-build
-      - envoy-integration-test-1_16_2-v2compat:
-          requires:
-            - dev-build
-      - envoy-integration-test-1_17_0:
+      - envoy-integration-test-1_18_2:
           requires:
             - dev-build
 

agent/xds/clusters_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/xds/proxysupport"
+	"github.com/hashicorp/consul/lib/stringslice"
 	"github.com/hashicorp/consul/sdk/testutil"
 )
 
@@ -642,6 +643,7 @@ func TestClustersFromSnapshot(t *testing.T) {
 	}
 
 	latestEnvoyVersion := proxysupport.EnvoyVersions[0]
+	latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0]
 	for _, envoyVersion := range proxysupport.EnvoyVersions {
 		sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
 		require.NoError(t, err)
@@ -686,6 +688,9 @@ func TestClustersFromSnapshot(t *testing.T) {
 					})
 
 					t.Run("v2-compat", func(t *testing.T) {
+						if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) {
+							t.Skip()
+						}
 						respV2, err := convertDiscoveryResponseToV2(r)
 						require.NoError(t, err)
 
@@ -698,7 +703,7 @@ func TestClustersFromSnapshot(t *testing.T) {
 
 						gName += ".v2compat"
 
-						require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
+						require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON)
 					})
 				})
 			}

agent/xds/endpoints_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/xds/proxysupport"
+	"github.com/hashicorp/consul/lib/stringslice"
 	"github.com/hashicorp/consul/sdk/testutil"
 )
 
@@ -566,6 +567,7 @@ func TestEndpointsFromSnapshot(t *testing.T) {
 	}
 
 	latestEnvoyVersion := proxysupport.EnvoyVersions[0]
+	latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0]
 	for _, envoyVersion := range proxysupport.EnvoyVersions {
 		sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
 		require.NoError(t, err)
@@ -609,6 +611,9 @@ func TestEndpointsFromSnapshot(t *testing.T) {
 					})
 
 					t.Run("v2-compat", func(t *testing.T) {
+						if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) {
+							t.Skip()
+						}
 						respV2, err := convertDiscoveryResponseToV2(r)
 						require.NoError(t, err)
 
@@ -621,7 +626,7 @@ func TestEndpointsFromSnapshot(t *testing.T) {
 
 						gName += ".v2compat"
 
-						require.JSONEq(t, goldenEnvoy(t, filepath.Join("endpoints", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
+						require.JSONEq(t, goldenEnvoy(t, filepath.Join("endpoints", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON)
 					})
 				})
 			}

agent/xds/envoy_versioning.go

@@ -11,7 +11,7 @@ import (
 var (
 	// minSupportedVersion is the oldest mainline version we support. This should always be
 	// the zero'th point release of the last element of proxysupport.EnvoyVersions.
-	minSupportedVersion = version.Must(version.NewVersion("1.14.0"))
+	minSupportedVersion = version.Must(version.NewVersion("1.15.0"))
 
 	minVersionAllowingEmptyGatewayClustersWithIncrementalXDS = version.Must(version.NewVersion("1.16.0"))
 	minVersionAllowingMultipleIncrementalXDSChanges          = version.Must(version.NewVersion("1.16.0"))

agent/xds/envoy_versioning_test.go

@@ -98,12 +98,19 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) {
 		"1.13.5": {expectErr: "Envoy 1.13.5 " + errTooOld},
 		"1.13.6": {expectErr: "Envoy 1.13.6 " + errTooOld},
 		"1.13.7": {expectErr: "Envoy 1.13.7 " + errTooOld},
+		"1.14.0": {expectErr: "Envoy 1.14.0 " + errTooOld},
+		"1.14.1": {expectErr: "Envoy 1.14.1 " + errTooOld},
+		"1.14.2": {expectErr: "Envoy 1.14.2 " + errTooOld},
+		"1.14.3": {expectErr: "Envoy 1.14.3 " + errTooOld},
+		"1.14.4": {expectErr: "Envoy 1.14.4 " + errTooOld},
+		"1.14.5": {expectErr: "Envoy 1.14.5 " + errTooOld},
+		"1.14.6": {expectErr: "Envoy 1.14.6 " + errTooOld},
+		"1.14.7": {expectErr: "Envoy 1.14.7 " + errTooOld},
 	}
 
 	// Insert a bunch of valid versions.
 	for _, v := range []string{
-		"1.14.1", "1.14.2", "1.14.3", "1.14.4", "1.14.5", "1.14.6",
-		"1.15.0", "1.15.1", "1.15.2", "1.15.3",
+		"1.15.0", "1.15.1", "1.15.2", "1.15.3", "1.15.4",
 	} {
 		cases[v] = testcase{expect: supportedProxyFeatures{
 			GatewaysNeedStubClusterWhenEmptyWithIncrementalXDS: true,
@@ -111,8 +118,9 @@ func TestDetermineSupportedProxyFeaturesFromString(t *testing.T) {
 		}}
 	}
 	for _, v := range []string{
-		"1.16.0", "1.16.1", "1.16.2",
-		"1.17.0",
+		"1.16.0", "1.16.1", "1.16.2", "1.16.3",
+		"1.17.0", "1.17.1", "1.17.2",
+		"1.18.0", "1.18.1", "1.18.2",
 	} {
 		cases[v] = testcase{expect: supportedProxyFeatures{}}
 	}

agent/xds/listeners.go

@@ -14,11 +14,11 @@ import (
 	envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	envoy_grpc_stats_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/grpc_stats/v3"
 	envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
 	envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
-	"github.com/hashicorp/consul/sdk/iptables"
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
@@ -29,6 +29,7 @@ import (
 	"github.com/hashicorp/consul/agent/connect"
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/sdk/iptables"
 )
 
 // listenersFromSnapshot returns the xDS API representation of the "listeners" in the snapshot.
@@ -1581,6 +1582,24 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error)
 		cfg.HttpFilters = append([]*envoy_http_v3.HttpFilter{{
 			Name: "envoy.filters.http.grpc_http1_bridge",
 		}}, cfg.HttpFilters...)
+
+		// In envoy 1.14.x the default value "stats_for_all_methods=true" was
+		// deprecated, and was changed to "false" in 1.18.x. Avoid using the
+		// default. TODO: we may want to expose this to users somehow easily.
+		grpcStatsFilter, err := makeEnvoyHTTPFilter(
+			"envoy.filters.http.grpc_stats",
+			&envoy_grpc_stats_v3.FilterConfig{
+				PerMethodStatSpecifier: &envoy_grpc_stats_v3.FilterConfig_StatsForAllMethods{
+					StatsForAllMethods: makeBoolValue(true),
+				},
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+		cfg.HttpFilters = append([]*envoy_http_v3.HttpFilter{
+			grpcStatsFilter,
+		}, cfg.HttpFilters...)
 	}
 
 	return makeFilter("envoy.filters.network.http_connection_manager", cfg)

agent/xds/listeners_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/xds/proxysupport"
+	"github.com/hashicorp/consul/lib/stringslice"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/types"
 )
@@ -554,6 +555,7 @@ func TestListenersFromSnapshot(t *testing.T) {
 	}
 
 	latestEnvoyVersion := proxysupport.EnvoyVersions[0]
+	latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0]
 	for _, envoyVersion := range proxysupport.EnvoyVersions {
 		sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
 		require.NoError(t, err)
@@ -603,6 +605,9 @@ func TestListenersFromSnapshot(t *testing.T) {
 					})
 
 					t.Run("v2-compat", func(t *testing.T) {
+						if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) {
+							t.Skip()
+						}
 						respV2, err := convertDiscoveryResponseToV2(r)
 						require.NoError(t, err)
 
@@ -615,7 +620,7 @@ func TestListenersFromSnapshot(t *testing.T) {
 
 						gName += ".v2compat"
 
-						require.JSONEq(t, goldenEnvoy(t, filepath.Join("listeners", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
+						require.JSONEq(t, goldenEnvoy(t, filepath.Join("listeners", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON)
 					})
 				})
 			}

agent/xds/proxysupport/proxysupport.go

@@ -7,8 +7,13 @@ package proxysupport
 //
 // see: https://www.consul.io/docs/connect/proxies/envoy#supported-versions
 var EnvoyVersions = []string{
-	"1.17.0",
-	"1.16.2",
-	"1.15.3",
-	"1.14.6",
+	"1.18.2",
+	"1.17.2",
+	"1.16.3",
+	"1.15.4",
+}
+
+var EnvoyVersionsV2 = []string{
+	"1.16.3",
+	"1.15.4",
 }

agent/xds/routes_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/agent/xds/proxysupport"
+	"github.com/hashicorp/consul/lib/stringslice"
 	"github.com/hashicorp/consul/sdk/testutil"
 )
 
@@ -238,6 +239,7 @@ func TestRoutesFromSnapshot(t *testing.T) {
 	}
 
 	latestEnvoyVersion := proxysupport.EnvoyVersions[0]
+	latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0]
 	for _, envoyVersion := range proxysupport.EnvoyVersions {
 		sf, err := determineSupportedProxyFeaturesFromString(envoyVersion)
 		require.NoError(t, err)
@@ -280,6 +282,9 @@ func TestRoutesFromSnapshot(t *testing.T) {
 					})
 
 					t.Run("v2-compat", func(t *testing.T) {
+						if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) {
+							t.Skip()
+						}
 						respV2, err := convertDiscoveryResponseToV2(r)
 						require.NoError(t, err)
 
@@ -292,7 +297,7 @@ func TestRoutesFromSnapshot(t *testing.T) {
 
 						gName += ".v2compat"
 
-						require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON)
+						require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON)
 					})
 				})
 			}