From c10ba43eb3b85cee9af88fc01cd6056912e662df Mon Sep 17 00:00:00 2001 From: malizz Date: Mon, 13 Feb 2023 12:50:32 -0800 Subject: [PATCH] get clusters from route if listener uses RDS (#16243) --- troubleshoot/proxy/testdata/listeners.json | 90 -------- .../proxy/testdata/upstreams/config.json | 192 ++++++++++++++++++ troubleshoot/proxy/upstreams.go | 55 ++++- troubleshoot/proxy/upstreams_test.go | 51 ++++- 4 files changed, 281 insertions(+), 107 deletions(-) delete mode 100644 troubleshoot/proxy/testdata/listeners.json create mode 100644 troubleshoot/proxy/testdata/upstreams/config.json diff --git a/troubleshoot/proxy/testdata/listeners.json b/troubleshoot/proxy/testdata/listeners.json deleted file mode 100644 index 3cbcdebc4d..0000000000 --- a/troubleshoot/proxy/testdata/listeners.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "name": "outbound_listener:127.0.0.1:15001", - "address": { - "socket_address": { - "address": "127.0.0.1", - "port_value": 15001 - } - }, - "filter_chains": [ - { - "filter_chain_match": { - "prefix_ranges": [ - { - "address_prefix": "10.244.0.63", - "prefix_len": 32 - }, - { - "address_prefix": "10.244.0.64", - "prefix_len": 32 - } - ] - }, - "filters": [ - { - "name": "envoy.filters.network.tcp_proxy", - "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", - "stat_prefix": "upstream.foo.default.default.dc1", - "cluster": "passthrough~foo.default.dc1.internal.dc1.consul" - } - } - ] - }, - { - "filter_chain_match": { - "prefix_ranges": [ - { - "address_prefix": "10.96.5.96", - "prefix_len": 32 - }, - { - "address_prefix": "240.0.0.1", - "prefix_len": 32 - } - ] - }, - "filters": [ - { - "name": "envoy.filters.network.http_connection_manager", - "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", - "stat_prefix": "upstream.foo.default.default.dc1", - "route_config": { - "name": "foo", - "virtual_hosts": [ - { - "name": "foo.default.default.dc1", - "domains": [ - "*" - ], - "routes": [ - { - "match": { - "prefix": "/" - }, - "route": { - "cluster": "foo.default.dc1.internal.dc1.consul" - } - } - ] - } - ] - }, - "http_filters": [ - { - "name": "envoy.filters.http.router", - "typed_config": { - "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" - } - } - ], - "tracing": { - "random_sampling": {} - } - } - } - ] - } - ] -} \ No newline at end of file diff --git a/troubleshoot/proxy/testdata/upstreams/config.json b/troubleshoot/proxy/testdata/upstreams/config.json new file mode 100644 index 0000000000..735e329597 --- /dev/null +++ b/troubleshoot/proxy/testdata/upstreams/config.json @@ -0,0 +1,192 @@ +{ + "configs": [ + { + "@type": "type.googleapis.com/envoy.admin.v3.ListenersConfigDump", + "dynamic_listeners": [ + { + "name": "outbound_listener:127.0.0.1:15001", + "active_state": { + "version_info": "620f4e9a96cf706bce79f49d47dace3d52806d70b4775adda5f82c15e6aac31a", + "listener": { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "outbound_listener:127.0.0.1:15001", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 15001 + } + }, + "filter_chains": [ + { + "filter_chain_match": { + "prefix_ranges": [ + { + "address_prefix": "10.244.0.63", + "prefix_len": 32 + }, + { + "address_prefix": "10.244.0.64", + "prefix_len": 32 + } + ] + }, + "filters": [ + { + "name": "envoy.filters.network.tcp_proxy", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy", + "stat_prefix": "upstream.foo.default.default.dc1", + "cluster": "passthrough~foo.default.dc1.internal.dc1.consul" + } + } + ] + }, + { + "filter_chain_match": { + "prefix_ranges": [ + { + "address_prefix": "10.96.5.96", + "prefix_len": 32 + }, + { + "address_prefix": "240.0.0.1", + "prefix_len": 32 + } + ] + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "upstream.foo.default.default.dc1", + "route_config": { + "name": "foo", + "virtual_hosts": [ + { + "name": "foo.default.default.dc1", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "foo.default.dc1.internal.dc1.consul" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "random_sampling": {} + } + } + } + ] + }, + { + "filter_chain_match": { + "prefix_ranges": [ + { + "address_prefix": "10.4.6.160", + "prefix_len": 32 + }, + { + "address_prefix": "240.0.0.3", + "prefix_len": 32 + } + ] + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "upstream.backend.default.default.dc1", + "rds": { + "config_source": { + "ads": {}, + "resource_api_version": "V3" + }, + "route_config_name": "backend" + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "tracing": { + "random_sampling": {} + } + } + } + ] + } + ] + } + + } + } + ] + }, + { + "@type": "type.googleapis.com/envoy.admin.v3.RoutesConfigDump", + "dynamic_route_configs": [ + { + "version_info": "8d6d5cdcfdbb614ca333b13b5f4aadb14aba24094b3142108b60e0292ccfe19c", + "route_config": { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "backend", + "virtual_hosts": [ + { + "name": "backend", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "weighted_clusters": { + "clusters": [ + { + "name": "backend.default.dc1.internal.domain.consul", + "weight": 5000 + }, + { + "name": "backend2.default.dc1.internal.domain.consul", + "weight": 5000 + } + ], + "total_weight": 10000 + } + } + } + ] + } + ], + "validate_clusters": true + }, + "last_updated": "2023-02-09T17:38:12.738Z" + } + ] + } + ] +} \ No newline at end of file diff --git a/troubleshoot/proxy/upstreams.go b/troubleshoot/proxy/upstreams.go index 3c22daa60e..abd9711abf 100644 --- a/troubleshoot/proxy/upstreams.go +++ b/troubleshoot/proxy/upstreams.go @@ -1,12 +1,14 @@ package troubleshoot import ( + "fmt" envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" envoy_resource_v3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "google.golang.org/protobuf/proto" "github.com/hashicorp/consul/envoyextensions/extensioncommon" - "google.golang.org/protobuf/proto" ) type UpstreamIP struct { @@ -49,7 +51,7 @@ func (t *Troubleshoot) GetUpstreams() ([]string, []UpstreamIP, error) { return nil, nil, err } - upstream_ips, err = getUpstreamIPsFromFilterChain(l.GetFilterChains()) + upstream_ips, err = getUpstreamIPsFromFilterChain(l.GetFilterChains(), t.envoyConfigDump) if err != nil { return nil, nil, err } @@ -60,7 +62,9 @@ func (t *Troubleshoot) GetUpstreams() ([]string, []UpstreamIP, error) { return upstream_envoy_ids, upstream_ips, nil } -func getUpstreamIPsFromFilterChain(filterChains []*envoy_listener_v3.FilterChain) ([]UpstreamIP, error) { +func getUpstreamIPsFromFilterChain(filterChains []*envoy_listener_v3.FilterChain, + cfgDump *envoy_admin_v3.ConfigDump) ([]UpstreamIP, error) { + var err error if filterChains == nil { return []UpstreamIP{}, nil } @@ -71,7 +75,6 @@ func getUpstreamIPsFromFilterChain(filterChains []*envoy_listener_v3.FilterChain if fc.GetFilters() == nil { continue } - if fc.GetFilterChainMatch() == nil { continue } @@ -94,13 +97,22 @@ func getUpstreamIPsFromFilterChain(filterChains []*envoy_listener_v3.FilterChain } clusterNames := map[string]struct{}{} - if config := envoy_resource_v3.GetHTTPConnectionManager(filter); config != nil { isVirtual = true - cfg := config.GetRouteConfig() - clusterNames = extensioncommon.RouteClusterNames(cfg) + if cfg != nil { + clusterNames = extensioncommon.RouteClusterNames(cfg) + } else { + // If there are no route configs, look for RDS. + routeName := config.GetRds().GetRouteConfigName() + if routeName != "" { + clusterNames, err = getClustersFromRoutes(routeName, cfgDump) + if err != nil { + return nil, fmt.Errorf("error in getting clusters for route %q: %w", routeName, err) + } + } + } } if config := extensioncommon.GetTCPProxy(filter); config != nil { if config.GetCluster() != "" { @@ -118,3 +130,32 @@ func getUpstreamIPsFromFilterChain(filterChains []*envoy_listener_v3.FilterChain return upstreamIPs, nil } + +func getClustersFromRoutes(routeName string, cfgDump *envoy_admin_v3.ConfigDump) (map[string]struct{}, error) { + + for _, cfg := range cfgDump.Configs { + switch cfg.TypeUrl { + case routes: + rcd := &envoy_admin_v3.RoutesConfigDump{} + + err := proto.Unmarshal(cfg.GetValue(), rcd) + if err != nil { + return nil, err + } + + for _, route := range rcd.GetDynamicRouteConfigs() { + + routeConfig := &envoy_route_v3.RouteConfiguration{} + err = proto.Unmarshal(route.GetRouteConfig().GetValue(), routeConfig) + if err != nil { + return nil, err + } + + if routeConfig.GetName() == routeName { + return extensioncommon.RouteClusterNames(routeConfig), nil + } + } + } + } + return nil, nil +} diff --git a/troubleshoot/proxy/upstreams_test.go b/troubleshoot/proxy/upstreams_test.go index 2ae5d9fe28..6493b5349d 100644 --- a/troubleshoot/proxy/upstreams_test.go +++ b/troubleshoot/proxy/upstreams_test.go @@ -1,17 +1,18 @@ package troubleshoot import ( - "io" - "os" - "testing" - + envoy_admin_v3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "io" + "os" + "testing" ) func TestGetUpstreamIPsFromFilterChain(t *testing.T) { - file, err := os.Open("testdata/listeners.json") + file, err := os.Open("testdata/upstreams/config.json") require.NoError(t, err) jsonBytes, err := io.ReadAll(file) require.NoError(t, err) @@ -37,17 +38,47 @@ func TestGetUpstreamIPsFromFilterChain(t *testing.T) { "foo.default.dc1.internal.dc1.consul": {}, }, }, + { + IPs: []string{ + "10.4.6.160", + "240.0.0.3", + }, + IsVirtual: true, + ClusterNames: map[string]struct{}{ + "backend.default.dc1.internal.domain.consul": {}, + "backend2.default.dc1.internal.domain.consul": {}, + }, + }, } - var listener envoy_listener_v3.Listener + var upstreamIPs []UpstreamIP + cfgDump := &envoy_admin_v3.ConfigDump{} unmarshal := &protojson.UnmarshalOptions{ DiscardUnknown: true, } - err = unmarshal.Unmarshal(jsonBytes, &listener) + err = unmarshal.Unmarshal(jsonBytes, cfgDump) require.NoError(t, err) - upstream_ips, err := getUpstreamIPsFromFilterChain(listener.GetFilterChains()) - require.NoError(t, err) + for _, cfg := range cfgDump.Configs { + switch cfg.TypeUrl { + case listeners: + lcd := &envoy_admin_v3.ListenersConfigDump{} - require.Equal(t, expected, upstream_ips) + err := proto.Unmarshal(cfg.GetValue(), lcd) + require.NoError(t, err) + + for _, listener := range lcd.GetDynamicListeners() { + l := &envoy_listener_v3.Listener{} + err = proto.Unmarshal(listener.GetActiveState().GetListener().GetValue(), l) + require.NoError(t, err) + + upstreamIPs, err = getUpstreamIPsFromFilterChain(l.GetFilterChains(), cfgDump) + require.NoError(t, err) + + } + } + } + + require.NoError(t, err) + require.Equal(t, expected, upstreamIPs) }