xds: only try to create an ipv6 expose checks listener if ipv6 is supported by the kernel (#9765)

Fixes #9311

This only fails if the kernel has ipv6 hard-disabled. It is not sufficient to merely not provide an ipv6 address for a network interface.
This commit is contained in:
R.B. Boyer 2021-02-19 14:38:43 -06:00 committed by hashicorp-ci
parent 510ccae055
commit 0fa96a2fa8
15 changed files with 674 additions and 5 deletions

3
.changelog/9765.txt Normal file
View File

@ -0,0 +1,3 @@
```release-note:improvement
xds: only try to create an ipv6 expose checks listener if ipv6 is supported by the kernel
```

View File

@ -688,12 +688,22 @@ func (s *Server) makeExposedCheckListener(cfgSnap *proxycfg.ConfigSnapshot, clus
advertiseLen = 128
}
ranges := make([]*envoycore.CidrRange, 0, 3)
ranges = append(ranges,
&envoycore.CidrRange{AddressPrefix: "127.0.0.1", PrefixLen: &wrappers.UInt32Value{Value: 8}},
&envoycore.CidrRange{AddressPrefix: advertise, PrefixLen: &wrappers.UInt32Value{Value: uint32(advertiseLen)}},
)
if ok, err := kernelSupportsIPv6(); err != nil {
return nil, err
} else if ok {
ranges = append(ranges,
&envoycore.CidrRange{AddressPrefix: "::1", PrefixLen: &wrappers.UInt32Value{Value: 128}},
)
}
chain.FilterChainMatch = &envoylistener.FilterChainMatch{
SourcePrefixRanges: []*envoycore.CidrRange{
{AddressPrefix: "127.0.0.1", PrefixLen: &wrappers.UInt32Value{Value: 8}},
{AddressPrefix: "::1", PrefixLen: &wrappers.UInt32Value{Value: 128}},
{AddressPrefix: advertise, PrefixLen: &wrappers.UInt32Value{Value: uint32(advertiseLen)}},
},
SourcePrefixRanges: ranges,
}
}

View File

@ -6,6 +6,7 @@ import (
"sort"
"testing"
"text/template"
"time"
envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
"github.com/envoyproxy/go-control-plane/pkg/wellknown"
@ -16,6 +17,7 @@ import (
"github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/xds/proxysupport"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/types"
)
func TestListenersFromSnapshot(t *testing.T) {
@ -28,6 +30,7 @@ func TestListenersFromSnapshot(t *testing.T) {
// test input.
setup func(snap *proxycfg.ConfigSnapshot)
overrideGoldenName string
serverSetup func(*Server)
}{
{
name: "defaults",
@ -292,6 +295,38 @@ func TestListenersFromSnapshot(t *testing.T) {
}
},
},
{
// NOTE: if IPv6 is not supported in the kernel per
// kernelSupportsIPv6() then this test will fail because the golden
// files were generated assuming ipv6 support was present
name: "expose-checks",
create: proxycfg.TestConfigSnapshotExposeConfig,
setup: func(snap *proxycfg.ConfigSnapshot) {
snap.Proxy.Expose = structs.ExposeConfig{
Checks: true,
}
},
serverSetup: func(s *Server) {
s.CfgFetcher = configFetcherFunc(func() string {
return "192.0.2.1"
})
s.CheckFetcher = httpCheckFetcherFunc(func(sid structs.ServiceID) []structs.CheckType {
if sid != structs.NewServiceID("web", nil) {
return nil
}
return []structs.CheckType{{
CheckID: types.CheckID("http"),
Name: "http",
HTTP: "http://127.0.0.1:8181/debug",
ProxyHTTP: "http://:21500/debug",
Method: "GET",
Interval: 10 * time.Second,
Timeout: 1 * time.Second,
}}
})
},
},
{
name: "mesh-gateway",
create: proxycfg.TestConfigSnapshotMeshGateway,
@ -507,6 +542,9 @@ func TestListenersFromSnapshot(t *testing.T) {
// Need server just for logger dependency
s := Server{Logger: testutil.Logger(t)}
if tt.serverSetup != nil {
tt.serverSetup(&s)
}
cInfo := connectionInfo{
Token: "my-token",
@ -765,3 +803,19 @@ func customHTTPListenerJSON(t *testing.T, opts customHTTPListenerJSONOptions) st
require.NoError(t, customHTTPListenerJSONTemplate.Execute(&buf, opts))
return buf.String()
}
type httpCheckFetcherFunc func(serviceID structs.ServiceID) []structs.CheckType
var _ HTTPCheckFetcher = (httpCheckFetcherFunc)(nil)
func (f httpCheckFetcherFunc) ServiceHTTPBasedChecks(serviceID structs.ServiceID) []structs.CheckType {
return f(serviceID)
}
type configFetcherFunc func() string
var _ ConfigFetcher = (configFetcherFunc)(nil)
func (f configFetcherFunc) AdvertiseAddrLAN() string {
return f()
}

View File

@ -0,0 +1,7 @@
// +build !linux
package xds
func kernelSupportsIPv6() (bool, error) {
return true, nil
}

35
agent/xds/net_linux.go Normal file
View File

@ -0,0 +1,35 @@
// +build linux
package xds
import (
"fmt"
"os"
"sync"
)
const ipv6SupportProcFile = "/proc/net/if_inet6"
var (
ipv6SupportOnce sync.Once
ipv6Supported bool
ipv6SupportedErr error
)
func kernelSupportsIPv6() (bool, error) {
ipv6SupportOnce.Do(func() {
ipv6Supported, ipv6SupportedErr = checkIfKernelSupportsIPv6()
})
return ipv6Supported, ipv6SupportedErr
}
func checkIfKernelSupportsIPv6() (bool, error) {
_, err := os.Stat(ipv6SupportProcFile)
if os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, fmt.Errorf("error checking for ipv6 support file %s: %w", ipv6SupportProcFile, err)
}
return true, nil
}

View File

@ -0,0 +1,109 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "exposed_path_debug:1.2.3.4:21500",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 21500
}
},
"filterChains": [
{
"filterChainMatch": {
"sourcePrefixRanges": [
{
"addressPrefix": "127.0.0.1",
"prefixLen": 8
},
{
"addressPrefix": "192.0.2.1",
"prefixLen": 32
},
{
"addressPrefix": "::1",
"prefixLen": 128
}
]
},
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"route_config": {
"name": "exposed_path_filter_debug_21500",
"virtual_hosts": [
{
"domains": [
"*"
],
"name": "exposed_path_filter_debug_21500",
"routes": [
{
"match": {
"path": "/debug"
},
"route": {
"cluster": "exposed_cluster_8181"
}
}
]
}
]
},
"stat_prefix": "exposed_path_filter_debug_21500",
"tracing": {
"random_sampling": {
}
}
}
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "public_listener:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"tlsContext": {
"requireClientCertificate": true
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"config": {
"rules": {
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "local_app",
"stat_prefix": "public_listener"
}
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,109 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "exposed_path_debug:1.2.3.4:21500",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 21500
}
},
"filterChains": [
{
"filterChainMatch": {
"sourcePrefixRanges": [
{
"addressPrefix": "127.0.0.1",
"prefixLen": 8
},
{
"addressPrefix": "192.0.2.1",
"prefixLen": 32
},
{
"addressPrefix": "::1",
"prefixLen": 128
}
]
},
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"route_config": {
"name": "exposed_path_filter_debug_21500",
"virtual_hosts": [
{
"domains": [
"*"
],
"name": "exposed_path_filter_debug_21500",
"routes": [
{
"match": {
"path": "/debug"
},
"route": {
"cluster": "exposed_cluster_8181"
}
}
]
}
]
},
"stat_prefix": "exposed_path_filter_debug_21500",
"tracing": {
"random_sampling": {
}
}
}
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "public_listener:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"tlsContext": {
"requireClientCertificate": true
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"config": {
"rules": {
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "local_app",
"stat_prefix": "public_listener"
}
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,109 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "exposed_path_debug:1.2.3.4:21500",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 21500
}
},
"filterChains": [
{
"filterChainMatch": {
"sourcePrefixRanges": [
{
"addressPrefix": "127.0.0.1",
"prefixLen": 8
},
{
"addressPrefix": "192.0.2.1",
"prefixLen": 32
},
{
"addressPrefix": "::1",
"prefixLen": 128
}
]
},
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"route_config": {
"name": "exposed_path_filter_debug_21500",
"virtual_hosts": [
{
"domains": [
"*"
],
"name": "exposed_path_filter_debug_21500",
"routes": [
{
"match": {
"path": "/debug"
},
"route": {
"cluster": "exposed_cluster_8181"
}
}
]
}
]
},
"stat_prefix": "exposed_path_filter_debug_21500",
"tracing": {
"random_sampling": {
}
}
}
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "public_listener:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"tlsContext": {
"requireClientCertificate": true
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"config": {
"rules": {
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "local_app",
"stat_prefix": "public_listener"
}
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,109 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "exposed_path_debug:1.2.3.4:21500",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 21500
}
},
"filterChains": [
{
"filterChainMatch": {
"sourcePrefixRanges": [
{
"addressPrefix": "127.0.0.1",
"prefixLen": 8
},
{
"addressPrefix": "192.0.2.1",
"prefixLen": 32
},
{
"addressPrefix": "::1",
"prefixLen": 128
}
]
},
"filters": [
{
"name": "envoy.http_connection_manager",
"config": {
"http_filters": [
{
"name": "envoy.router"
}
],
"route_config": {
"name": "exposed_path_filter_debug_21500",
"virtual_hosts": [
{
"domains": [
"*"
],
"name": "exposed_path_filter_debug_21500",
"routes": [
{
"match": {
"path": "/debug"
},
"route": {
"cluster": "exposed_cluster_8181"
}
}
]
}
]
},
"stat_prefix": "exposed_path_filter_debug_21500",
"tracing": {
"random_sampling": {
}
}
}
}
]
}
]
},
{
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "public_listener:1.2.3.4:8080",
"address": {
"socketAddress": {
"address": "1.2.3.4",
"portValue": 8080
}
},
"filterChains": [
{
"tlsContext": {
"requireClientCertificate": true
},
"filters": [
{
"name": "envoy.filters.network.rbac",
"config": {
"rules": {
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.tcp_proxy",
"config": {
"cluster": "local_app",
"stat_prefix": "public_listener"
}
}
]
}
]
}
],
"typeUrl": "type.googleapis.com/envoy.api.v2.Listener",
"nonce": "00000001"
}

View File

@ -0,0 +1,4 @@
#!/bin/bash
snapshot_envoy_admin localhost:19000 s1 || true
snapshot_envoy_admin localhost:19001 s2 || true

View File

@ -0,0 +1,16 @@
services {
name = "s1"
port = 8080
connect {
sidecar_service {
proxy {
upstreams = [
{
destination_name = "s2"
local_bind_port = 5000
}
]
}
}
}
}

View File

@ -0,0 +1,22 @@
services {
name = "s2"
port = 8181
connect {
sidecar_service {
proxy {
expose {
checks = true
}
}
}
}
checks = [
{
name = "http"
http = "http://127.0.0.1:8181/debug"
method = "GET"
interval = "10s"
timeout = "1s"
},
]
}

View File

@ -0,0 +1,8 @@
#!/bin/bash
set -eEuo pipefail
register_services primary
gen_envoy_bootstrap s1 19000 primary
gen_envoy_bootstrap s2 19001 primary

View File

@ -0,0 +1,41 @@
#!/usr/bin/env bats
load helpers
@test "s1 proxy is running correct version" {
assert_envoy_version 19000
}
@test "s1 proxy admin is up on :19000" {
retry_default curl -f -s localhost:19000/stats -o /dev/null
}
@test "s2 proxy admin is up on :19001" {
retry_default curl -f -s localhost:19001/stats -o /dev/null
}
@test "s1 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21000 s1
}
@test "s2 proxy listener should be up and have right cert" {
assert_proxy_presents_cert_uri localhost:21001 s2
}
@test "s2 proxy should be healthy" {
assert_service_has_healthy_instances s2 1
}
@test "s1 upstream should have healthy endpoints for s2" {
assert_upstream_has_endpoints_in_status 127.0.0.1:19000 s2.default.primary HEALTHY 1
}
@test "s1 upstream should be able to connect to s2" {
run retry_default curl -s -f -d hello localhost:5000
[ "$status" -eq 0 ]
[ "$output" = "hello" ]
}
@test "s2 exposes checks on a new listener" {
assert_envoy_expose_checks_listener_count localhost:19001 /debug
}

View File

@ -152,6 +152,39 @@ function assert_envoy_version {
echo $VERSION | grep "/$ENVOY_VERSION/"
}
function assert_envoy_expose_checks_listener_count {
local HOSTPORT=$1
local EXPECT_PATH=$2
# scrape this once
BODY=$(get_envoy_expose_checks_listener_once $HOSTPORT)
echo "BODY = $BODY"
CHAINS=$(echo "$BODY" | jq '.active_state.listener.filter_chains | length')
echo "CHAINS = $CHAINS (expect 1)"
[ "${CHAINS:-0}" -eq 1 ]
RANGES=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filter_chain_match.source_prefix_ranges | length')
echo "RANGES = $RANGES (expect 3)"
# note: if IPv6 is not supported in the kernel per
# agent/xds:kernelSupportsIPv6() then this will only be 2
[ "${RANGES:-0}" -eq 3 ]
HCM=$(echo "$BODY" | jq '.active_state.listener.filter_chains[0].filters[0]')
HCM_NAME=$(echo "$HCM" | jq -r '.name')
HCM_PATH=$(echo "$HCM" | jq -r '.config.route_config.virtual_hosts[0].routes[0].match.path')
echo "HCM = $HCM"
[ "${HCM_NAME:-}" == "envoy.http_connection_manager" ]
[ "${HCM_PATH:-}" == "${EXPECT_PATH}" ]
}
function get_envoy_expose_checks_listener_once {
local HOSTPORT=$1
run curl -s -f $HOSTPORT/config_dump
[ "$status" -eq 0 ]
echo "$output" | jq --raw-output '.configs[] | select(.["@type"] == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump") | .dynamic_listeners[] | select(.name | startswith("exposed_path_"))'
}
function assert_envoy_http_rbac_policy_count {
local HOSTPORT=$1
local EXPECT_COUNT=$2