Implement traffic redirection exclusion based on proxy config and user-provided values (#10134)

* Use proxy outbound port from TransparentProxyConfig if provided
* If -proxy-id is provided to the redirect-traffic command, exclude any listener ports
  from inbound traffic redirection. This includes envoy_prometheus_bind_addr,
  envoy_stats_bind_addr, and the ListenerPort from the Expose configuration.
* Allow users to provide additional inbound and outbound ports, outbound CIDRs
  and additional user IDs to be excluded from traffic redirection.
  This affects both the traffic-redirect command and the iptables SDK package.
This commit is contained in:
Iryna Shustava 2021-04-29 09:21:15 -07:00 committed by hc-github-team-consul-core
parent f79302d44a
commit f383452e87
6 changed files with 485 additions and 11 deletions

11
.changelog/10134.txt Normal file
View File

@ -0,0 +1,11 @@
```release-note:feature
cli: Add additional flags to the `consul connect redirect-traffic` command to allow excluding inbound and outbound ports,
outbound CIDRs, and additional user IDs from traffic redirection.
```
```release-note:feature
cli: Automatically exclude ports from `envoy_prometheus_bind_addr`, `envoy_stats_bind_addr`, and `ListenerPort` from `Expose` config
from inbound traffic redirection rules if `proxy-id` flag is provided to the `consul connect redirect-traffic` command.
```
```release-note:feature
sdk: Allow excluding inbound and outbound ports, outbound CIDRs, and additional user IDs from traffic redirection in the `iptables` package.
```

View File

@ -3,6 +3,8 @@ package redirecttraffic
import (
"flag"
"fmt"
"net"
"strconv"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/flags"
@ -38,6 +40,10 @@ type cmd struct {
proxyID string
proxyInboundPort int
proxyOutboundPort int
excludeInboundPorts []string
excludeOutboundPorts []string
excludeOutboundCIDRs []string
excludeUIDs []string
}
func (c *cmd) init() {
@ -48,6 +54,14 @@ func (c *cmd) init() {
c.flags.IntVar(&c.proxyInboundPort, "proxy-inbound-port", 0, "The inbound port that the proxy is listening on.")
c.flags.IntVar(&c.proxyOutboundPort, "proxy-outbound-port", iptables.DefaultTProxyOutboundPort,
"The outbound port that the proxy is listening on. When not provided, 15001 is used by default.")
c.flags.Var((*flags.AppendSliceValue)(&c.excludeInboundPorts), "exclude-inbound-port",
"Inbound port to exclude from traffic redirection. May be provided multiple times.")
c.flags.Var((*flags.AppendSliceValue)(&c.excludeOutboundPorts), "exclude-outbound-port",
"Outbound port to exclude from traffic redirection. May be provided multiple times.")
c.flags.Var((*flags.AppendSliceValue)(&c.excludeOutboundCIDRs), "exclude-outbound-cidr",
"Outbound CIDR to exclude from traffic redirection. May be provided multiple times.")
c.flags.Var((*flags.AppendSliceValue)(&c.excludeUIDs), "exclude-uid",
"Additional user ID to exclude from traffic redirection. May be provided multiple times.")
c.http = &flags.HTTPFlags{}
flags.Merge(c.flags, c.http.ClientFlags())
@ -106,11 +120,17 @@ func (c *cmd) Help() string {
// to apply traffic redirection rules.
type trafficRedirectProxyConfig struct {
BindPort int `mapstructure:"bind_port"`
PrometheusBindAddr string `mapstructure:"envoy_prometheus_bind_addr"`
StatsBindAddr string `mapstructure:"envoy_stats_bind_addr"`
}
// generateConfigFromFlags generates iptables.Config based on command flags.
func (c *cmd) generateConfigFromFlags() (iptables.Config, error) {
cfg := iptables.Config{ProxyUserID: c.proxyUID}
cfg := iptables.Config{
ProxyUserID: c.proxyUID,
ProxyInboundPort: c.proxyInboundPort,
ProxyOutboundPort: c.proxyOutboundPort,
}
// When proxyID is provided, we set up cfg with values
// from proxy's service registration in Consul.
@ -132,21 +152,67 @@ func (c *cmd) generateConfigFromFlags() (iptables.Config, error) {
return iptables.Config{}, fmt.Errorf("service %s is not a proxy service", c.proxyID)
}
cfg.ProxyInboundPort = svc.Port
// Decode proxy's opaque config so that we can use it later to configure
// traffic redirection with iptables.
var trCfg trafficRedirectProxyConfig
if err := mapstructure.WeakDecode(svc.Proxy.Config, &trCfg); err != nil {
return iptables.Config{}, fmt.Errorf("failed parsing Proxy.Config: %s", err)
}
// Set the proxy's inbound port.
cfg.ProxyInboundPort = svc.Port
if trCfg.BindPort != 0 {
cfg.ProxyInboundPort = trCfg.BindPort
}
// todo: Change once it's configurable
// Set the proxy's outbound port.
cfg.ProxyOutboundPort = iptables.DefaultTProxyOutboundPort
} else {
cfg.ProxyInboundPort = c.proxyInboundPort
cfg.ProxyOutboundPort = c.proxyOutboundPort
if svc.Proxy.TransparentProxy != nil && svc.Proxy.TransparentProxy.OutboundListenerPort != 0 {
cfg.ProxyOutboundPort = svc.Proxy.TransparentProxy.OutboundListenerPort
}
// Exclude envoy_prometheus_bind_addr port from inbound redirection rules.
if trCfg.PrometheusBindAddr != "" {
_, port, err := net.SplitHostPort(trCfg.PrometheusBindAddr)
if err != nil {
return iptables.Config{}, fmt.Errorf("failed parsing host and port from envoy_prometheus_bind_addr: %s", err)
}
cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, port)
}
// Exclude envoy_stats_bind_addr port from inbound redirection rules.
if trCfg.StatsBindAddr != "" {
_, port, err := net.SplitHostPort(trCfg.StatsBindAddr)
if err != nil {
return iptables.Config{}, fmt.Errorf("failed parsing host and port from envoy_stats_bind_addr: %s", err)
}
cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, port)
}
// Exclude the ListenerPort from Expose configs from inbound traffic redirection.
for _, exposePath := range svc.Proxy.Expose.Paths {
if exposePath.ListenerPort != 0 {
cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposePath.ListenerPort))
}
}
}
for _, port := range c.excludeInboundPorts {
cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, port)
}
for _, port := range c.excludeOutboundPorts {
cfg.ExcludeOutboundPorts = append(cfg.ExcludeOutboundPorts, port)
}
for _, cidr := range c.excludeOutboundCIDRs {
cfg.ExcludeOutboundCIDRs = append(cfg.ExcludeOutboundCIDRs, cidr)
}
for _, uid := range c.excludeUIDs {
cfg.ExcludeUIDs = append(cfg.ExcludeUIDs, uid)
}
return cfg, nil

View File

@ -177,6 +177,35 @@ func TestGenerateConfigFromFlags(t *testing.T) {
iptables.Config{},
"failed parsing Proxy.Config: 1 error(s) decoding:\n\n* cannot parse 'bind_port' as int:",
},
{
"proxyID with proxy outbound port",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
TransparentProxy: &api.TransparentProxyConfig{
OutboundListenerPort: 21000,
},
},
},
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 20000,
ProxyOutboundPort: 21000,
},
"",
},
{
"proxyID provided, but Consul is not reachable",
func() cmd {
@ -243,6 +272,228 @@ func TestGenerateConfigFromFlags(t *testing.T) {
},
"",
},
{
"exclude inbound ports are provided",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyInboundPort = 15000
c.excludeInboundPorts = []string{"8080", "21000"}
return c
},
nil,
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 15000,
ProxyOutboundPort: 15001,
ExcludeInboundPorts: []string{"8080", "21000"},
},
"",
},
{
"exclude outbound ports are provided",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyInboundPort = 15000
c.excludeOutboundPorts = []string{"8080", "21000"}
return c
},
nil,
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 15000,
ProxyOutboundPort: 15001,
ExcludeOutboundPorts: []string{"8080", "21000"},
},
"",
},
{
"exclude outbound CIDRs are provided",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyInboundPort = 15000
c.excludeOutboundCIDRs = []string{"1.1.1.1", "2.2.2.2/24"}
return c
},
nil,
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 15000,
ProxyOutboundPort: 15001,
ExcludeOutboundCIDRs: []string{"1.1.1.1", "2.2.2.2/24"},
},
"",
},
{
"exclude UIDs are provided",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyInboundPort = 15000
c.excludeUIDs = []string{"2345", "3456"}
return c
},
nil,
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 15000,
ProxyOutboundPort: 15001,
ExcludeUIDs: []string{"2345", "3456"},
},
"",
},
{
"proxy config has envoy_prometheus_bind_addr set",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
Config: map[string]interface{}{
"envoy_prometheus_bind_addr": "0.0.0.0:9000",
},
},
},
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 20000,
ProxyOutboundPort: iptables.DefaultTProxyOutboundPort,
ExcludeInboundPorts: []string{"9000"},
},
"",
},
{
"proxy config has an invalid envoy_prometheus_bind_addr set",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
Config: map[string]interface{}{
"envoy_prometheus_bind_addr": "9000",
},
},
},
iptables.Config{},
"failed parsing host and port from envoy_prometheus_bind_addr: address 9000: missing port in address",
},
{
"proxy config has envoy_stats_bind_addr set",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
Config: map[string]interface{}{
"envoy_stats_bind_addr": "0.0.0.0:8000",
},
},
},
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 20000,
ProxyOutboundPort: iptables.DefaultTProxyOutboundPort,
ExcludeInboundPorts: []string{"8000"},
},
"",
},
{
"proxy config has an invalid envoy_stats_bind_addr set",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
Config: map[string]interface{}{
"envoy_stats_bind_addr": "8000",
},
},
},
iptables.Config{},
"failed parsing host and port from envoy_stats_bind_addr: address 8000: missing port in address",
},
{
"proxy config has expose paths with listener port set",
func() cmd {
var c cmd
c.init()
c.proxyUID = "1234"
c.proxyID = "test-proxy-id"
return c
},
&api.AgentServiceRegistration{
Kind: api.ServiceKindConnectProxy,
ID: "test-proxy-id",
Name: "test-proxy",
Port: 20000,
Address: "1.1.1.1",
Proxy: &api.AgentServiceConnectProxyConfig{
DestinationServiceName: "foo",
Expose: api.ExposeConfig{
Paths: []api.ExposePath{
{
ListenerPort: 23000,
LocalPathPort: 8080,
Path: "/health",
},
},
},
},
},
iptables.Config{
ProxyUserID: "1234",
ProxyInboundPort: 20000,
ProxyOutboundPort: iptables.DefaultTProxyOutboundPort,
ExcludeInboundPorts: []string{"23000"},
},
"",
},
}
for _, c := range cases {
@ -251,6 +502,7 @@ func TestGenerateConfigFromFlags(t *testing.T) {
if c.proxyService != nil {
testServer, err := testutil.NewTestServerConfigT(t, nil)
require.NoError(t, err)
testServer.WaitForLeader(t)
defer testServer.Stop()
client, err := api.NewClient(&api.Config{Address: testServer.HTTPAddr})

View File

@ -33,6 +33,22 @@ type Config struct {
// ProxyInboundPort is the port of the proxy's outbound listener.
ProxyOutboundPort int
// ExcludeInboundPorts is the list of ports that should be excluded
// from inbound traffic redirection.
ExcludeInboundPorts []string
// ExcludeOutboundPorts is the list of ports that should be excluded
// from outbound traffic redirection.
ExcludeOutboundPorts []string
// ExcludeOutboundCIDRs is the list of IP CIDRs that should be excluded
// from outbound traffic redirection.
ExcludeOutboundCIDRs []string
// ExcludeUIDs is the list of additional user IDs to exclude
// from traffic redirection.
ExcludeUIDs []string
// IptablesProvider is the Provider that will apply iptables rules.
IptablesProvider Provider
}
@ -90,6 +106,19 @@ func Setup(cfg Config) error {
// Redirect remaining outbound traffic to Envoy.
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-A", ProxyOutputChain, "-j", ProxyOutputRedirectChain)
// We are using "insert" (-I) instead of "append" (-A) so the the provided rules take precedence over default ones.
for _, outboundPort := range cfg.ExcludeOutboundPorts {
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-I", ProxyOutputChain, "-p", "tcp", "--dport", outboundPort, "-j", "RETURN")
}
for _, outboundIP := range cfg.ExcludeOutboundCIDRs {
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-I", ProxyOutputChain, "-d", outboundIP, "-j", "RETURN")
}
for _, uid := range cfg.ExcludeUIDs {
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-I", ProxyOutputChain, "-m", "owner", "--uid-owner", uid, "-j", "RETURN")
}
}
// Configure inbound rules.
@ -102,6 +131,10 @@ func Setup(cfg Config) error {
// Redirect remaining inbound traffic to Envoy.
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-A", ProxyInboundChain, "-p", "tcp", "-j", ProxyInboundRedirectChain)
for _, inboundPort := range cfg.ExcludeInboundPorts {
cfg.IptablesProvider.AddRule("iptables", "-t", "nat", "-I", ProxyInboundChain, "-p", "tcp", "--dport", inboundPort, "-j", "RETURN")
}
}
return cfg.IptablesProvider.ApplyRules()

View File

@ -58,6 +58,110 @@ func TestSetup(t *testing.T) {
"iptables -t nat -A CONSUL_PROXY_INBOUND -p tcp -j CONSUL_PROXY_IN_REDIRECT",
},
},
{
"exclude inbound ports is set",
Config{
ProxyUserID: "123",
ProxyInboundPort: 20000,
ProxyOutboundPort: 21000,
ExcludeInboundPorts: []string{"22000", "22500"},
IptablesProvider: &fakeIptablesProvider{},
},
[]string{
"iptables -t nat -N CONSUL_PROXY_INBOUND",
"iptables -t nat -N CONSUL_PROXY_IN_REDIRECT",
"iptables -t nat -N CONSUL_PROXY_OUTPUT",
"iptables -t nat -N CONSUL_PROXY_REDIRECT",
"iptables -t nat -A CONSUL_PROXY_REDIRECT -p tcp -j REDIRECT --to-port 21000",
"iptables -t nat -A OUTPUT -p tcp -j CONSUL_PROXY_OUTPUT",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -m owner --uid-owner 123 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -j CONSUL_PROXY_REDIRECT",
"iptables -t nat -A CONSUL_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 20000",
"iptables -t nat -A PREROUTING -p tcp -j CONSUL_PROXY_INBOUND",
"iptables -t nat -A CONSUL_PROXY_INBOUND -p tcp -j CONSUL_PROXY_IN_REDIRECT",
"iptables -t nat -I CONSUL_PROXY_INBOUND -p tcp --dport 22000 -j RETURN",
"iptables -t nat -I CONSUL_PROXY_INBOUND -p tcp --dport 22500 -j RETURN",
},
},
{
"exclude outbound ports is set",
Config{
ProxyUserID: "123",
ProxyInboundPort: 20000,
ProxyOutboundPort: 21000,
ExcludeOutboundPorts: []string{"22000", "22500"},
IptablesProvider: &fakeIptablesProvider{},
},
[]string{
"iptables -t nat -N CONSUL_PROXY_INBOUND",
"iptables -t nat -N CONSUL_PROXY_IN_REDIRECT",
"iptables -t nat -N CONSUL_PROXY_OUTPUT",
"iptables -t nat -N CONSUL_PROXY_REDIRECT",
"iptables -t nat -A CONSUL_PROXY_REDIRECT -p tcp -j REDIRECT --to-port 21000",
"iptables -t nat -A OUTPUT -p tcp -j CONSUL_PROXY_OUTPUT",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -m owner --uid-owner 123 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -j CONSUL_PROXY_REDIRECT",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -p tcp --dport 22000 -j RETURN",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -p tcp --dport 22500 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 20000",
"iptables -t nat -A PREROUTING -p tcp -j CONSUL_PROXY_INBOUND",
"iptables -t nat -A CONSUL_PROXY_INBOUND -p tcp -j CONSUL_PROXY_IN_REDIRECT",
},
},
{
"exclude outbound CIDRs is set",
Config{
ProxyUserID: "123",
ProxyInboundPort: 20000,
ProxyOutboundPort: 21000,
ExcludeOutboundCIDRs: []string{"1.1.1.1", "2.2.2.2/24"},
IptablesProvider: &fakeIptablesProvider{},
},
[]string{
"iptables -t nat -N CONSUL_PROXY_INBOUND",
"iptables -t nat -N CONSUL_PROXY_IN_REDIRECT",
"iptables -t nat -N CONSUL_PROXY_OUTPUT",
"iptables -t nat -N CONSUL_PROXY_REDIRECT",
"iptables -t nat -A CONSUL_PROXY_REDIRECT -p tcp -j REDIRECT --to-port 21000",
"iptables -t nat -A OUTPUT -p tcp -j CONSUL_PROXY_OUTPUT",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -m owner --uid-owner 123 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -j CONSUL_PROXY_REDIRECT",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -d 1.1.1.1 -j RETURN",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -d 2.2.2.2/24 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 20000",
"iptables -t nat -A PREROUTING -p tcp -j CONSUL_PROXY_INBOUND",
"iptables -t nat -A CONSUL_PROXY_INBOUND -p tcp -j CONSUL_PROXY_IN_REDIRECT",
},
},
{
"exclude UIDs is set",
Config{
ProxyUserID: "123",
ProxyInboundPort: 20000,
ProxyOutboundPort: 21000,
ExcludeUIDs: []string{"456", "789"},
IptablesProvider: &fakeIptablesProvider{},
},
[]string{
"iptables -t nat -N CONSUL_PROXY_INBOUND",
"iptables -t nat -N CONSUL_PROXY_IN_REDIRECT",
"iptables -t nat -N CONSUL_PROXY_OUTPUT",
"iptables -t nat -N CONSUL_PROXY_REDIRECT",
"iptables -t nat -A CONSUL_PROXY_REDIRECT -p tcp -j REDIRECT --to-port 21000",
"iptables -t nat -A OUTPUT -p tcp -j CONSUL_PROXY_OUTPUT",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -m owner --uid-owner 123 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_OUTPUT -j CONSUL_PROXY_REDIRECT",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -m owner --uid-owner 456 -j RETURN",
"iptables -t nat -I CONSUL_PROXY_OUTPUT -m owner --uid-owner 789 -j RETURN",
"iptables -t nat -A CONSUL_PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 20000",
"iptables -t nat -A PREROUTING -p tcp -j CONSUL_PROXY_INBOUND",
"iptables -t nat -A CONSUL_PROXY_INBOUND -p tcp -j CONSUL_PROXY_IN_REDIRECT",
},
},
}
for _, c := range cases {

View File

@ -45,6 +45,14 @@ Usage: `consul connect redirect-traffic [options]`
- `-proxy-uid` - The user ID of the proxy to exclude from traffic redirection.
- `-exclude-inbound-port` - Inbound port to exclude from traffic redirection. May be provided multiple times.
- `exclude-outbound-cidr` - Outbound CIDR to exclude from traffic redirection. May be provided multiple times.
- `exclude-outbound-port` - Outbound port to exclude from traffic redirection. May be provided multiple times.
- `exclude-uid` - Additional user ID to exclude from traffic redirection. May be provided multiple times.
#### Enterprise Options
@include 'http_api_namespace_options.mdx'