status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

mesh: only build tproxy outbound listener once per destination (#18836) 

Previously, when using implicit upstreams, we'd build outbound listener per destination instead of one for all destinations. This will result in port conflicts when trying to send this config to envoy.

This PR also makes sure that leaf and root references are always added (before we would only add it if there are inbound non-mesh ports).

Also, black-hole traffic when there are no inbound ports other than mesh
This commit is contained in:
Iryna Shustava 2023-09-18 18:26:13 -06:00 committed by GitHub
parent 91e6c3a82f
commit 212793a4ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 276 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
agent/xdsv2/cluster_resources.go
View File

@ -154,14 +154,15 @@ func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string,
}
func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
endpointList, ok := pr.proxyState.Endpoints[name]
if !ok || endpointList == nil {
return nil, fmt.Errorf("static cluster %q is missing endpoints", name)
}
cluster := &envoy_cluster_v3.Cluster{
Name: name,
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC},
LoadAssignment: makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints),
}
// todo (ishustava/v2): we need to be able to handle the case when empty endpoints are allowed on a cluster.
endpointList, ok := pr.proxyState.Endpoints[name]
if ok {
cluster.LoadAssignment = makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints)
}
err := addHttpProtocolOptions(protocol, cluster)
if err != nil {

4
envoyextensions/xdscommon/xdscommon.go
View File

@ -54,6 +54,10 @@ const (
SecretType = apiTypePrefix + "envoy.extensions.transport_sockets.tls.v3.Secret"
FailoverClusterNamePrefix = "failover-target~"
// BlackHoleClusterName is the cluster we use for black-holing traffic for cases when a workload
// has no inbound ports to route to.
BlackHoleClusterName = "black-hole-cluster"
)
type IndexedResources struct {

11
internal/mesh/internal/controllers/sidecarproxy/builder/builder.go
View File

@ -45,6 +45,17 @@ func New(
}
func (b *Builder) Build() *pbmesh.ProxyStateTemplate {
workloadIdentity := b.proxyStateTemplate.ProxyState.Identity.Name
b.proxyStateTemplate.RequiredLeafCertificates[workloadIdentity] = &pbproxystate.LeafCertificateRef{
Name: workloadIdentity,
Namespace: b.id.Tenancy.Namespace,
Partition: b.id.Tenancy.Partition,
}
b.proxyStateTemplate.RequiredTrustBundles[b.id.Tenancy.PeerName] = &pbproxystate.TrustBundleRef{
Peer: b.id.Tenancy.PeerName,
}
return b.proxyStateTemplate
}

10
internal/mesh/internal/controllers/sidecarproxy/builder/destination_builder.go
View File

@ -25,8 +25,7 @@ import (
// and adds them to the proxyState.
func (b *Builder) BuildDestinations(destinations []*intermediate.Destination) *Builder {
var lb *ListenerBuilder
if b.proxyCfg.GetDynamicConfig() != nil &&
b.proxyCfg.DynamicConfig.Mode == pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT {
if b.proxyCfg.IsTransparentProxy() {
lb = b.addTransparentProxyOutboundListener(b.proxyCfg.DynamicConfig.TransparentProxy.OutboundListenerPort)
}
@ -34,6 +33,10 @@ func (b *Builder) BuildDestinations(destinations []*intermediate.Destination) *B
b.buildDestination(lb, destination)
}
if b.proxyCfg.IsTransparentProxy() {
lb.buildListener()
}
return b
}
@ -248,7 +251,10 @@ func (b *Builder) buildDestination(
}
}
// Build outbound listener if the destination is explicit.
if destination.Explicit != nil {
lb.buildListener()
}
if needsNullRouteCluster {
b.addNullRouteCluster()

52
internal/mesh/internal/controllers/sidecarproxy/builder/local_app.go
View File

@ -25,10 +25,12 @@ func (b *Builder) BuildLocalApp(workload *pbcatalog.Workload, ctp *pbauth.Comput
// Note that the order of ports is non-deterministic here but the xds generation
// code should make sure to send it in the same order to Envoy to avoid unnecessary
// updates.
foundInboundNonMeshPorts := false
for portName, port := range workload.Ports {
clusterName := fmt.Sprintf("%s:%s", xdscommon.LocalAppClusterName, portName)
if port.Protocol != pbcatalog.Protocol_PROTOCOL_MESH {
foundInboundNonMeshPorts = true
lb.addInboundRouter(clusterName, port, portName, trafficPermissions[portName]).
addInboundTLS()
@ -37,6 +39,12 @@ func (b *Builder) BuildLocalApp(workload *pbcatalog.Workload, ctp *pbauth.Comput
}
}
// If there are no inbound ports other than the mesh port, we black-hole all inbound traffic.
if !foundInboundNonMeshPorts {
lb.addBlackHoleRouter()
b.addBlackHoleCluster()
}
return b
}
@ -265,6 +273,28 @@ func (l *ListenerBuilder) addInboundRouter(clusterName string, port *pbcatalog.W
return l
}
func (l *ListenerBuilder) addBlackHoleRouter() *ListenerBuilder {
if l.listener == nil {
return l
}
r := &pbproxystate.Router{
Destination: &pbproxystate.Router_L4{
L4: &pbproxystate.L4Destination{
Destination: &pbproxystate.L4Destination_Cluster{
Cluster: &pbproxystate.DestinationCluster{
Name: xdscommon.BlackHoleClusterName,
},
},
StatPrefix: l.listener.Name,
},
},
}
l.listener.Routers = append(l.listener.Routers, r)
return l
}
func getAlpnProtocolFromPortName(portName string) string {
return fmt.Sprintf("consul~%s", portName)
}
@ -283,6 +313,19 @@ func (b *Builder) addLocalAppCluster(clusterName string) *Builder {
return b
}
func (b *Builder) addBlackHoleCluster() *Builder {
b.proxyStateTemplate.ProxyState.Clusters[xdscommon.BlackHoleClusterName] = &pbproxystate.Cluster{
Group: &pbproxystate.Cluster_EndpointGroup{
EndpointGroup: &pbproxystate.EndpointGroup{
Group: &pbproxystate.EndpointGroup_Static{
Static: &pbproxystate.StaticEndpointGroup{},
},
},
},
}
return b
}
func (b *Builder) addLocalAppStaticEndpoints(clusterName string, port *pbcatalog.WorkloadPort) *Builder {
// We're adding endpoints statically as opposed to creating an endpoint ref
// because this endpoint is less likely to change as we're not tracking the health.
@ -319,15 +362,6 @@ func (l *ListenerBuilder) addInboundTLS() *ListenerBuilder {
},
},
}
l.builder.proxyStateTemplate.RequiredLeafCertificates[workloadIdentity] = &pbproxystate.LeafCertificateRef{
Name: workloadIdentity,
Namespace: l.builder.id.Tenancy.Namespace,
Partition: l.builder.id.Tenancy.Partition,
}
l.builder.proxyStateTemplate.RequiredTrustBundles[l.builder.id.Tenancy.PeerName] = &pbproxystate.TrustBundleRef{
Peer: l.builder.id.Tenancy.PeerName,
}
for i := range l.listener.Routers {
l.listener.Routers[i].InboundTls = inboundTLS

15
internal/mesh/internal/controllers/sidecarproxy/builder/local_app_multiport_test.go
View File

@ -4,10 +4,11 @@
package builder
import (
"github.com/hashicorp/consul/internal/testing/golden"
"sort"
"testing"
"github.com/hashicorp/consul/internal/testing/golden"
"github.com/stretchr/testify/require"
pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v1alpha1"
@ -71,6 +72,18 @@ func TestBuildLocalApp_Multiport(t *testing.T) {
},
},
},
"source/multiport-l4-workload-with-only-mesh-port": {
workload: &pbcatalog.Workload{
Addresses: []*pbcatalog.WorkloadAddress{
{
Host: "10.0.0.1",
},
},
Ports: map[string]*pbcatalog.WorkloadPort{
"mesh": {Port: 20000, Protocol: pbcatalog.Protocol_PROTOCOL_MESH},
},
},
},
}
for name, c := range cases {

48
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-implicit-and-explicit-destinations-tproxy.golden vendored
View File

@ -61,6 +61,24 @@
}
},
"listeners": [
{
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "1.1.1.1",
"port": 1234
},
"name": "api-1:tcp:1.1.1.1:1234",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-1.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-1.default.default.dc1"
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
@ -94,24 +112,6 @@
}
}
]
},
{
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "1.1.1.1",
"port": 1234
},
"name": "api-1:tcp:1.1.1.1:1234",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-1.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-1.default.default.dc1"
}
}
]
}
]
},
@ -148,5 +148,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

12
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-multi-destination.golden vendored
View File

@ -155,5 +155,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

63
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-multiple-implicit-destinations-tproxy.golden vendored
View File

@ -61,57 +61,6 @@
}
},
"listeners": [
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-1.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-1.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l4": {
"cluster": {
"name": "tcp.api-2.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-2.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
@ -198,5 +147,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

12
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-single-destination-ip-port-bind-address.golden vendored
View File

@ -120,5 +120,17 @@
"tcp.api-2.default.dc1.internal.foo.consul": {
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

12
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-single-destination-unix-socket-bind-address.golden vendored
View File

@ -73,5 +73,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

12
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/l4-single-implicit-destination-tproxy.golden vendored
View File

@ -85,5 +85,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

12
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/mixed-multi-destination.golden vendored
View File

@ -284,5 +284,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

261
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/multiport-l4-multiple-implicit-destinations-tproxy.golden vendored
View File

@ -111,255 +111,6 @@
}
},
"listeners": [
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-app.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l4": {
"cluster": {
"name": "tcp.api-app2.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app2.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-app.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l4": {
"cluster": {
"name": "tcp.api-app2.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app2.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-app.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l4": {
"cluster": {
"name": "tcp.api-app2.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app2.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
@ -533,5 +284,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

56
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/multiport-l4-single-implicit-destination-tproxy.golden vendored
View File

@ -61,50 +61,6 @@
}
},
"listeners": [
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-app.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
@ -207,5 +163,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

56
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/destination/multiport-l4-single-implicit-destination-with-multiple-workloads-tproxy.golden vendored
View File

@ -61,50 +61,6 @@
}
},
"listeners": [
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-app.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-app.default.default.dc1"
},
"match": {
"destinationPort": 8080,
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
},
{
"l7": {
"name": "outbound_listener",
"statPrefix": "upstream."
},
"match": {
"prefixRanges": [
{
"addressPrefix": "1.1.1.1",
"prefixLen": 32
}
]
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
@ -207,5 +163,17 @@
},
"port": "mesh"
}
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}

51
internal/mesh/internal/controllers/sidecarproxy/builder/testdata/source/multiport-l4-workload-with-only-mesh-port.golden vendored Normal file
View File

@ -0,0 +1,51 @@
{
"proxyState": {
"clusters": {
"black-hole-cluster": {
"endpointGroup": {
"static": {}
}
}
},
"identity": {
"name": "test-identity",
"tenancy": {
"namespace": "default",
"partition": "default",
"peerName": "local"
}
},
"listeners": [
{
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",
"port": 20000
},
"name": "public_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "black-hole-cluster"
},
"statPrefix": "public_listener"
}
}
]
}
]
},
"requiredLeafCertificates": {
"test-identity": {
"name": "test-identity",
"namespace": "default",
"partition": "default"
}
},
"requiredTrustBundles": {
"local": {
"peer": "local"
}
}
}