xds: deduplicate mesh gateway listeners in a stable way (#9650)

In a situation where the mesh gateway is configured to bind to multiple
network interfaces, we use a feature called 'tagged addresses'.
Sometimes an address is duplicated across multiple tags such as 'lan'
and 'lan_ipv4'.

There is code to deduplicate these things when creating envoy listeners,
but that code doesn't ensure that the same tag wins every time. If the
winning tag flaps between xDS discovery requests it will cause the
listener to be drained and replaced.

.changelog/9650.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+xds: deduplicate mesh gateway listeners by address in a stable way to prevent some LDS churn
+```

agent/xds/listeners.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/url"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -198,13 +199,11 @@ func (s *Server) listenersFromSnapshotGateway(cInfo connectionInfo, cfgSnap *pro
 		s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err)
 	}
 
-	// Prevent invalid configurations of binding to the same port/addr twice
-	// including with the any addresses
+	// We'll collect all of the desired listeners first, and deduplicate them later.
 	type namedAddress struct {
 		name string
 		structs.ServiceAddress
 	}
-	seen := make(map[structs.ServiceAddress]bool)
 	addrs := make([]namedAddress, 0)
 
 	var resources []proto.Message
@@ -218,10 +217,7 @@ func (s *Server) listenersFromSnapshotGateway(cInfo connectionInfo, cfgSnap *pro
 			Address: addr,
 			Port:    cfgSnap.Port,
 		}
-		if !seen[a] {
 		addrs = append(addrs, namedAddress{name: "default", ServiceAddress: a})
-			seen[a] = true
-		}
 	}
 
 	if cfg.BindTaggedAddresses {
@@ -230,10 +226,7 @@ func (s *Server) listenersFromSnapshotGateway(cInfo connectionInfo, cfgSnap *pro
 				Address: addrCfg.Address,
 				Port:    addrCfg.Port,
 			}
-			if !seen[a] {
 			addrs = append(addrs, namedAddress{name: name, ServiceAddress: a})
-				seen[a] = true
-			}
 		}
 	}
 
@@ -242,14 +235,27 @@ func (s *Server) listenersFromSnapshotGateway(cInfo connectionInfo, cfgSnap *pro
 			Address: addrCfg.Address,
 			Port:    addrCfg.Port,
 		}
-		if !seen[a] {
 		addrs = append(addrs, namedAddress{name: name, ServiceAddress: a})
-			seen[a] = true
-		}
 	}
 
-	// Make listeners once deduplicated
+	// Prevent invalid configurations of binding to the same port/addr twice
+	// including with the any addresses
+	//
+	// Sort the list and then if two items share a service address, take the
+	// first one to ensure we generate one listener per address and it's
+	// stable.
+	sort.Slice(addrs, func(i, j int) bool {
+		return addrs[i].name < addrs[j].name
+	})
+
+	// Make listeners and deduplicate on the fly.
+	seen := make(map[structs.ServiceAddress]bool)
 	for _, a := range addrs {
+		if seen[a.ServiceAddress] {
+			continue
+		}
+		seen[a.ServiceAddress] = true
+
 		var l *envoy.Listener
 
 		switch cfgSnap.Kind {

agent/xds/listeners_test.go

@@ -345,15 +345,15 @@ func TestListenersFromSnapshot(t *testing.T) {
 					"envoy_gateway_no_default_bind":       true,
 					"envoy_gateway_bind_tagged_addresses": true,
 					"envoy_gateway_bind_addresses": map[string]structs.ServiceAddress{
+						// This bind address should not get a listener due to deduplication and it sorts to the end
+						"z-duplicate-of-tagged-wan-addr": {
+							Address: "198.18.0.1",
+							Port:    443,
+						},
 						"foo": {
 							Address: "198.17.2.3",
 							Port:    8080,
 						},
-						// This bind address should not get a listener due to deduplication
-						"duplicate-of-tagged-wan-addr": {
-							Address: "198.18.0.1",
-							Port:    443,
-						},
 					},
 				}
 			},