Fix flapping of mesh gateway connect-service watches (#7575)

agent/proxycfg/state.go

@@ -810,6 +810,10 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 		svcMap := make(map[structs.ServiceID]struct{})
 		for _, svc := range services.Services {
 			sid := svc.ToServiceID()
+			// Make sure to add every service to this map, we use it to cancel
+			// watches below.
+			svcMap[sid] = struct{}{}
+
 			if _, ok := snap.MeshGateway.WatchedServices[sid]; !ok {
 				ctx, cancel := context.WithCancel(s.ctx)
 				err := s.cache.Notify(ctx, cachetype.HealthServicesName, &structs.ServiceSpecificRequest{
@@ -829,12 +833,12 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 					return err
 				}
 				snap.MeshGateway.WatchedServices[sid] = cancel
-				svcMap[sid] = struct{}{}
 			}
 		}
 
 		for sid, cancelFn := range snap.MeshGateway.WatchedServices {
 			if _, ok := svcMap[sid]; !ok {
+				meshLogger.Debug("canceling watch for service", "service", sid.String())
 				delete(snap.MeshGateway.WatchedServices, sid)
 				cancelFn()
 			}

agent/proxycfg/state_test.go

@@ -610,6 +610,61 @@ func TestState_WatchesAndUpdates(t *testing.T) {
 				},
 			},
 		},
+		"mesh-gateway-do-not-cancel-service-watches": testCase{
+			ns: structs.NodeService{
+				Kind:    structs.ServiceKindMeshGateway,
+				ID:      "mesh-gateway",
+				Service: "mesh-gateway",
+				Address: "10.0.1.1",
+				Port:    443,
+			},
+			sourceDC: "dc1",
+			stages: []verificationStage{
+				verificationStage{
+					requiredWatches: map[string]verifyWatchRequest{
+						rootsWatchID:       genVerifyRootsWatch("dc1"),
+						serviceListWatchID: genVerifyListServicesWatch("dc1"),
+						datacentersWatchID: verifyDatacentersWatch,
+					},
+					events: []cache.UpdateEvent{
+						rootWatchEvent(),
+						cache.UpdateEvent{
+							CorrelationID: serviceListWatchID,
+							Result: &structs.IndexedServiceList{
+								Services: structs.ServiceList{
+									{Name: "web"},
+								},
+							},
+							Err: nil,
+						},
+					},
+					verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
+						require.True(t, snap.Valid(), "gateway with service list is vaild")
+						require.Len(t, snap.MeshGateway.WatchedServices, 1)
+						require.True(t, snap.MeshGateway.WatchedServicesSet)
+					},
+				},
+				verificationStage{
+					events: []cache.UpdateEvent{
+						cache.UpdateEvent{
+							CorrelationID: serviceListWatchID,
+							Result: &structs.IndexedServiceList{
+								Services: structs.ServiceList{
+									{Name: "web"},
+									{Name: "api"},
+								},
+							},
+							Err: nil,
+						},
+					},
+					verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) {
+						require.True(t, snap.Valid(), "gateway with service list is vaild")
+						require.Len(t, snap.MeshGateway.WatchedServices, 2)
+						require.True(t, snap.MeshGateway.WatchedServicesSet)
+					},
+				},
+			},
+		},
 		"connect-proxy":                    newConnectProxyCase(structs.MeshGatewayModeDefault),
 		"connect-proxy-mesh-gateway-local": newConnectProxyCase(structs.MeshGatewayModeLocal),
 	}

test/integration/connect/envoy/case-gateways-local/primary/verify.bats

@@ -27,6 +27,10 @@ load helpers
    assert_upstream_has_endpoints_in_status 127.0.0.1:19002 secondary HEALTHY 1
 }
 
+@test "gateway-secondary should have healthy endpoints for s2" {
+   assert_upstream_has_endpoints_in_status consul-secondary:19003 s2 HEALTHY 1
+}
+
 @test "s1 upstream should be able to connect to s2" {
   run retry_default curl -s -f -d hello localhost:5000
   [ "$status" -eq 0 ]