[NET-6725] test: Address occasional flakes in sidecarproxy/controller_test.go (#19760)

test: Address occasional flakes in sidecarproxy/controller_test.go

We've observed an occasional flake in this test where some state check
fails. Adding in some wait wrappers to these state checks will hopefully
address the issue, assuming it is a simple flake.

internal/mesh/internal/controllers/sidecarproxy/controller_test.go

@@ -610,7 +610,9 @@ func (suite *controllerTestSuite) TestController() {
 		testutil.RunStep(suite.T(), "add explicit destinations and check that new proxy state is generated", func(t *testing.T) {
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
 
-			requireExplicitDestinationsFound(t, "api", webProxyStateTemplate)
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
+				requireExplicitDestinationsFound(rt, "api", tmpl)
+			})
 		})
 
 		testutil.RunStep(suite.T(), "update api's ports to be non-mesh", func(t *testing.T) {
@@ -673,7 +675,9 @@ func (suite *controllerTestSuite) TestController() {
 			// We should get a new web proxy template resource because this destination should be removed.
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
 
-			requireExplicitDestinationsNotFound(t, "api", webProxyStateTemplate)
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
+				requireExplicitDestinationsNotFound(rt, "api", tmpl)
+			})
 		})
 
 		testutil.RunStep(suite.T(), "update ports to be mesh again", func(t *testing.T) {
@@ -704,7 +708,9 @@ func (suite *controllerTestSuite) TestController() {
 			// We should also get a new web proxy template resource as this destination should be added again.
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
 
-			requireExplicitDestinationsFound(t, "api", webProxyStateTemplate)
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
+				requireExplicitDestinationsFound(rt, "api", tmpl)
+			})
 		})
 
 		testutil.RunStep(suite.T(), "delete the proxy state template and check re-generation", func(t *testing.T) {
@@ -714,7 +720,10 @@ func (suite *controllerTestSuite) TestController() {
 			require.NoError(suite.T(), err)
 
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
-			requireExplicitDestinationsFound(t, "api", webProxyStateTemplate)
+
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
+				requireExplicitDestinationsFound(rt, "api", tmpl)
+			})
 		})
 
 		testutil.RunStep(suite.T(), "add implicit upstream and enable tproxy", func(t *testing.T) {
@@ -746,8 +755,10 @@ func (suite *controllerTestSuite) TestController() {
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
 			apiProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), apiProxyStateTemplateID, apiProxyStateTemplate.Version)
 
-			requireImplicitDestinationsFound(t, "api", webProxyStateTemplate)
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
-			requireImplicitDestinationsFound(t, "db", webProxyStateTemplate)
+				requireImplicitDestinationsFound(rt, "api", tmpl)
+				requireImplicitDestinationsFound(rt, "db", tmpl)
+			})
 		})
 
 		testutil.RunStep(suite.T(), "traffic permissions", func(t *testing.T) {
@@ -819,8 +830,10 @@ func (suite *controllerTestSuite) TestController() {
 
 			webProxyStateTemplate = suite.client.WaitForNewVersion(suite.T(), webProxyStateTemplateID, webProxyStateTemplate.Version)
 
-			requireImplicitDestinationsFound(t, "api", webProxyStateTemplate)
+			suite.waitForProxyStateTemplateState(t, webProxyStateTemplateID, func(rt resourcetest.T, tmpl *pbmesh.ProxyStateTemplate) {
-			requireImplicitDestinationsFound(t, "db", webProxyStateTemplate)
+				requireImplicitDestinationsFound(rt, "api", tmpl)
+				requireImplicitDestinationsFound(rt, "db", tmpl)
+			})
 		})
 	})
 }
@@ -859,21 +872,17 @@ func TestMeshController(t *testing.T) {
 	suite.Run(t, new(controllerTestSuite))
 }
 
-func requireExplicitDestinationsFound(t *testing.T, name string, tmplResource *pbresource.Resource) {
+func requireExplicitDestinationsFound(t resourcetest.T, name string, tmpl *pbmesh.ProxyStateTemplate) {
-	requireExplicitDestinations(t, name, tmplResource, true)
+	requireExplicitDestinations(t, name, tmpl, true)
 }
 
-func requireExplicitDestinationsNotFound(t *testing.T, name string, tmplResource *pbresource.Resource) {
+func requireExplicitDestinationsNotFound(t resourcetest.T, name string, tmpl *pbmesh.ProxyStateTemplate) {
-	requireExplicitDestinations(t, name, tmplResource, false)
+	requireExplicitDestinations(t, name, tmpl, false)
 }
 
-func requireExplicitDestinations(t *testing.T, name string, tmplResource *pbresource.Resource, found bool) {
+func requireExplicitDestinations(t resourcetest.T, name string, tmpl *pbmesh.ProxyStateTemplate, found bool) {
 	t.Helper()
 
-	var tmpl pbmesh.ProxyStateTemplate
-	err := tmplResource.Data.UnmarshalTo(&tmpl)
-	require.NoError(t, err)
-
 	// Check outbound listener.
 	var foundListener bool
 	for _, l := range tmpl.ProxyState.Listeners {
@@ -885,16 +894,12 @@ func requireExplicitDestinations(t *testing.T, name string, tmplResource *pbreso
 
 	require.Equal(t, found, foundListener)
 
-	requireClustersAndEndpoints(t, name, &tmpl, found)
+	requireClustersAndEndpoints(t, name, tmpl, found)
 }
 
-func requireImplicitDestinationsFound(t *testing.T, name string, tmplResource *pbresource.Resource) {
+func requireImplicitDestinationsFound(t resourcetest.T, name string, tmpl *pbmesh.ProxyStateTemplate) {
 	t.Helper()
 
-	var tmpl pbmesh.ProxyStateTemplate
-	err := tmplResource.Data.UnmarshalTo(&tmpl)
-	require.NoError(t, err)
-
 	// Check outbound listener.
 	var foundListener bool
 	for _, l := range tmpl.ProxyState.Listeners {
@@ -932,10 +937,10 @@ func requireImplicitDestinationsFound(t *testing.T, name string, tmplResource *p
 	}
 	require.True(t, foundListener)
 
-	requireClustersAndEndpoints(t, name, &tmpl, true)
+	requireClustersAndEndpoints(t, name, tmpl, true)
 }
 
-func requireClustersAndEndpoints(t *testing.T, name string, tmpl *pbmesh.ProxyStateTemplate, found bool) {
+func requireClustersAndEndpoints(t resourcetest.T, name string, tmpl *pbmesh.ProxyStateTemplate, found bool) {
 	t.Helper()
 
 	var foundCluster bool
@@ -959,6 +964,16 @@ func requireClustersAndEndpoints(t *testing.T, name string, tmpl *pbmesh.ProxySt
 	require.Equal(t, found, foundEndpoints)
 }
 
+func (suite *controllerTestSuite) waitForProxyStateTemplateState(t *testing.T, id *pbresource.ID, verify func(resourcetest.T, *pbmesh.ProxyStateTemplate)) {
+	suite.client.WaitForResourceState(t, id, func(rt resourcetest.T, res *pbresource.Resource) {
+		var tmpl pbmesh.ProxyStateTemplate
+		err := res.Data.UnmarshalTo(&tmpl)
+		require.NoError(rt, err)
+
+		verify(rt, &tmpl)
+	})
+}
+
 func resourceID(rtype *pbresource.Type, name string, tenancy *pbresource.Tenancy) *pbresource.ID {
 	return &pbresource.ID{
 		Type:    rtype,